From 4ff0afd6c38bb30a4a5a4f6810ac597ea12c4123 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Thu, 27 Jun 2024 02:09:17 +0300 Subject: [PATCH 01/48] Add C++ code from unifrac's implementation --- R/unifrac_cpp/Makefile | 196 +++ R/unifrac_cpp/R_interface/README.html | 441 ++++++ R/unifrac_cpp/R_interface/README.md | 39 + R/unifrac_cpp/R_interface/rapi_test.R | 52 + R/unifrac_cpp/R_interface/test.biom | Bin 0 -> 33800 bytes R/unifrac_cpp/R_interface/test.tre | 1 + R/unifrac_cpp/affinity.hpp | 125 ++ R/unifrac_cpp/api.cpp | 1476 +++++++++++++++++++ R/unifrac_cpp/api.hpp | 577 ++++++++ R/unifrac_cpp/benchtest.sh | 17 + R/unifrac_cpp/biom.cpp | 324 +++++ R/unifrac_cpp/biom.hpp | 114 ++ R/unifrac_cpp/biom_interface.hpp | 72 + R/unifrac_cpp/capi_test.c | 71 + R/unifrac_cpp/cmd.cpp | 1 + R/unifrac_cpp/cmd.hpp | 32 + R/unifrac_cpp/faithpd.cpp | 84 ++ R/unifrac_cpp/skbio_alt.cpp | 617 ++++++++ R/unifrac_cpp/skbio_alt.hpp | 44 + R/unifrac_cpp/su.cpp | 492 +++++++ R/unifrac_cpp/su_R.cpp | 44 + R/unifrac_cpp/task_parameters.hpp | 35 + R/unifrac_cpp/test_api.cpp | 743 ++++++++++ R/unifrac_cpp/test_ska.cpp | 516 +++++++ R/unifrac_cpp/test_su.cpp | 1872 +++++++++++++++++++++++++ R/unifrac_cpp/tree.cpp | 462 ++++++ R/unifrac_cpp/tree.hpp | 138 ++ R/unifrac_cpp/unifrac.cpp | 494 +++++++ R/unifrac_cpp/unifrac.hpp | 112 ++ R/unifrac_cpp/unifrac_cmp.cpp | 395 ++++++ R/unifrac_cpp/unifrac_cmp.hpp | 38 + R/unifrac_cpp/unifrac_internal.cpp | 290 ++++ R/unifrac_cpp/unifrac_internal.hpp | 96 ++ R/unifrac_cpp/unifrac_task.cpp | 785 +++++++++++ R/unifrac_cpp/unifrac_task.hpp | 577 ++++++++ 35 files changed, 11372 insertions(+) create mode 100644 R/unifrac_cpp/Makefile create mode 100644 R/unifrac_cpp/R_interface/README.html create mode 100644 R/unifrac_cpp/R_interface/README.md create mode 100644 R/unifrac_cpp/R_interface/rapi_test.R create mode 100644 R/unifrac_cpp/R_interface/test.biom create mode 100644 R/unifrac_cpp/R_interface/test.tre create mode 100644 R/unifrac_cpp/affinity.hpp create mode 100644 R/unifrac_cpp/api.cpp create mode 100644 R/unifrac_cpp/api.hpp create mode 100644 R/unifrac_cpp/benchtest.sh create mode 100644 R/unifrac_cpp/biom.cpp create mode 100644 R/unifrac_cpp/biom.hpp create mode 100644 R/unifrac_cpp/biom_interface.hpp create mode 100644 R/unifrac_cpp/capi_test.c create mode 100644 R/unifrac_cpp/cmd.cpp create mode 100644 R/unifrac_cpp/cmd.hpp create mode 100644 R/unifrac_cpp/faithpd.cpp create mode 100644 R/unifrac_cpp/skbio_alt.cpp create mode 100644 R/unifrac_cpp/skbio_alt.hpp create mode 100644 R/unifrac_cpp/su.cpp create mode 100644 R/unifrac_cpp/su_R.cpp create mode 100644 R/unifrac_cpp/task_parameters.hpp create mode 100644 R/unifrac_cpp/test_api.cpp create mode 100644 R/unifrac_cpp/test_ska.cpp create mode 100644 R/unifrac_cpp/test_su.cpp create mode 100644 R/unifrac_cpp/tree.cpp create mode 100644 R/unifrac_cpp/tree.hpp create mode 100644 R/unifrac_cpp/unifrac.cpp create mode 100644 R/unifrac_cpp/unifrac.hpp create mode 100644 R/unifrac_cpp/unifrac_cmp.cpp create mode 100644 R/unifrac_cpp/unifrac_cmp.hpp create mode 100644 R/unifrac_cpp/unifrac_internal.cpp create mode 100644 R/unifrac_cpp/unifrac_internal.hpp create mode 100644 R/unifrac_cpp/unifrac_task.cpp create mode 100644 R/unifrac_cpp/unifrac_task.hpp diff --git a/R/unifrac_cpp/Makefile b/R/unifrac_cpp/Makefile new file mode 100644 index 000000000..0ecac3ec8 --- /dev/null +++ b/R/unifrac_cpp/Makefile @@ -0,0 +1,196 @@ +H5CXX := h5c++ + +PLATFORM := $(shell uname -s) +COMPILER := $(shell ($(H5CXX) -v 2>&1) | tr A-Z a-z ) + +ifdef DEBUG + OPT = -O0 -DDEBUG=1 --debug -g -ggdb +else + ifneq (,$(findstring gcc,$(COMPILER))) + OPT = -O4 + TGTFLAGS = -fwhole-program + else + OPT = -O3 + endif +endif + +ifeq ($(PREFIX),) + PREFIX := $(CONDA_PREFIX) +endif + +ifeq ($(PLATFORM),Darwin) + AVX2 := $(shell sysctl -a | grep -c AVX2) + LDDFLAGS = -dynamiclib -install_name @rpath/libssu.so +else + AVX2 := $(shell grep "^flags" /proc/cpuinfo | head -n 1 | grep -c avx2) + LDDFLAGS = -shared +endif + +EXEFLAGS = + +MPFLAG = -fopenmp + +LDDFLAGS += $(MPFLAG) +CPPFLAGS += $(MPFLAG) + +ifeq ($(PERFORMING_CONDA_BUILD),True) + CPPFLAGS += -mtune=generic +else + CPPFLAGS += -mfma -march=native +endif + +CPPFLAGS += -Wextra -Wno-unused-parameter + +ifeq ($(PLATFORM),Darwin) + BLASLIB=-llapacke -lcblas +else + BLASLIB=-lcblas +endif + + +LDDFLAGS += -L$(CONDA_PREFIX)/lib +CPPFLAGS += -Wall -std=c++11 -pedantic -I. $(OPT) -fPIC -L$(CONDA_PREFIX)/lib + +ifeq ($(PLATFORM),Darwin) + LDDFLAGS += -Wl,-rpath,$(PREFIX)/lib +else + LDDFLAGS += -Wl,-rpath-link,$(PREFIX)/lib +endif +BASE_LDDFLAGS = $(LDDFLAGS) + +R_LDFLAGS = -llz4 $(BLASLIB) + +UFCMP_LIBS=libssu_cpu.so +UFCMP_LINK=-lssu_cpu +ifdef ACC_CXX + # Tell the generic code we will be building the ACC code, too + CPPFLAGS += -DUNIFRAC_ENABLE_ACC=1 + + UFCMP_LIBS+= libssu_acc.so + UFCMP_LINK+= -lssu_acc + + R_LDFLAGS += -lssu_acc + + ACC_CPPFLAGS += -mp -acc + ACC_CPPFLAGS += -Wall -std=c++11 -pedantic -I. -fPIC -L$(CONDA_PREFIX)/lib + + ifdef DEBUG + ACC_OPT = -g + else + ACC_OPT = -fast + endif + ACC_CPPFLAGS += $(ACC_OPT) + + ifeq ($(PERFORMING_CONDA_BUILD),True) + ACC_CPPFLAGS += -ta=tesla:ccall + else + ACC_CPPFLAGS += -ta=tesla + endif + # optional info + ACC_CPPFLAGS += -Minfo=accel + + # use the GNU OMP library to avoid conflicts + ACC_LDDFLAGS = -shared -mp -acc -Wl,-rpath-link,$(PREFIX)/lib -L$(CONDA_PREFIX)/lib -lgomp -Bstatic_pgi + + ifeq ($(PERFORMING_CONDA_BUILD),True) + ACC_CPPFLAGS += -tp=px + endif +endif + +ifeq ($(PLATFORM),Darwin) + TEST_DEPS = -lssu +else + TEST_DEPS = -lssu -lssu_internal +endif + +test_su: test_su.cpp libssu.so + $(H5CXX) $(CPPFLAGS) $(EXEFLAGS) test_su.cpp -o test_su $(TEST_DEPS) -lpthread + +test_ska: test_ska.cpp libssu.so + $(H5CXX) $(CPPFLAGS) $(EXEFLAGS) test_ska.cpp -o test_ska $(TEST_DEPS) -lpthread + +test_api: test_api.cpp libssu.so + $(H5CXX) $(CPPFLAGS) $(EXEFLAGS) test_api.cpp -o test_api $(TEST_DEPS) -lpthread + +test: test_su test_ska test_api + # test = (su,ska,api) + +ssu: su.cpp libssu.so + $(H5CXX) $(CPPFLAGS) $(EXEFLAGS) su.cpp -o ssu -lssu -lpthread + cp ssu ${PREFIX}/bin/ + +faithpd: faithpd.cpp libssu.so + $(H5CXX) $(CPPFLAGS) $(EXEFLAGS) faithpd.cpp -o faithpd -lssu -lpthread + cp faithpd ${PREFIX}/bin/ + +main: ssu faithpd + # main == (ssu,faithpd) + +rapi_test: main + mkdir -p ~/.R + if [ -e ~/.R/Makevars ] ; \ + then \ + echo "WARNING: OVERWRITING ~/.R/Makevars" ; \ + echo "The original Makevars file has been copied to ~/.R/Makevars" ;\ + cp ~/.R/Makevars Makevars-original ; \ + fi; + echo CXX1X=h5c++ > ~/.R/Makevars + echo CXX=h5c++ >> ~/.R/Makevars + echo CC=h5c++ >> ~/.R/Makevars + echo LDFLAGS=$(R_LDFLAGS) >> ~/.R/Makevars + Rscript R_interface/rapi_test.R + +ifeq ($(PLATFORM),Darwin) + +# We never use ACC under MacOS, so keep it simple + +libssu.so: tree.o biom.o unifrac.o unifrac_internal.o unifrac_cmp_cpu.o cmd.o skbio_alt.o api.o + $(H5CXX) $(LDDFLAGS) -o libssu.so tree.o biom.o unifrac.o unifrac_internal.o unifrac_cmp_cpu.o cmd.o skbio_alt.o api.o -lc -lhdf5_cpp -llz4 $(BLASLIB) + cp libssu.so ${PREFIX}/lib/ + +else + +libssu.so: biom.o unifrac.o cmd.o api.o $(UFCMP_LIBS) libssu_internal.so + $(H5CXX) $(LDDFLAGS) -o libssu.so biom.o unifrac.o cmd.o api.o $(UFCMP_LINK) -lssu_internal -lc -llz4 + cp libssu.so ${PREFIX}/lib/ + +libssu_internal.so: tree.o skbio_alt.o unifrac_internal.o + $(H5CXX) $(LDDFLAGS) -o libssu_internal.so tree.o skbio_alt.o unifrac_internal.o -lc -lhdf5_cpp $(BLASLIB) + cp libssu_internal.so ${PREFIX}/lib/ + +libssu_cpu.so: unifrac_cmp_cpu.o libssu_internal.so + $(CXX) $(BASE_LDDFLAGS) -o libssu_cpu.so unifrac_cmp_cpu.o -lssu_internal -lc + cp libssu_cpu.so ${PREFIX}/lib/ + +libssu_acc.so: unifrac_cmp_acc.o libssu_internal.so + $(ACC_CXX) $(ACC_LDDFLAGS) -o libssu_acc.so unifrac_cmp_acc.o -lssu_internal -lc + cp libssu_acc.so ${PREFIX}/lib/ + +endif + +api: libssu.so + # api == libssu.so + +capi_test: api + gcc -std=c99 capi_test.c -lssu -L${PREFIX}/lib -Wl,-rpath,${PREFIX}/lib -o capi_test + export LD_LIBRARY_PATH="${PREFIX}/lib":"./capi_test" + +api.o: api.cpp api.hpp unifrac.hpp skbio_alt.hpp biom.hpp tree.hpp + $(H5CXX) $(CPPFLAGS) api.cpp -c -o api.o -fPIC + +unifrac.o: unifrac.cpp unifrac.hpp unifrac_internal.hpp unifrac_cmp.hpp biom_interface.hpp tree.hpp + $(CXX) $(CPPFLAGS) -c $< -o $@ + +unifrac_cmp_cpu.o: unifrac_cmp.cpp unifrac_cmp.hpp unifrac_internal.hpp unifrac.hpp unifrac_task.cpp unifrac_task.hpp biom_interface.hpp tree.hpp + $(CXX) $(CPPFLAGS) -Wno-unknown-pragmas -c $< -o $@ + +unifrac_cmp_acc.o: unifrac_cmp.cpp unifrac_cmp.hpp unifrac_internal.hpp unifrac.hpp unifrac_task.cpp unifrac_task.hpp biom_interface.hpp tree.hpp + $(ACC_CXX) $(ACC_CPPFLAGS) -c $< -o $@ + + +%.o: %.cpp %.hpp + $(H5CXX) $(CPPFLAGS) -c $< -o $@ + +clean: + -rm -f *.o *.so ssu faithpd test_su test_api test_ska + diff --git a/R/unifrac_cpp/R_interface/README.html b/R/unifrac_cpp/R_interface/README.html new file mode 100644 index 000000000..d83388128 --- /dev/null +++ b/R/unifrac_cpp/R_interface/README.html @@ -0,0 +1,441 @@ + + + + + + + + + + + + + +README + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + +
+

R interface for Strided State Unifrac

+

This provides an R interface for Unweighted Unifrac. This interface +works using R’s Rcpp library. To load this, in R use +library(Rcpp) and sourceCpp("su_R.cpp"). The +Unifrac method takes in three arguments: a file path to an HDF5 +formatted BIOM table, a filepath to a newick formatted tree file, and +the number of threads to be used It is expected that the observations +described in the BIOM table correspond to a subset of the tips of the +input tree. The method returns a list containing an int +n_samples, denoting the number of samples in the table, a +boolean is_upper_triangle, denoting whether +Unifrac generated a square matrix and if it has returned the upper +triangle , an int cf_size, denoting the size +of the condensed form of the matrix, and c_form, an array +representation of the condensed form of the matrix, obtained by taking +the upper triangle.

+
> library(Rcpp)
+> sourceCpp("su_R.cpp")
+> table = "../test.biom"
+> tree = "../test.tre"
+> nthreads = 2
+> unif = unifrac(table, tree, nthreads)
+> unif
+$n_samples
+[1] 6
+
+$is_sqaure
+[1] TRUE
+
+$cf_size
+[1] 15
+
+$c_form
+ [1] 0.2000000 0.5714286 0.6000000 0.5000000 0.2000000 0.4285714 0.6666667
+ [8] 0.6000000 0.3333333 0.7142857 0.8571429 0.4285714 0.3333333 0.4000000
+[15] 0.6000000
+
+
+ + + + +
+ + + + + + + + + + + + + + + diff --git a/R/unifrac_cpp/R_interface/README.md b/R/unifrac_cpp/R_interface/README.md new file mode 100644 index 000000000..3fb7e2d51 --- /dev/null +++ b/R/unifrac_cpp/R_interface/README.md @@ -0,0 +1,39 @@ +# R interface for Strided State Unifrac + +This provides an R interface for Unweighted Unifrac. This interface works using +R's Rcpp library. To load this, in R use `library(Rcpp)` and +`sourceCpp("su_R.cpp")`. The Unifrac method takes in three arguments: a file +path to an HDF5 formatted BIOM table, a filepath to a newick formatted tree +file, and the number of threads to be used It is expected that the observations +described in the BIOM table correspond to a subset of the tips of the input +tree. The method returns a list containing an `int` `n_samples`, denoting the +number of samples in the table, a `boolean` `is_upper_triangle`, denoting +whether Unifrac generated a square matrix and if it has returned the upper +triangle , an `int` `cf_size`, denoting the size of the condensed form of the +matrix, and `c_form`, an array representation of the condensed form of the +matrix, obtained by taking the upper triangle. + +```R +> library(Rcpp) +> sourceCpp("su_R.cpp") +> table = "../test.biom" +> tree = "../test.tre" +> nthreads = 2 +> unif = unifrac(table, tree, nthreads) +> unif +$n_samples +[1] 6 + +$is_sqaure +[1] TRUE + +$cf_size +[1] 15 + +$c_form + [1] 0.2000000 0.5714286 0.6000000 0.5000000 0.2000000 0.4285714 0.6666667 + [8] 0.6000000 0.3333333 0.7142857 0.8571429 0.4285714 0.3333333 0.4000000 +[15] 0.6000000 + +``` + diff --git a/R/unifrac_cpp/R_interface/rapi_test.R b/R/unifrac_cpp/R_interface/rapi_test.R new file mode 100644 index 000000000..97dde62b7 --- /dev/null +++ b/R/unifrac_cpp/R_interface/rapi_test.R @@ -0,0 +1,52 @@ +library(Rcpp) + +equals <- function(x, y, msg){ + if (x!=y) + stop(msg) + + +} +aboutEquals <- function(x, y, msg){ + if((x-y)>0.005) + stop(msg) + + +} +source = "R/unifrac_cpp/su_R.cpp" +sourceCpp(source) +table = "test.biom" +tree = "test.tre" +nthreads = 1 + +print('Testing UniFrac..') +unif = unifrac(table, tree, nthreads) + +exp = c(0.2000000, 0.5714286, 0.6000000, 0.5000000, 0.2000000, + 0.4285714, 0.6666667, 0.6000000, 0.3333333, 0.7142857, + 0.8571429, 0.4285714, 0.3333333, 0.4000000, 0.6000000) + +equals(unif["n_samples"][[1]], 6, "n_samples != 6") +equals(unif["cf_size"][[1]], 15, "cf_size != 15") +equals(unif["is_upper_triangle"][[1]], TRUE, "is_upper_triagnle != TRUE") + + +for ( i in 1:15){ + aboutEquals(unif["c_form"][[1]][i], exp[i], "Output not as expected") +} +print('Success.') + +print('Testing Faith PD..') + +faith = faith_pd(table, tree) + +exp = c(4, 5, 6, 3, 2, 5) + +equals(faith["n_samples"][[1]], 6, "n_samples != 6") +for ( i in 1:6){ + aboutEquals(faith["faith_pd"][[1]][i], exp[i], "Output not as expected") +} + +print('Success.') + +print('All tests pass') + diff --git a/R/unifrac_cpp/R_interface/test.biom b/R/unifrac_cpp/R_interface/test.biom new file mode 100644 index 0000000000000000000000000000000000000000..b3c019bf8515f8e05176fac398e0cf114281bc82 GIT binary patch literal 33800 zcmeI5O-vg{6oB8_2HZdbsj8Ixc3Y(>J(yr{g2N#pzqo2dXhS1%N$dp!mH;DTBB!E} zQL2hYtrV$J6;(Z?Kap}o>Lu!lG^vyxD&^1vm2yCB#i@rxttu*2N@sT7TjIs*B?(S2 zek0-8nR)YO=9?dew=?GB$dO~~*KJt`()s=1gF5q(J|3d;f2d+8OzBa4(9oYk|GGy# zXs`}sT0O?sp?=+~A~Gn~{qE7DAS6jY`Irr&sDSiPJp1KAAae9zmuzVdHdjag4vrrT zLedzEr_ROFiR6eGH5MBl9g545rCgW&{=Kj=})GHW9h(nYRHUs z)wfBAQN85UtN|0Z&{r#->jCwhi>Jm+v~{8lsu}HO0h*>-OWk4jiT`3f*PHbmh>yfm zv2?sIV4RnV+}d3wWT>7#@w}*SgQ>4K6*oIS&?l5GL*JH1^i|~Ot2Em^HW)KIWV!8? zI;5*qp3u=`K51fQtr5gVM&38WOZ1_9E&7@uFPYJLN8@k?3z(B2gmYUr2e&CX(xXEB zSuO!!8-;Tl97#A1<3WI(KAV`kKFJRK8LC`Rc6spR-*U@?~FvrYmIe#hYx$4yze zAPPtMuD*Vf?~B*POewd4W`qhZM%0HiR5rV ztx=81)BuRtrW{Tah-cI<0-B$4-b>fq6buELS_94dx`TV$f_vIREsZU~a7$~eN~d{D zxH>w{p6Gt>Y>4#Q;`dOPe3WmY_SoW^)y9j~QUH_>5+5q(q*?utSVNS{jo$<7^TzLm z4SC~R#HyZ8Pc#0u1IX1ML}`F9DYRnvfeA1HCcp%k025#WOn?b60Vco%m;k`Z&J#y8 z@Sv>_;0pc$PIdshJ&WvAnRgUDlDmrGnUm#7)yW=sceCPgf2;D{?1A@pDLLX?o;1t1 z)pmQJjwtHTPE2IT_LmsxOZ3LaK=`BS)S~^)vLjLk7hkOdK4?I>8=dNb=Jy9fxZ8t)5l=a#Dyhsj z=~8r~e!zNP6RL*LLo}pTh*+8K@hY{|pqmf)I!8RqbPf9A{X_DN3fWctsMqW9%0i#j35|Cn?dD`;_aQtRKBECjSkGBEmEPF`YPB6?e$T(M*5|FaajO z1eieS5>We4ueDK`-d?yZv=0q86_%s@WoBPtxwVp-Inx`3$=ZLSmk!?64&7R)>DpiQ zMfj6V*-N*6xjC2~zY>DU4{ExFn308JdVi&{N$;;Pb~U_m`=g&O-fo|$-wM8O;dQA; zIG?dspZ6JO^!X|Sv@ao$IrF=Sj?6{D+o)FtV-uaB{ZohLzx(RJuaosK5cZrlYoKkc zYWi^ot{s^DtmD?-)$NH54WBkXytxBrRpEU?Nu_bD;pZPfJ06+1yltxX>)6ocZQ$Dp zI+pD*PRX)I^m)y=pjUz?3auh(0E^f3{*K@N`sTsEcNgvrPV5Nn-(Ba0&;RiJxmLQ8 z+rb2w025#WOkmjr9^VdCC#*E$T(mH>C>xS)mbC)*rwE=-lcpRpPvx;c^_b-q>%++o z^{PXGXC3TNRs$UJaWoTP0!)AjFaah|x&++zPjv5qe6Ki}U!?uhRGY#mi?X&KOW)r; zpZycr)$Bc;we6pXlhpuw1`aa;Ccp%k025#WB~QR@|3vpZ$)|HrPWQ`Kzz%iiu%ar9 zq}PUd>`>449Mo$LJ$=QteN;He4N@1wT&6jn+5&^tz9IoeNfC-c?0k{1V{XYoui+CTUepq3YMOoXA)v|xWui0GE z(f`@@PsGV;z>*vs!33B96JP>NfC-d30k{1V-A^T--h&qLK1$Cy1yvSFX2a+H9(1Qe zN5A}LZUS+#DxRCnAtt~Cm;e)C0!&~H5O6wAYCjL>cAlTlldgfm@Dxq3pC_erboaBl mO6go?S8ntZhn&W*{TwNe-zP>~78#w!?=+r6CI~A%e*XvBCayjJ literal 0 HcmV?d00001 diff --git a/R/unifrac_cpp/R_interface/test.tre b/R/unifrac_cpp/R_interface/test.tre new file mode 100644 index 000000000..1ba14a5b6 --- /dev/null +++ b/R/unifrac_cpp/R_interface/test.tre @@ -0,0 +1 @@ +(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1); diff --git a/R/unifrac_cpp/affinity.hpp b/R/unifrac_cpp/affinity.hpp new file mode 100644 index 000000000..55d62c983 --- /dev/null +++ b/R/unifrac_cpp/affinity.hpp @@ -0,0 +1,125 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#ifdef __LINUX__ +#include +#endif + + +#ifdef __APPLE__ +#include +#include +#include +#include +#include + +// OSX code adapted from +// http://yyshen.github.io/2015/01/18/binding_threads_to_cores_osx.html +// these macros and methods don't exist on OSX + +#define SYSCTL_CORE_COUNT "machdep.cpu.core_count" + +typedef struct cpu_set { + uint32_t count; +} cpu_set_t; + +static inline void +CPU_ZERO(cpu_set_t *cs) { cs->count = 0; } + +static inline void +CPU_SET(int num, cpu_set_t *cs) { cs->count |= (1 << num); } + +static inline int +CPU_ISSET(int num, cpu_set_t *cs) { return (cs->count & (1 << num)); } + +static inline int +CPU_COUNT(cpu_set_t *cs) { return __builtin_popcount(cs->count); } + +#define CPU_SETSIZE 32 + +static int sched_getaffinity(pid_t pid, size_t cpu_size, cpu_set_t *cpu_set) +{ + int32_t core_count = 0; + size_t len = sizeof(core_count); + int ret = sysctlbyname(SYSCTL_CORE_COUNT, &core_count, &len, 0, 0); + if (ret) { + return -1; + } + cpu_set->count = 0; + for (int i = 0; i < core_count; i++) { + cpu_set->count |= (1 << i); + } + + return 0; +} + +static int pthread_setaffinity_np(pthread_t thread, size_t cpu_size, + cpu_set_t *cpu_set) +{ + thread_port_t mach_thread; + int core = 0; + + for (core = 0; core < 8 * cpu_size; core++) { + if (CPU_ISSET(core, cpu_set)) break; + } + thread_affinity_policy_data_t policy = { core }; + mach_thread = pthread_mach_thread_np(thread); + thread_policy_set(mach_thread, THREAD_AFFINITY_POLICY, + (thread_policy_t)&policy, 1); + return 0; +} + +#endif + +#define handle_error_en(en, msg) \ + do { errno = en; perror(msg); exit(EXIT_FAILURE); } while (0) + +static int bind_to_core(int core) { + /* bind the calling thread to the requested core + * + * The use of this method is for better NUMA utilization. The + * default NUMA policy is local, where memory is allocated on the NUMA node + * relative to the core if possible. The intention with this method is to + * bind to a core first, and then allocate memory. A beneficial side effect + * is that threads should not hop between cores either. + * + * This method is cgroup safe. + */ + // https://stackoverflow.com/a/11583550/19741 + // http://blog.saliya.org/2015/07/get-and-set-process-affinity-in-c.html + pthread_t thread = pthread_self(); + pid_t pid = getpid(); + + cpu_set_t current_set, new_set; + int j, ret; + + CPU_ZERO(¤t_set); + CPU_ZERO(&new_set); + + ret = sched_getaffinity(pid, sizeof(current_set), ¤t_set); + + // find which core in our cpu_set corresponds to the callers + // request + int target = -1; + for(j = 0; j < CPU_SETSIZE; j++) { + if(CPU_ISSET(j, ¤t_set)) { + target++; + } + if(target == core) + break; + } + + if(target != core) { + fprintf(stderr, "Unable to bind this thread to core %d. Are sufficient processors available?", thread); + return -1; + } + + CPU_SET(j, &new_set); + int serr = pthread_setaffinity_np(thread, sizeof(new_set), &new_set); + return serr; +} diff --git a/R/unifrac_cpp/api.cpp b/R/unifrac_cpp/api.cpp new file mode 100644 index 000000000..d6b1995dc --- /dev/null +++ b/R/unifrac_cpp/api.cpp @@ -0,0 +1,1476 @@ +#include "api.hpp" +#include "biom.hpp" +#include "tree.hpp" +#include "unifrac.hpp" +#include "skbio_alt.hpp" +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#define MMAP_FD_MASK 0x0fff +#define MMAP_FLAG 0x1000 + +/* O_NOATIME is defined at fcntl.h when supported */ +#ifndef O_NOATIME +#define O_NOATIME 0 +#endif + + +#define CHECK_FILE(filename, err) if(!is_file_exists(filename)) { \ + return err; \ + } + +#define SET_METHOD(requested_method, err) Method method; \ + if(std::strcmp(requested_method, "unweighted") == 0) \ + method = unweighted; \ + else if(std::strcmp(requested_method, "weighted_normalized") == 0) \ + method = weighted_normalized; \ + else if(std::strcmp(requested_method, "weighted_unnormalized") == 0) \ + method = weighted_unnormalized; \ + else if(std::strcmp(requested_method, "generalized") == 0) \ + method = generalized; \ + else if(std::strcmp(requested_method, "unweighted_fp32") == 0) \ + method = unweighted_fp32; \ + else if(std::strcmp(requested_method, "weighted_normalized_fp32") == 0) \ + method = weighted_normalized_fp32; \ + else if(std::strcmp(requested_method, "weighted_unnormalized_fp32") == 0) \ + method = weighted_unnormalized_fp32; \ + else if(std::strcmp(requested_method, "generalized_fp32") == 0) \ + method = generalized_fp32; \ + else { \ + return err; \ + } + +#define PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) std::ifstream ifs(tree_filename); \ + std::string content = std::string(std::istreambuf_iterator(ifs), \ + std::istreambuf_iterator()); \ + su::BPTree tree = su::BPTree(content); \ + su::biom table = su::biom(biom_filename); \ + if(table.n_samples <= 0 | table.n_obs <= 0) { \ + return table_empty; \ + } \ + std::string bad_id = su::test_table_ids_are_subset_of_tree(table, tree); \ + if(bad_id != "") { \ + return table_and_tree_do_not_overlap; \ + } \ + std::unordered_set to_keep(table.obs_ids.begin(), \ + table.obs_ids.end()); \ + su::BPTree tree_sheared = tree.shear(to_keep).collapse(); + + +using namespace su; +using namespace std; + +// https://stackoverflow.com/a/19841704/19741 +bool is_file_exists(const char *fileName) { + std::ifstream infile(fileName); + return infile.good(); +} + + +void destroy_stripes(vector &dm_stripes, vector &dm_stripes_total, unsigned int n_samples, + unsigned int stripe_start, unsigned int stripe_stop) { + unsigned int n_rotations = (n_samples + 1) / 2; + + if(stripe_stop == 0) { + for(unsigned int i = 0; i < n_rotations; i++) { + free(dm_stripes[i]); + if(dm_stripes_total[i] != NULL) + free(dm_stripes_total[i]); + } + } else { + // if a stripe_stop is specified, and if we're in the stripe window, do not free + // dm_stripes. this is done as the pointers in dm_stripes are assigned to the partial_mat_t + // and subsequently freed in destroy_partial_mat. but, we do need to free dm_stripes_total + // if appropriate + for(unsigned int i = stripe_start; i < stripe_stop; i++) { + if(dm_stripes_total[i] != NULL) + free(dm_stripes_total[i]); + } + } +} + + +void initialize_mat(mat_t* &result, biom &table, bool is_upper_triangle) { + result = (mat_t*)malloc(sizeof(mat)); + result->n_samples = table.n_samples; + + result->cf_size = su::comb_2(table.n_samples); + result->is_upper_triangle = is_upper_triangle; + result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); + result->condensed_form = (double*)malloc(sizeof(double) * su::comb_2(table.n_samples)); + + for(unsigned int i = 0; i < result->n_samples; i++) { + size_t len = table.sample_ids[i].length(); + result->sample_ids[i] = (char*)malloc(sizeof(char) * len + 1); + table.sample_ids[i].copy(result->sample_ids[i], len); + result->sample_ids[i][len] = '\0'; + } +} + +void initialize_results_vec(r_vec* &result, biom& table){ + // Stores results for Faith PD + result = (r_vec*)malloc(sizeof(results_vec)); + result->n_samples = table.n_samples; + result->values = (double*)malloc(sizeof(double) * result->n_samples); + result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); + + for(unsigned int i = 0; i < result->n_samples; i++) { + size_t len = table.sample_ids[i].length(); + result->sample_ids[i] = (char*)malloc(sizeof(char) * len + 1); + table.sample_ids[i].copy(result->sample_ids[i], len); + result->sample_ids[i][len] = '\0'; + result->values[i] = 0; + } + +} + +void initialize_mat_no_biom(mat_t* &result, char** sample_ids, unsigned int n_samples, bool is_upper_triangle) { + result = (mat_t*)malloc(sizeof(mat)); + result->n_samples = n_samples; + + result->cf_size = su::comb_2(n_samples); + result->is_upper_triangle = is_upper_triangle; + result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); + result->condensed_form = (double*)malloc(sizeof(double) * su::comb_2(n_samples)); + + for(unsigned int i = 0; i < n_samples; i++) { + result->sample_ids[i] = strdup(sample_ids[i]); + } +} + +template +void initialize_mat_full_no_biom_T(TMat* &result, const char* const * sample_ids, unsigned int n_samples, + const char *mmap_dir /* if NULL or "", use malloc */) { + result = (TMat*)malloc(sizeof(mat)); + result->n_samples = n_samples; + + uint64_t n_samples_64 = result->n_samples; // force 64bit to avoit overflow problems + + result->sample_ids = (char**)malloc(sizeof(char*) * n_samples_64); + result->flags=0; + + if (mmap_dir!=NULL) { + if (mmap_dir[0]==0) mmap_dir = NULL; // easier to have a simple test going on + } + + uint64_t msize = sizeof(TReal) * n_samples_64 * n_samples_64; + if (mmap_dir==NULL) { + result->matrix = (TReal*)malloc(msize); + } else { + std::string mmap_template(mmap_dir); + mmap_template+="/su_mmap_XXXXXX"; + // note: mkostemp will update mmap_template in place + int fd=mkostemp((char *) mmap_template.c_str(), O_NOATIME ); + if (fd<0) { + result->matrix = NULL; + // leave error handling to the caller + } else { + // remove the file name, so it will be destroyed on close + unlink(mmap_template.c_str()); + // make it big enough + ftruncate(fd,msize); + // now can be used, just like a malloc-ed buffer + result->matrix = (TReal*)mmap(NULL, msize,PROT_READ|PROT_WRITE, MAP_SHARED|MAP_NORESERVE, fd, 0); + result->flags=(uint32_t(fd) & MMAP_FD_MASK) | MMAP_FLAG; + } + } + + for(unsigned int i = 0; i < n_samples; i++) { + result->sample_ids[i] = strdup(sample_ids[i]); + } +} + +void initialize_partial_mat(partial_mat_t* &result, biom &table, std::vector &dm_stripes, + unsigned int stripe_start, unsigned int stripe_stop, bool is_upper_triangle) { + result = (partial_mat_t*)malloc(sizeof(partial_mat)); + result->n_samples = table.n_samples; + + result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); + for(unsigned int i = 0; i < result->n_samples; i++) { + size_t len = table.sample_ids[i].length(); + result->sample_ids[i] = (char*)malloc(sizeof(char) * len + 1); + table.sample_ids[i].copy(result->sample_ids[i], len); + result->sample_ids[i][len] = '\0'; + } + + result->stripes = (double**)malloc(sizeof(double*) * (stripe_stop - stripe_start)); + result->stripe_start = stripe_start; + result->stripe_stop = stripe_stop; + result->is_upper_triangle = is_upper_triangle; + result->stripe_total = dm_stripes.size(); + + for(unsigned int i = stripe_start; i < stripe_stop; i++) { + result->stripes[i - stripe_start] = dm_stripes[i]; + } +} + +void destroy_results_vec(r_vec** result) { + // for Faith PD + for(unsigned int i = 0; i < (*result)->n_samples; i++) { + free((*result)->sample_ids[i]); + }; + free((*result)->sample_ids); + free((*result)->values); + free(*result); +} + +void destroy_mat(mat_t** result) { + for(unsigned int i = 0; i < (*result)->n_samples; i++) { + free((*result)->sample_ids[i]); + }; + free((*result)->sample_ids); + if (((*result)->condensed_form)!=NULL) { + free((*result)->condensed_form); + } + free(*result); +} + +template +inline void destroy_mat_full_T(TMat** result) { + for(uint32_t i = 0; i < (*result)->n_samples; i++) { + free((*result)->sample_ids[i]); + }; + free((*result)->sample_ids); + if (((*result)->matrix)!=NULL) { + if (((*result)->flags & MMAP_FLAG) == 0) { + free((*result)->matrix); + } else { + uint64_t n_samples = (*result)->n_samples; + munmap((*result)->matrix, sizeof(TReal)*n_samples*n_samples); + + int fd = (*result)->flags & MMAP_FD_MASK; + close(fd); + } + (*result)->matrix=NULL; + } + free(*result); +} + + +void destroy_mat_full_fp64(mat_full_fp64_t** result) { + destroy_mat_full_T(result); +} + +void destroy_mat_full_fp32(mat_full_fp32_t** result) { + destroy_mat_full_T(result); +} + +void destroy_partial_mat(partial_mat_t** result) { + for(unsigned int i = 0; i < (*result)->n_samples; i++) { + if((*result)->sample_ids[i] != NULL) + free((*result)->sample_ids[i]); + }; + if((*result)->sample_ids != NULL) + free((*result)->sample_ids); + + unsigned int n_stripes = (*result)->stripe_stop - (*result)->stripe_start; + for(unsigned int i = 0; i < n_stripes; i++) + if((*result)->stripes[i] != NULL) + free((*result)->stripes[i]); + if((*result)->stripes != NULL) + free((*result)->stripes); + + free(*result); +} + +void destroy_partial_dyn_mat(partial_dyn_mat_t** result) { + for(unsigned int i = 0; i < (*result)->n_samples; i++) { + if((*result)->sample_ids[i] != NULL) + free((*result)->sample_ids[i]); + }; + if((*result)->sample_ids != NULL) + free((*result)->sample_ids); + + unsigned int n_stripes = (*result)->stripe_stop - (*result)->stripe_start; + for(unsigned int i = 0; i < n_stripes; i++) + if((*result)->stripes[i] != NULL) + free((*result)->stripes[i]); + if((*result)->stripes != NULL) + free((*result)->stripes); + if((*result)->offsets != NULL) + free((*result)->offsets); + if((*result)->filename != NULL) + free((*result)->filename); + + free(*result); +} + + +void set_tasks(std::vector &tasks, + double alpha, + unsigned int n_samples, + unsigned int stripe_start, + unsigned int stripe_stop, + bool bypass_tips, + unsigned int nthreads) { + + // compute from start to the max possible stripe if stop doesn't make sense + if(stripe_stop <= stripe_start) + stripe_stop = (n_samples + 1) / 2; + + /* chunking strategy is to balance as much as possible. eg if there are 15 stripes + * and 4 threads, our goal is to assign 4 stripes to 3 threads, and 3 stripes to one thread. + * + * we use the remaining the chunksize for bins which cannot be full maximally + */ + unsigned int fullchunk = ((stripe_stop - stripe_start) + nthreads - 1) / nthreads; // this computes the ceiling + unsigned int smallchunk = (stripe_stop - stripe_start) / nthreads; + + unsigned int n_fullbins = (stripe_stop - stripe_start) % nthreads; + if(n_fullbins == 0) + n_fullbins = nthreads; + + unsigned int start = stripe_start; + + for(unsigned int tid = 0; tid < nthreads; tid++) { + tasks[tid].tid = tid; + tasks[tid].start = start; // stripe start + tasks[tid].bypass_tips = bypass_tips; + + if(tid < n_fullbins) { + tasks[tid].stop = start + fullchunk; // stripe end + start = start + fullchunk; + } else { + tasks[tid].stop = start + smallchunk; // stripe end + start = start + smallchunk; + } + + tasks[tid].n_samples = n_samples; + tasks[tid].g_unifrac_alpha = alpha; + } +} + +compute_status partial(const char* biom_filename, const char* tree_filename, + const char* unifrac_method, bool variance_adjust, double alpha, bool bypass_tips, + unsigned int nthreads, unsigned int stripe_start, unsigned int stripe_stop, + partial_mat_t** result) { + + CHECK_FILE(biom_filename, table_missing) + CHECK_FILE(tree_filename, tree_missing) + SET_METHOD(unifrac_method, unknown_method) + PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) + + // we resize to the largest number of possible stripes even if only computing + // partial, however we do not allocate arrays for non-computed stripes so + // there is a little memory waste here but should be on the order of + // 8 bytes * N samples per vector. + std::vector dm_stripes((table.n_samples + 1) / 2); + std::vector dm_stripes_total((table.n_samples + 1) / 2); + + if(nthreads > dm_stripes.size()) { + fprintf(stderr, "More threads were requested than stripes. Using %d threads.\n", dm_stripes.size()); + nthreads = dm_stripes.size(); + } + + std::vector tasks(nthreads); + std::vector threads(nthreads); + + if(((table.n_samples + 1) / 2) < stripe_stop) { + fprintf(stderr, "Stopping stripe is out-of-bounds, max %d\n", (table.n_samples + 1) / 2); + exit(EXIT_FAILURE); + } + + set_tasks(tasks, alpha, table.n_samples, stripe_start, stripe_stop, bypass_tips, nthreads); + su::process_stripes(table, tree_sheared, method, variance_adjust, dm_stripes, dm_stripes_total, threads, tasks); + + initialize_partial_mat(*result, table, dm_stripes, stripe_start, stripe_stop, true); // true -> is_upper_triangle + destroy_stripes(dm_stripes, dm_stripes_total, table.n_samples, stripe_start, stripe_stop); + + return okay; +} + +compute_status faith_pd_one_off(const char* biom_filename, const char* tree_filename, + r_vec** result){ + CHECK_FILE(biom_filename, table_missing) + CHECK_FILE(tree_filename, tree_missing) + PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) + + initialize_results_vec(*result, table); + + // compute faithpd + su::faith_pd(table, tree_sheared, std::ref((*result)->values)); + + return okay; +} + +compute_status one_off(const char* biom_filename, const char* tree_filename, + const char* unifrac_method, bool variance_adjust, double alpha, + bool bypass_tips, unsigned int nthreads, mat_t** result) { + + CHECK_FILE(biom_filename, table_missing) + CHECK_FILE(tree_filename, tree_missing) + SET_METHOD(unifrac_method, unknown_method) + PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) + + const unsigned int stripe_stop = (table.n_samples + 1) / 2; + std::vector dm_stripes(stripe_stop); + std::vector dm_stripes_total(stripe_stop); + + if(nthreads > dm_stripes.size()) { + fprintf(stderr, "More threads were requested than stripes. Using %d threads.\n", dm_stripes.size()); + nthreads = dm_stripes.size(); + } + + std::vector tasks(nthreads); + std::vector threads(nthreads); + + set_tasks(tasks, alpha, table.n_samples, 0, stripe_stop, bypass_tips, nthreads); + su::process_stripes(table, tree_sheared, method, variance_adjust, dm_stripes, dm_stripes_total, threads, tasks); + + initialize_mat(*result, table, true); // true -> is_upper_triangle + for(unsigned int tid = 0; tid < threads.size(); tid++) { + threads[tid] = std::thread(su::stripes_to_condensed_form, + std::ref(dm_stripes), + table.n_samples, + std::ref((*result)->condensed_form), + tasks[tid].start, + tasks[tid].stop); + } + for(unsigned int tid = 0; tid < threads.size(); tid++) { + threads[tid].join(); + } + + destroy_stripes(dm_stripes, dm_stripes_total, table.n_samples, 0, 0); + + return okay; +} + +// TMat mat_full_fp32_t +template +compute_status one_off_matrix_T(const char* biom_filename, const char* tree_filename, + const char* unifrac_method, bool variance_adjust, double alpha, + bool bypass_tips, unsigned int nthreads, + const char *mmap_dir, + TMat** result) { + if (mmap_dir!=NULL) { + if (mmap_dir[0]==0) mmap_dir = NULL; // easier to have a simple test going on + } + + CHECK_FILE(biom_filename, table_missing) + CHECK_FILE(tree_filename, tree_missing) + SET_METHOD(unifrac_method, unknown_method) + PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) + + const unsigned int stripe_stop = (table.n_samples + 1) / 2; + partial_mat_t *partial_mat = NULL; + + { + std::vector dm_stripes(stripe_stop); + std::vector dm_stripes_total(stripe_stop); + + std::vector tasks(nthreads); + std::vector threads(nthreads); + + set_tasks(tasks, alpha, table.n_samples, 0, stripe_stop, bypass_tips, nthreads); + su::process_stripes(table, tree_sheared, method, variance_adjust, dm_stripes, dm_stripes_total, threads, tasks); + + initialize_partial_mat(partial_mat, table, dm_stripes, 0, stripe_stop, true); // true -> is_upper_triangle + if ((partial_mat==NULL) || (partial_mat->stripes==NULL) || (partial_mat->sample_ids==NULL) ) { + fprintf(stderr, "Memory allocation error! (initialize_partial_mat)\n"); + exit(EXIT_FAILURE); + } + destroy_stripes(dm_stripes, dm_stripes_total, table.n_samples, 0, stripe_stop); + } + + initialize_mat_full_no_biom_T(*result, partial_mat->sample_ids, partial_mat->n_samples,mmap_dir); + + if (((*result)==NULL) || ((*result)->matrix==NULL) || ((*result)->sample_ids==NULL) ) { + fprintf(stderr, "Memory allocation error! (initialize_mat)\n"); + exit(EXIT_FAILURE); + } + + + { + MemoryStripes ps(partial_mat->stripes); + const uint32_t tile_size = (mmap_dir==NULL) ? \ + (128/sizeof(TReal)) : /* keep it small for memory access, to fit in chip cache */ \ + (4096/sizeof(TReal)); /* make it larger for mmap, as the limiting factor is swapping */ + su::stripes_to_matrix_T(ps, partial_mat->n_samples, partial_mat->stripe_total, (*result)->matrix, tile_size); + } + destroy_partial_mat(&partial_mat); + + return okay; +} + + +compute_status one_off_matrix(const char* biom_filename, const char* tree_filename, + const char* unifrac_method, bool variance_adjust, double alpha, + bool bypass_tips, unsigned int nthreads, + const char *mmap_dir, + mat_full_fp64_t** result) { + return one_off_matrix_T(biom_filename,tree_filename,unifrac_method,variance_adjust,alpha,bypass_tips,nthreads,mmap_dir,result); +} + +compute_status one_off_matrix_fp32(const char* biom_filename, const char* tree_filename, + const char* unifrac_method, bool variance_adjust, double alpha, + bool bypass_tips, unsigned int nthreads, + const char *mmap_dir, + mat_full_fp32_t** result) { + return one_off_matrix_T(biom_filename,tree_filename,unifrac_method,variance_adjust,alpha,bypass_tips,nthreads,mmap_dir,result); +} + +inline compute_status is_fp64(const std::string &method_string, const std::string &format_string, bool &fp64) { + if (format_string == "hdf5_fp32") { + fp64 = false; + } else if (format_string == "hdf5_fp64") { + fp64 = true; + } else if (format_string == "hdf5") { + if ((method_string=="unweighted_fp32") || (method_string=="weighted_normalized_fp32") || (method_string=="weighted_unnormalized_fp32") || (method_string=="generalized_fp32")) { + fp64 = false; + } else if ((method_string=="unweighted") || (method_string=="weighted_normalized") || (method_string=="weighted_unnormalized") || (method_string=="generalized")) { + fp64 = true; + } else { + return unknown_method; + } + } else { + return unknown_method; + } + + return okay; +} + + +compute_status unifrac_to_file(const char* biom_filename, const char* tree_filename, const char* out_filename, + const char* unifrac_method, bool variance_adjust, double alpha, + bool bypass_tips, unsigned int threads, const char* format, + unsigned int pcoa_dims, const char *mmap_dir) +{ + bool fp64; + compute_status rc = is_fp64(unifrac_method, format, fp64); + + if (rc==okay) { + if (fp64) { + mat_full_fp64_t* result; + rc = one_off_matrix(biom_filename, tree_filename, + unifrac_method, variance_adjust, alpha, + bypass_tips, threads, mmap_dir, + &result); + + if (rc==okay) { + // we have no alternative to hdf5 right now + IOStatus iostatus = write_mat_from_matrix_hdf5(out_filename, result, pcoa_dims); + destroy_mat_full_fp64(&result); + + if (iostatus!=write_okay) rc=output_error; + } + } else { + mat_full_fp32_t* result; + rc = one_off_matrix_fp32(biom_filename, tree_filename, + unifrac_method, variance_adjust, alpha, + bypass_tips, threads, mmap_dir, + &result); + + if (rc==okay) { + // we have no alternative to hdf5 right now + IOStatus iostatus = write_mat_from_matrix_hdf5_fp32(out_filename, result, pcoa_dims); + destroy_mat_full_fp32(&result); + + if (iostatus!=write_okay) rc=output_error; + } + } + } + + return rc; +} + +IOStatus write_mat(const char* output_filename, mat_t* result) { + std::ofstream output; + output.open(output_filename); + + uint64_t comb_N = su::comb_2(result->n_samples); + uint64_t comb_N_minus = 0; + double v; + + for(unsigned int i = 0; i < result->n_samples; i++) + output << "\t" << result->sample_ids[i]; + output << std::endl; + + for(unsigned int i = 0; i < result->n_samples; i++) { + output << result->sample_ids[i]; + for(unsigned int j = 0; j < result->n_samples; j++) { + if(i < j) { // upper triangle + comb_N_minus = su::comb_2(result->n_samples - i); + v = result->condensed_form[comb_N - comb_N_minus + (j - i - 1)]; + } else if (i > j) { // lower triangle + comb_N_minus = su::comb_2(result->n_samples - j); + v = result->condensed_form[comb_N - comb_N_minus + (i - j - 1)]; + } else { + v = 0.0; + } + output << std::setprecision(16) << "\t" << v; + } + output << std::endl; + } + output.close(); + + return write_okay; +} + +IOStatus write_mat_from_matrix(const char* output_filename, mat_full_fp64_t* result) { + const double *buf2d = result->matrix; + + std::ofstream output; + output.open(output_filename); + + double v; + const uint64_t n_samples_64 = result->n_samples; // 64-bit to avoid overflow + + for(unsigned int i = 0; i < result->n_samples; i++) + output << "\t" << result->sample_ids[i]; + output << std::endl; + + for(unsigned int i = 0; i < result->n_samples; i++) { + output << result->sample_ids[i]; + for(unsigned int j = 0; j < result->n_samples; j++) { + v = buf2d[i*n_samples_64+j]; + output << std::setprecision(16) << "\t" << v; + } + output << std::endl; + } + output.close(); + + return write_okay; +} + +herr_t write_hdf5_string(hid_t output_file_id,const char *dname, const char *str) +{ + // this is the convoluted way to store a string + // Will use the FORTRAN forma, so we do not depend on null termination + hid_t filetype_id = H5Tcopy (H5T_FORTRAN_S1); + H5Tset_size(filetype_id, strlen(str)); + hid_t memtype_id = H5Tcopy (H5T_C_S1); + H5Tset_size(memtype_id, strlen(str)+1); + + hsize_t dims[1] = {1}; + hid_t dataspace_id = H5Screate_simple (1, dims, NULL); + + hid_t dataset_id = H5Dcreate(output_file_id,dname, filetype_id, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + herr_t status = H5Dwrite(dataset_id, memtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, str); + + H5Dclose(dataset_id); + H5Sclose(dataspace_id); + H5Tclose(memtype_id); + H5Tclose(filetype_id); + + return status; +} + +// Internal: Make sure TReal and real_id match +template +IOStatus write_mat_from_matrix_hdf5_T(const char* output_filename, TMat * result, hid_t real_id, unsigned int pcoa_dims) { + /* Create a new file using default properties. */ + hid_t output_file_id = H5Fcreate(output_filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if (output_file_id<0) return write_error; + + // simple header + if (write_hdf5_string(output_file_id,"format","BDSM")<0) { + H5Fclose (output_file_id); + return write_error; + } + if (write_hdf5_string(output_file_id,"version","2020.12")<0) { + H5Fclose (output_file_id); + return write_error; + } + + // save the ids + { + hsize_t dims[1]; + dims[0] = result->n_samples; + hid_t dataspace_id = H5Screate_simple(1, dims, NULL); + + // this is the convoluted way to store an array of strings + hid_t datatype_id = H5Tcopy(H5T_C_S1); + H5Tset_size(datatype_id,H5T_VARIABLE); + + hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); + + hid_t dataset_id = H5Dcreate1(output_file_id, "order", datatype_id, dataspace_id, dcpl_id); + + herr_t status = H5Dwrite(dataset_id, datatype_id, H5S_ALL, H5S_ALL, + H5P_DEFAULT, result->sample_ids); + + H5Dclose(dataset_id); + H5Tclose(datatype_id); + H5Sclose(dataspace_id); + H5Pclose(dcpl_id); + + // check status after cleanup, for simplicity + if (status<0) { + H5Fclose (output_file_id); + return write_error; + } + } + + // save the matrix + { + hsize_t dims[2]; + dims[0] = result->n_samples; + dims[1] = result->n_samples; + hid_t dataspace_id = H5Screate_simple(2, dims, NULL); + + hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); + + hid_t dataset_id = H5Dcreate2(output_file_id, "matrix",real_id, dataspace_id, + H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, + result->matrix); + + + H5Pclose(dcpl_id); + H5Dclose(dataset_id); + H5Sclose(dataspace_id); + + // check status after cleanup, for simplicity + if (status<0) { + H5Fclose (output_file_id); + return write_error; + } + } + + if (pcoa_dims>0) { + // compute pcoa and save it in the file + // use inplace variant to keep memory use in check; we don't need matrix anymore + TReal * eigenvalues; + TReal * samples; + TReal * proportion_explained; + + su::pcoa_inplace(result->matrix, result->n_samples, pcoa_dims, eigenvalues, samples, proportion_explained); + + + if (write_hdf5_string(output_file_id,"pcoa_method","FSVD")<0) { + H5Fclose (output_file_id); + return write_error; + } + + // save the eigenvalues + { + hsize_t dims[1]; + dims[0] = pcoa_dims; + hid_t dataspace_id = H5Screate_simple(1, dims, NULL); + + hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); + + hid_t dataset_id = H5Dcreate2(output_file_id, "pcoa_eigvals",real_id, dataspace_id, + H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, + eigenvalues); + + + H5Pclose(dcpl_id); + H5Dclose(dataset_id); + H5Sclose(dataspace_id); + + // check status after cleanup, for simplicity + if (status<0) { + H5Fclose (output_file_id); + free(samples); + free(proportion_explained); + free(eigenvalues); + return write_error; + } + } + + // save the proportion_explained + { + hsize_t dims[1]; + dims[0] = pcoa_dims; + hid_t dataspace_id = H5Screate_simple(1, dims, NULL); + + hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); + + hid_t dataset_id = H5Dcreate2(output_file_id, "pcoa_proportion_explained",real_id, dataspace_id, + H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, + proportion_explained); + + + H5Pclose(dcpl_id); + H5Dclose(dataset_id); + H5Sclose(dataspace_id); + + // check status after cleanup, for simplicity + if (status<0) { + H5Fclose (output_file_id); + free(samples); + free(proportion_explained); + free(eigenvalues); + return write_error; + } + } + + // save the samples + { + hsize_t dims[2]; + dims[0] = result->n_samples; + dims[1] = pcoa_dims; + hid_t dataspace_id = H5Screate_simple(2, dims, NULL); + + hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); + + hid_t dataset_id = H5Dcreate2(output_file_id, "pcoa_samples",real_id, dataspace_id, + H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, + samples); + + + H5Pclose(dcpl_id); + H5Dclose(dataset_id); + H5Sclose(dataspace_id); + + // check status after cleanup, for simplicity + if (status<0) { + H5Fclose (output_file_id); + free(samples); + free(proportion_explained); + free(eigenvalues); + return write_error; + } + } + + free(samples); + free(proportion_explained); + free(eigenvalues); + + } + + H5Fclose (output_file_id); + return write_okay; +} + +// Internal: Make sure TReal and real_id match +template +IOStatus write_mat_hdf5_T(const char* output_filename, mat_t* result,hid_t real_id, unsigned int pcoa_dims) { + // compute the matrix + TMat mat_full; + mat_full.n_samples = result->n_samples; + + const uint64_t n_samples = result->n_samples; + mat_full.flags = 0; + mat_full.matrix = (TReal*) malloc(n_samples*n_samples*sizeof(TReal)); + if (mat_full.matrix==NULL) { + return open_error; // we don't have a better error code + } + + mat_full.sample_ids = result->sample_ids; // just link + + condensed_form_to_matrix_T(result->condensed_form, n_samples, mat_full.matrix); + IOStatus err = write_mat_from_matrix_hdf5_T(output_filename, &mat_full, real_id, pcoa_dims); + + free(mat_full.matrix); + return err; +} + +IOStatus write_mat_hdf5(const char* output_filename, mat_t* result, unsigned int pcoa_dims) { + return write_mat_hdf5_T(output_filename,result,H5T_IEEE_F64LE,pcoa_dims); +} + +IOStatus write_mat_hdf5_fp32(const char* output_filename, mat_t* result, unsigned int pcoa_dims) { + return write_mat_hdf5_T(output_filename,result,H5T_IEEE_F32LE,pcoa_dims); +} + +IOStatus write_mat_from_matrix_hdf5(const char* output_filename, mat_full_fp64_t* result, unsigned int pcoa_dims) { + return write_mat_from_matrix_hdf5_T(output_filename,result,H5T_IEEE_F64LE,pcoa_dims); +} + +IOStatus write_mat_from_matrix_hdf5_fp32(const char* output_filename, mat_full_fp32_t* result, unsigned int pcoa_dims) { + return write_mat_from_matrix_hdf5_T(output_filename,result,H5T_IEEE_F32LE,pcoa_dims); +} + +IOStatus write_vec(const char* output_filename, r_vec* result) { + std::ofstream output; + output.open(output_filename); + + // write sample ids in first column of file and faith's pd in second column + output << "#SampleID\tfaith_pd" << std::endl; + for(unsigned int i = 0; i < result->n_samples; i++) { + output << result->sample_ids[i]; + output << std::setprecision(16) << "\t" << result->values[i]; + output << std::endl; + } + output.close(); + + return write_okay; +} + +IOStatus write_partial(const char* output_filename, const partial_mat_t* result) { + int fd = open(output_filename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR ); + if (fd==-1) return write_error; + + int cnt = -1; + + uint32_t n_stripes = result->stripe_stop - result->stripe_start; + + uint32_t sample_id_length = 0; + for(unsigned int i = 0; i < result->n_samples; i++) { + sample_id_length += strlen(result->sample_ids[i])+1; + } + + { + char * const samples_buf = (char *)malloc(sample_id_length); + + char *samples_ptr = samples_buf; + + /* sample IDs */ + for(unsigned int i = 0; i < result->n_samples; i++) { + uint32_t length = strlen(result->sample_ids[i])+1; + memcpy(samples_ptr,result->sample_ids[i],length); + samples_ptr+= length; + } + + int max_compressed = LZ4_compressBound(sample_id_length); + char * const cmp_buf = (char *)malloc(max_compressed); + + int sample_id_length_compressed = LZ4_compress_default(samples_buf,cmp_buf,sample_id_length,max_compressed); + if (sample_id_length_compressed<1) {close(fd); return open_error;} + + uint32_t header[8]; + header[0] = PARTIAL_MAGIC_V2; + header[1] = result->n_samples; + header[2] = n_stripes; + header[3] = result->stripe_start; + header[4] = result->stripe_total; + header[5] = result->is_upper_triangle; + header[6] = sample_id_length; + header[7] = sample_id_length_compressed; + + cnt=write(fd,header, 8 * sizeof(uint32_t)); + if (cnt<1) {close(fd); return write_error;} + + cnt=write(fd,cmp_buf, sample_id_length_compressed); + if (cnt<1) {close(fd); return write_error;} + + free(cmp_buf); + free(samples_buf); + } + + { + int max_compressed = LZ4_compressBound(sizeof(double) * result->n_samples); + char * const cmp_buf_raw = (char *)malloc(max_compressed+sizeof(uint32_t)); + char * const cmp_buf = cmp_buf_raw + sizeof(uint32_t); + + /* stripe information */ + for(unsigned int i = 0; i < n_stripes; i++) { + int cmp_size = LZ4_compress_default((const char *) result->stripes[i],cmp_buf,sizeof(double) * result->n_samples,max_compressed); + if (cmp_size<1) {close(fd); return open_error;} + + uint32_t *cmp_buf_size_p = (uint32_t *)cmp_buf_raw; + *cmp_buf_size_p = cmp_size; + + cnt=write(fd, cmp_buf_raw, cmp_size+sizeof(uint32_t)); + if (cnt<1) {return write_error;} + } + + free(cmp_buf_raw); + } + + /* footer */ + { + uint32_t header[1]; + header[0] = PARTIAL_MAGIC_V2; + + cnt=write(fd,header, 1 * sizeof(uint32_t)); + if (cnt<1) {close(fd); return open_error;} + } + + close(fd); + + return write_okay; +} + +IOStatus _is_partial_file(const char* input_filename) { + int fd = open(input_filename, O_RDONLY ); + if (fd==-1) return open_error; + + uint32_t header[1]; + int cnt = read(fd,header,sizeof(uint32_t)); + close(fd); + + if (cnt!=sizeof(uint32_t)) return magic_incompatible; + if ( header[0] != PARTIAL_MAGIC_V2) return magic_incompatible; + + return read_okay; +} + +template +inline IOStatus read_partial_header_fd(int fd, TPMat &result) { + int cnt=-1; + + uint32_t header[8]; + cnt = read(fd,header,8*sizeof(uint32_t)); + if (cnt != (8*sizeof(uint32_t))) {return magic_incompatible;} + + if ( header[0] != PARTIAL_MAGIC_V2) {return magic_incompatible;} + + const uint32_t n_samples = header[1]; + const uint32_t n_stripes = header[2]; + const uint32_t stripe_start = header[3]; + const uint32_t stripe_total = header[4]; + const bool is_upper_triangle = header[5]; + + /* sanity check header */ + if(n_samples <= 0 || n_stripes <= 0 || stripe_total <= 0 || is_upper_triangle < 0) + {return bad_header;} + if(stripe_total >= n_samples || n_stripes > stripe_total || stripe_start >= stripe_total || stripe_start + n_stripes > stripe_total) + {return bad_header;} + + /* initialize the partial result structure */ + result.n_samples = n_samples; + result.sample_ids = (char**)malloc(sizeof(char*) * n_samples); + result.stripes = (double**)malloc(sizeof(double*) * (n_stripes)); + result.stripe_start = stripe_start; + result.stripe_stop = stripe_start + n_stripes; + result.is_upper_triangle = is_upper_triangle; + result.stripe_total = stripe_total; + + /* load samples */ + { + const uint32_t sample_id_length = header[6]; + const uint32_t sample_id_length_compressed = header[7]; + + /* sanity check header */ + if (sample_id_length<=0 || sample_id_length_compressed <=0) + { return bad_header;} + + char * const cmp_buf = (char *)malloc(sample_id_length_compressed); + if (cmp_buf==NULL) { return bad_header;} // no better error code + cnt = read(fd,cmp_buf,sample_id_length_compressed); + if (cnt != sample_id_length_compressed) {free(cmp_buf); return magic_incompatible;} + + char *samples_buf = (char *)malloc(sample_id_length); + if (samples_buf==NULL) { free(cmp_buf); return bad_header;} // no better error code + + cnt = LZ4_decompress_safe(cmp_buf,samples_buf,sample_id_length_compressed,sample_id_length); + if (cnt!=sample_id_length) {free(samples_buf); free(cmp_buf); return magic_incompatible;} + + const char *samples_ptr = samples_buf; + + for(int i = 0; i < n_samples; i++) { + uint32_t sample_length = strlen(samples_ptr); + if ((samples_ptr+sample_length+1)>(samples_buf+sample_id_length)) {free(samples_buf); free(cmp_buf); return magic_incompatible;} + + result.sample_ids[i] = (char*)malloc(sample_length + 1); + memcpy(result.sample_ids[i],samples_ptr,sample_length + 1); + samples_ptr += sample_length + 1; + } + free(samples_buf); + free(cmp_buf); + } + + return read_okay; +} + +template +inline IOStatus read_partial_data_fd(int fd, TPMat &result) { + int cnt=-1; + + const uint32_t n_samples = result.n_samples; + const uint32_t n_stripes = result.stripe_stop-result.stripe_start; + + /* load stripes */ + { + int max_compressed = LZ4_compressBound(sizeof(double) * n_samples); + char * const cmp_buf = (char *)malloc(max_compressed+sizeof(uint32_t)); + if (cmp_buf==NULL) { return bad_header;} // no better error code + + uint32_t *cmp_buf_size_p = (uint32_t *)cmp_buf; + + cnt = read(fd,cmp_buf_size_p , sizeof(uint32_t) ); + if (cnt != sizeof(uint32_t) ) {free(cmp_buf); return magic_incompatible;} + + for(int i = 0; i < n_stripes; i++) { + uint32_t cmp_size = *cmp_buf_size_p; + + uint32_t read_size = cmp_size; + if ( (i+1) +inline IOStatus read_partial_one_stripe_fd(int fd, TPMat &result, uint32_t stripe_idx) { + int cnt=-1; + + const uint32_t n_samples = result.n_samples; + + /* load stripes */ + { + int max_compressed = LZ4_compressBound(sizeof(double) * n_samples); + char * const cmp_buf = (char *)malloc(max_compressed+sizeof(uint32_t)); + if (cmp_buf==NULL) { return bad_header;} // no better error code + + uint32_t *cmp_buf_size_p = (uint32_t *)cmp_buf; + + uint32_t curr_idx = stripe_idx; + while (result.offsets[curr_idx]==0) --curr_idx; // must start reading from the first known offset + + for (;curr_idx(fd, *result); + if (sts==read_okay) + sts = read_partial_data_fd(fd, *result); + + if (sts==read_okay) { + IOStatus sts = read_okay; + /* sanity check the footer */ + uint32_t header[1]; + header[0] = 0; + int cnt = read(fd,header,sizeof(uint32_t)); + if (cnt != (sizeof(uint32_t))) {sts= magic_incompatible;} + + if (sts==read_okay) { + if ( header[0] != PARTIAL_MAGIC_V2) {sts= magic_incompatible;} + } + } + + close(fd); + + if (sts==read_okay) { + (*result_out) = result; + } else { + free(result); + (*result_out) = NULL; + } + return sts; +} + +IOStatus read_partial_header(const char* input_filename, partial_dyn_mat_t** result_out) { + int fd = open(input_filename, O_RDONLY ); + if (fd==-1) return open_error; + + /* initialize the partial result structure */ + partial_dyn_mat_t* result = (partial_dyn_mat_t*)malloc(sizeof(partial_dyn_mat)); + { + IOStatus sts = read_partial_header_fd(fd, *result); + if (sts!=read_okay) {free(result); close(fd); return sts;} + } + + // save the offset of the first stripe + const uint32_t n_stripes = result->stripe_stop-result->stripe_start; + result->stripes = (double**) calloc(n_stripes,sizeof(double*)); + result->offsets = (uint64_t*) calloc(n_stripes,sizeof(uint64_t)); + result->offsets[0] = lseek(fd,0,SEEK_CUR); + + close(fd); + + result->filename= strdup(input_filename); + + (*result_out) = result; + return read_okay; +} + +IOStatus read_partial_one_stripe(partial_dyn_mat_t* result, uint32_t stripe_idx) { + if (result->stripes[stripe_idx]!=0) return read_okay; // will not re-read + + int fd = open(result->filename, O_RDONLY ); + if (fd==-1) return open_error; + + IOStatus sts = read_partial_one_stripe_fd(fd, *result, stripe_idx); + + close(fd); + return sts; +} + + +template +MergeStatus check_partial(const TPMat* const * partial_mats, int n_partials) { + if(n_partials <= 0) { + fprintf(stderr, "Zero or less partials.\n"); + exit(EXIT_FAILURE); + } + + // sanity check + int n_samples = partial_mats[0]->n_samples; + bool *stripe_map = (bool*)calloc(sizeof(bool), partial_mats[0]->stripe_total); + int stripe_count = 0; + + for(int i = 0; i < n_partials; i++) { + if(partial_mats[i]->n_samples != n_samples) { + free(stripe_map); + return partials_mismatch; + } + + if(partial_mats[0]->stripe_total != partial_mats[i]->stripe_total) { + free(stripe_map); + return partials_mismatch; + } + if(partial_mats[0]->is_upper_triangle != partial_mats[i]->is_upper_triangle) { + free(stripe_map); + return square_mismatch; + } + for(int j = 0; j < n_samples; j++) { + if(strcmp(partial_mats[0]->sample_ids[j], partial_mats[i]->sample_ids[j]) != 0) { + free(stripe_map); + return sample_id_consistency; + } + } + for(int j = partial_mats[i]->stripe_start; j < partial_mats[i]->stripe_stop; j++) { + if(stripe_map[j]) { + free(stripe_map); + return stripes_overlap; + } + stripe_map[j] = true; + stripe_count += 1; + } + } + free(stripe_map); + + if(stripe_count != partial_mats[0]->stripe_total) { + return incomplete_stripe_set; + } + + return merge_okay; +} + +MergeStatus merge_partial(partial_mat_t** partial_mats, int n_partials, unsigned int nthreads, mat_t** result) { + MergeStatus err = check_partial(partial_mats, n_partials); + if (err!=merge_okay) return err; + + int n_samples = partial_mats[0]->n_samples; + std::vector stripes(partial_mats[0]->stripe_total); + std::vector stripes_totals(partial_mats[0]->stripe_total); // not actually used but destroy_stripes needs this to "exist" + for(int i = 0; i < n_partials; i++) { + int n_stripes = partial_mats[i]->stripe_stop - partial_mats[i]->stripe_start; + for(int j = 0; j < n_stripes; j++) { + // as this is potentially a large amount of memory, don't copy, just adopt + *&(stripes[j + partial_mats[i]->stripe_start]) = partial_mats[i]->stripes[j]; + } + } + + initialize_mat_no_biom(*result, partial_mats[0]->sample_ids, n_samples, partial_mats[0]->is_upper_triangle); + if ((*result)==NULL) return incomplete_stripe_set; + if ((*result)->condensed_form==NULL) return incomplete_stripe_set; + if ((*result)->sample_ids==NULL) return incomplete_stripe_set; + + su::stripes_to_condensed_form(stripes, n_samples, (*result)->condensed_form, 0, partial_mats[0]->stripe_total); + + destroy_stripes(stripes, stripes_totals, n_samples, 0, n_partials); + + return merge_okay; +} + +// Will keep only the strictly necessary stripes in memory... reading just in time +class PartialStripes : public su::ManagedStripes { + private: + const uint32_t n_partials; + mutable partial_dyn_mat_t* * partial_mats; // link only, not owned + + static bool in_range(const partial_dyn_mat_t &partial_mat, uint32_t stripe) { + return (stripe>=partial_mat.stripe_start) && (stripestripe_start; + + if (partial_mat->stripes[sidx]==NULL) { + read_partial_one_stripe(partial_mat,sidx); + // ignore any errors, not clear what to do + // will just return NULL + } + + return partial_mat->stripes[sidx]; + } + virtual void release_stripe(uint32_t stripe) const { + uint32_t pidx = find_partial_idx(stripe); + partial_dyn_mat_t * const partial_mat = partial_mats[pidx]; + uint32_t sidx = stripe-partial_mat->stripe_start; + + if (partial_mat->stripes[sidx]!=NULL) { + free(partial_mat->stripes[sidx]); + partial_mat->stripes[sidx]=NULL; + } + } +}; + +template +MergeStatus merge_partial_to_matrix_T(partial_dyn_mat_t* * partial_mats, int n_partials, + const char *mmap_dir, /* if NULL or "", use malloc */ + TMat** result /* out */ ) { + if (mmap_dir!=NULL) { + if (mmap_dir[0]==0) mmap_dir = NULL; // easier to have a simple test going on + } + + MergeStatus err = check_partial(partial_mats, n_partials); + if (err!=merge_okay) return err; + + initialize_mat_full_no_biom_T(*result, partial_mats[0]->sample_ids, partial_mats[0]->n_samples,mmap_dir); + + if ((*result)==NULL) return incomplete_stripe_set; + if ((*result)->matrix==NULL) return incomplete_stripe_set; + if ((*result)->sample_ids==NULL) return incomplete_stripe_set; + + PartialStripes ps(n_partials,partial_mats); + const uint32_t tile_size = (mmap_dir==NULL) ? \ + (128/sizeof(TReal)) : /* keep it small for memory access, to fit in chip cache */ \ + (4096/sizeof(TReal)); /* make it larger for mmap, as the limiting factor is swapping */ + su::stripes_to_matrix_T(ps, partial_mats[0]->n_samples, partial_mats[0]->stripe_total, (*result)->matrix, tile_size); + + return merge_okay; +} + +MergeStatus merge_partial_to_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp64_t** result) { + return merge_partial_to_matrix_T(partial_mats, n_partials, NULL, result); +} + +MergeStatus merge_partial_to_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp32_t** result) { + return merge_partial_to_matrix_T(partial_mats, n_partials, NULL, result); +} + +MergeStatus merge_partial_to_mmap_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp64_t** result) { + return merge_partial_to_matrix_T(partial_mats, n_partials, mmap_dir, result); +} + +MergeStatus merge_partial_to_mmap_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp32_t** result) { + return merge_partial_to_matrix_T(partial_mats, n_partials, mmap_dir, result); +} + + +// skbio_alt pass-thoughs + + +// Find eigen values and vectors +// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. +// Original Paper: https://arxiv.org/abs/1007.5510 +// centered == n x n, must be symmetric, Note: will be used in-place as temp buffer + +void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double **eigenvalues, double **eigenvectors) { + su::find_eigens_fast(n_samples, n_dims, centered, *eigenvalues, *eigenvectors); +} + +void find_eigens_fast_fp32(const uint32_t n_samples, const uint32_t n_dims, float * centered, float **eigenvalues, float **eigenvectors) { + su::find_eigens_fast(n_samples, n_dims, centered, *eigenvalues, *eigenvectors); +} + +/* + Perform Principal Coordinate Analysis. + + Principal Coordinate Analysis (PCoA) is a method similar + to Principal Components Analysis (PCA) with the difference that PCoA + operates on distance matrices, typically with non-euclidian and thus + ecologically meaningful distances like UniFrac in microbiome research. + + In ecology, the euclidean distance preserved by Principal + Component Analysis (PCA) is often not a good choice because it + deals poorly with double zeros (Species have unimodal + distributions along environmental gradients, so if a species is + absent from two sites at the same site, it can't be known if an + environmental variable is too high in one of them and too low in + the other, or too low in both, etc. On the other hand, if an + species is present in two sites, that means that the sites are + similar.). + + Note that the returned eigenvectors are not normalized to unit length. +*/ + +// mat - in, result of unifrac compute +// n_samples - in, size of the matrix (n x n) +// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. +// eigenvalues - out, alocated buffer of size n_dims +// samples - out, alocated buffer of size n_dims x n_samples +// proportion_explained - out, allocated buffer of size n_dims + +void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, double * *eigenvalues, double * *samples, double * *proportion_explained) { + su::pcoa(mat, n_samples, n_dims, *eigenvalues, *samples, *proportion_explained); +} + +void pcoa_fp32(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained) { + su::pcoa(mat, n_samples, n_dims, *eigenvalues, *samples, *proportion_explained); +} + +void pcoa_mixed(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained) { + su::pcoa(mat, n_samples, n_dims, *eigenvalues, *samples, *proportion_explained); +} + diff --git a/R/unifrac_cpp/api.hpp b/R/unifrac_cpp/api.hpp new file mode 100644 index 000000000..7b856b6bb --- /dev/null +++ b/R/unifrac_cpp/api.hpp @@ -0,0 +1,577 @@ +#include "task_parameters.hpp" + +#ifdef __cplusplus +#include +#define EXTERN extern "C" + + +#else +#include +#define EXTERN +#endif + +#define PARTIAL_MAGIC "SSU-PARTIAL-01" +#define PARTIAL_MAGIC_V2 0x088ABA02 + + +typedef enum compute_status {okay=0, tree_missing, table_missing, table_empty, unknown_method, table_and_tree_do_not_overlap, output_error} ComputeStatus; +typedef enum io_status {read_okay=0, write_okay, open_error, read_error, magic_incompatible, bad_header, unexpected_end, write_error} IOStatus; +typedef enum merge_status {merge_okay=0, incomplete_stripe_set, sample_id_consistency, square_mismatch, partials_mismatch, stripes_overlap} MergeStatus; + +/* a result matrix + * + * n_samples the number of samples. + * cf_size the size of the condensed form. + * is_upper_triangle if true, indicates condensed_form represents a square + * matrix, and only the upper triangle is contained. if false, + * condensed_form represents the lower triangle of a matrix. + * condensed_form the matrix values of length cf_size. + * sample_ids the sample IDs of length n_samples. + */ +typedef struct mat { + unsigned int n_samples; + unsigned int cf_size; + bool is_upper_triangle; + double* condensed_form; + char** sample_ids; +} mat_t; + +/* a result matrix, full, fp64 + * + * n_samples the number of samples. + * matrix the matrix values, n_sample**2 size + * sample_ids the sample IDs of length n_samples. + */ +typedef struct mat_full_fp64 { + uint32_t n_samples; + uint32_t flags; //opaque, 0 for default behavior + double* matrix; + char** sample_ids; +} mat_full_fp64_t; + +/* a result matrix, full, fp32 + * + * n_samples the number of samples. + * matrix the matrix values, n_sample**2 size + * sample_ids the sample IDs of length n_samples. + */ +typedef struct mat_full_fp32 { + uint32_t n_samples; + uint32_t flags; //opaque, 0 for default behavior + float* matrix; + char** sample_ids; +} mat_full_fp32_t; + + + +/* a result vector + * + * n_samples the number of samples. + * values the score values of length n_samples. + * sample_ids the sample IDs of length n_samples. + */ +typedef struct results_vec{ + unsigned int n_samples; + double* values; + char** sample_ids; +} r_vec; + +/* a partial result containing stripe data + * + * n_samples the number of samples. + * sample_ids the sample IDs of length n_samples. + * stripes the stripe data of dimension (stripe_stop - stripe_start, n_samples) + * stripe_start the logical starting stripe in the final matrix. + * stripe_stop the logical stopping stripe in the final matrix. + * stripe_total the total number of stripes present in the final matrix. + * is_upper_triangle whether the stripes correspond to the upper triangle of the resulting matrix. + * This is useful for asymmetric unifrac metrics. + */ +typedef struct partial_mat { + uint32_t n_samples; + char** sample_ids; + double** stripes; + uint32_t stripe_start; + uint32_t stripe_stop; + uint32_t stripe_total; + bool is_upper_triangle; +} partial_mat_t; + +/* a partial resuly, can be populated dynamically + * + * n_samples the number of samples. + * sample_ids the sample IDs of length n_samples. + * offsets offsets to the stripes in the file; 0 means unknown + * stripes the stripe data of dimension (stripe_stop - stripe_start, n_samples) + * stripe_start the logical starting stripe in the final matrix. + * stripe_stop the logical stopping stripe in the final matrix. + * stripe_total the total number of stripes present in the final matrix. + * is_upper_triangle whether the stripes correspond to the upper triangle of the resulting matrix. + * This is useful for asymmetric unifrac metrics. + * filename Name of the file from which to read + */ +typedef struct partial_dyn_mat { + uint32_t n_samples; + char** sample_ids; + uint64_t* offsets; + double** stripes; + uint32_t stripe_start; + uint32_t stripe_stop; + uint32_t stripe_total; + bool is_upper_triangle; + char* filename; +} partial_dyn_mat_t; + + + +void destroy_mat(mat_t** result); +void destroy_mat_full_fp64(mat_full_fp64_t** result); +void destroy_mat_full_fp32(mat_full_fp32_t** result); +void destroy_partial_mat(partial_mat_t** result); +void destroy_partial_dyn_mat(partial_dyn_mat_t** result); +void destroy_results_vec(r_vec** result); + +/* Compute UniFrac - condensed form + * + * biom_filename the filename to the biom table. + * tree_filename the filename to the correspodning tree. + * unifrac_method the requested unifrac method. + * variance_adjust whether to apply variance adjustment. + * alpha GUniFrac alpha, only relevant if method == generalized. + * bypass_tips disregard tips, reduces compute by about 50% + * threads the number of threads to use. + * result the resulting distance matrix in condensed form, this is initialized within the method so using ** + * + * one_off returns the following error codes: + * + * okay : no problems encountered + * table_missing : the filename for the table does not exist + * tree_missing : the filename for the tree does not exist + * unknown_method : the requested method is unknown. + * table_empty : the table does not have any entries + */ +EXTERN ComputeStatus one_off(const char* biom_filename, const char* tree_filename, + const char* unifrac_method, bool variance_adjust, double alpha, + bool bypass_tips, unsigned int threads, mat_t** result); + +/* Compute UniFrac - matrix form + * + * biom_filename the filename to the biom table. + * tree_filename the filename to the correspodning tree. + * unifrac_method the requested unifrac method. + * variance_adjust whether to apply variance adjustment. + * alpha GUniFrac alpha, only relevant if method == generalized. + * bypass_tips disregard tips, reduces compute by about 50% + * threads the number of threads/blocks to use. + * mmap_dir If not NULL, area to use for temp memory storage + * result the resulting distance matrix in matrix form, this is initialized within the method so using ** + * + * one_off_matrix returns the following error codes: + * + * okay : no problems encountered + * table_missing : the filename for the table does not exist + * tree_missing : the filename for the tree does not exist + * unknown_method : the requested method is unknown. + * table_empty : the table does not have any entries + */ +EXTERN ComputeStatus one_off_matrix(const char* biom_filename, const char* tree_filename, + const char* unifrac_method, bool variance_adjust, double alpha, + bool bypass_tips, unsigned int nthreads, + const char *mmap_dir, + mat_full_fp64_t** result); + +/* Compute UniFrac - matrix form, fp32 variant + * + * biom_filename the filename to the biom table. + * tree_filename the filename to the correspodning tree. + * unifrac_method the requested unifrac method. + * variance_adjust whether to apply variance adjustment. + * alpha GUniFrac alpha, only relevant if method == generalized. + * bypass_tips disregard tips, reduces compute by about 50% + * threads the number of threads/blocks to use. + * mmap_dir If not NULL, area to use for temp memory storage + * result the resulting distance matrix in matrix form, this is initialized within the method so using ** + * + * one_off_matrix_fp32 returns the following error codes: + * + * okay : no problems encountered + * table_missing : the filename for the table does not exist + * tree_missing : the filename for the tree does not exist + * unknown_method : the requested method is unknown. + * table_empty : the table does not have any entries + */ +EXTERN ComputeStatus one_off_matrix_fp32(const char* biom_filename, const char* tree_filename, + const char* unifrac_method, bool variance_adjust, double alpha, + bool bypass_tips, unsigned int nthreads, + const char *mmap_dir, + mat_full_fp32_t** result); + + +/* compute Faith PD + * biom_filename the filename to the biom table. + * tree_filename the filename to the correspodning tree. + * result the resulting vector of computed Faith PD values + * + * faith_pd_one_off returns the following error codes: + * + * okay : no problems encountered + * table_missing : the filename for the table does not exist + * tree_missing : the filename for the tree does not exist + * table_empty : the table does not have any entries + */ +EXTERN ComputeStatus faith_pd_one_off(const char* biom_filename, const char* tree_filename, + r_vec** result); + +/* Compute UniFrac and save to file + * + * biom_filename the filename to the biom table. + * tree_filename the filename to the correspodning tree. + * out_filename the filename of the output file. + * unifrac_method the requested unifrac method. + * variance_adjust whether to apply variance adjustment. + * alpha GUniFrac alpha, only relevant if method == generalized. + * bypass_tips disregard tips, reduces compute by about 50% + * threads the number of threads to use. + * format output format to use. + * pcoa_dims if not 0, number of dimensions to use or PCoA + * mmap_dir if not empty, temp dir to use for disk-based memory + * + * unifrac_to_file returns the following error codes: + * + * okay : no problems encountered + * table_missing : the filename for the table does not exist + * tree_missing : the filename for the tree does not exist + * unknown_method : the requested method is unknown. + * table_empty : the table does not have any entries + * output_error : failed to properly write the output file + */ +EXTERN ComputeStatus unifrac_to_file(const char* biom_filename, const char* tree_filename, const char* out_filename, + const char* unifrac_method, bool variance_adjust, double alpha, + bool bypass_tips, unsigned int threads, const char* format, + unsigned int pcoa_dims, const char *mmap_dir); + +/* Write a matrix object + * + * filename the file to write into + * result the results object + * + * The following error codes are returned: + * + * write_okay : no problems + */ +EXTERN IOStatus write_mat(const char* filename, mat_t* result); + +/* Write a matrix object using hdf5 format + * + * filename the file to write into + * result the results object + * pcoa_dims PCoAdimensions to compute, if >0 + * + * The following error codes are returned: + * + * write_okay : no problems + */ +EXTERN IOStatus write_mat_hdf5(const char* filename, mat_t* result, unsigned int pcoa_dims); + +/* Write a matrix object using hdf5 format, using fp32 precision + * + * filename the file to write into + * result the results object + * pcoa_dims PCoAdimensions to compute, if >0 + * + * The following error codes are returned: + * + * write_okay : no problems + */ +EXTERN IOStatus write_mat_hdf5_fp32(const char* filename, mat_t* result, unsigned int pcoa_dims); + +/* Write a matrix object + * + * filename the file to write into + * result the results object + * + * The following error codes are returned: + * + * write_okay : no problems + */ +EXTERN IOStatus write_mat_from_matrix(const char* filename, mat_full_fp64_t* result); + + +/* Write a matrix object from buffer using hdf5 format + * + * filename the file to write into + * result the results object + * pcoa_dims PCoAdimensions to compute, if >0 + * + * The following error codes are returned: + * + * write_okay : no problems + */ +EXTERN IOStatus write_mat_from_matrix_hdf5(const char* filename, mat_full_fp64_t* result, unsigned int pcoa_dims); + +/* Write a matrix object from buffer using hdf5 format, using fp32 precision + * + * filename the file to write into + * result the results object + * pcoa_dims PCoAdimensions to compute, if >0 + * + * The following error codes are returned: + * + * write_okay : no problems + */ +EXTERN IOStatus write_mat_from_matrix_hdf5_fp32(const char* filename, mat_full_fp32_t* result, unsigned int pcoa_dims); + +/* Write a series + * + * filename the file to write into + * result the results object + * + * The following error codes are returned: + * + * write_okay : no problems + */ +EXTERN IOStatus write_vec(const char* filename, r_vec* result); + +/* Read a matrix object + * + * filename the file to write into + * result the results object + * + * The following error codes are returned: + * + * read_okay : no problems + * open_error : could not open the file + * magic_incompatible : format magic not found or incompatible + * unexpected_end : format end not found in expected location + */ +//EXTERN IOStatus read_mat(const char* filename, mat_t** result); + +/* Compute a subset of a UniFrac distance matrix + * + * biom_filename the filename to the biom table. + * tree_filename the filename to the correspodning tree. + * unifrac_method the requested unifrac method. + * variance_adjust whether to apply variance adjustment. + * alpha GUniFrac alpha, only relevant if method == generalized. + * bypass_tips disregard tips, reduces compute by about 50% + * threads the number of threads to use. + * stripe_start the starting stripe to compute + * stripe_stop the last stripe to compute + * dm_stripes the unique branch length stripes. This is expected to be + * uninitialized, and is an output parameter. + * dm_stripes_total the total branch length stripes. This is expected to be + * uninitialized, and is an output parameter. + * result the resulting distance matrix in condensed form, this is initialized within the method so using ** + * + * partial returns the following error codes: + * + * okay : no problems encountered + * table_missing : the filename for the table does not exist + * tree_missing : the filename for the tree does not exist + * unknown_method : the requested method is unknown. + */ + +EXTERN ComputeStatus partial(const char* biom_filename, const char* tree_filename, + const char* unifrac_method, bool variance_adjust, double alpha, + bool bypass_tips, unsigned int threads, unsigned int stripe_start, + unsigned int stripe_stop, partial_mat_t** result); + +/* Write a partial matrix object + * + * filename the file to write into + * result the partial results object + * + * The following error codes are returned: + * + * write_okay : no problems + * open_error : could not open the file + * + * The structure of the binary output file is as follows. Newlines added for clarity, but are not stored. + * The file has logical blocks, but are not explicitly denoted in the format. These logical blocks are + * just used to improve readability here, and are denoted by ### marks. + * + * ### HEADER ### + * : uint16_t, the length of the magic + * : char, e.g., SSU-PARTIAL-01 + * : uint32_t, the number of samples + * : uint32_t, the number of stripes represented in this file + * : uint32_t, the starting stripe number + * : uint32_t, the total number of stripes in the full matrix + * : uint8_t, zero is false, nonzero is true + * + * ### SAMPLE IDS ### + * : uint16_t, the length of the next sample ID + * : LEN bytes, char + * ... : ... repeated + * : uint16_t, the length of the next sample ID + * : LEN bytes, char + * + * ### STRIPE VALUES; SS -> STRIPE_START, NS -> N_STRIPES + * : double, the first value in the 0th stripe + * ... : ... repeated for N_SAMPLES values + * : double, the last value in the 0th stripe + * : double, the first value in the Kth stripe + * ... : ... repeated for N_SAMPLES values + * : double, the last value in the Kth stripe + * + * ### FOOTER ### + * : char, e.g., SSU-PARTIAL-01, same as starting magic + */ +EXTERN IOStatus write_partial(const char* filename, const partial_mat_t* result); + +/* Read a partial matrix object + * + * filename the file to write into + * result the partial results object, output parameter + * + * The following error codes are returned: + * + * read_okay : no problems + * open_error : could not open the file + * magic_incompatible : format magic not found or incompatible + * bad_header : header seems malformed + * unexpected_end : format end not found in expected location + */ +EXTERN IOStatus read_partial(const char* filename, partial_mat_t** result); + +/* Read a partial matrix object header + * + * filename the file to write into + * result the partial results object, output parameter + * + * The following error codes are returned: + * + * read_okay : no problems + * open_error : could not open the file + * magic_incompatible : format magic not found or incompatible + * bad_header : header seems malformed + * unexpected_end : format end not found in expected location + */ +EXTERN IOStatus read_partial_header(const char* input_filename, partial_dyn_mat_t** result_out); + +/* Read a stripe of a partial matrix + * + * filename the file to write into + * result the partial results object + * stripe_idx relative stripe number + * + * The following error codes are returned: + * + * read_okay : no problems + * open_error : could not open the file + * magic_incompatible : format magic not found or incompatible + * bad_header : header seems malformed + * unexpected_end : format end not found in expected location + */ +EXTERN IOStatus read_partial_one_stripe(partial_dyn_mat_t* result, uint32_t stripe_idx); + + +/* Merge partial results + * + * results an array of partial_mat_t*, the buffers will be destroyed in the process + * n_partials number of partial mats + * merged the full matrix, output parameters, this is initialized in the method so using ** + * + * The following error codes are returned: + * + * merge_okay : no problems + * incomplete_stripe_set : not all stripes needed to create a full matrix were foun + * sample_id_consistency : samples described by stripes are inconsistent + * square_mismatch : inconsistency on denotation of square matrix + */ +EXTERN MergeStatus merge_partial(partial_mat_t** partial_mats, int n_partials, unsigned int nthreads, mat_t** result); + +/* Merge partial results + * + * partial_mats an array of partial_dyn_mat_t* + * n_partials number of partial mats + * result the full matrix, output parameters, this is initialized in the method so using ** + * + * The following error codes are returned: + * + * merge_okay : no problems + * incomplete_stripe_set : not all stripes needed to create a full matrix were foun + * sample_id_consistency : samples described by stripes are inconsistent + * square_mismatch : inconsistency on denotation of square matrix + */ +MergeStatus merge_partial_to_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp64_t** result); + +/* Merge partial results + * + * partial_mats an array of partial_dyn_mat_t* + * n_partials number of partial mats + * result the full matrix, output parameters, this is initialized in the method so using ** + * + * The following error codes are returned: + * + * merge_okay : no problems + * incomplete_stripe_set : not all stripes needed to create a full matrix were foun + * sample_id_consistency : samples described by stripes are inconsistent + * square_mismatch : inconsistency on denotation of square matrix + */ +MergeStatus merge_partial_to_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp32_t** result); + + +/* Merge partial results + * + * partial_mats an array of partial_dyn_mat_t* + * n_partials number of partial mats + * mmap_dir Where to host the mmap file + * result the full matrix, output parameters, this is initialized in the method so using ** + * + * The following error codes are returned: + * + * merge_okay : no problems + * incomplete_stripe_set : not all stripes needed to create a full matrix were foun + * sample_id_consistency : samples described by stripes are inconsistent + * square_mismatch : inconsistency on denotation of square matrix + */ +MergeStatus merge_partial_to_mmap_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp64_t** result); + +/* Merge partial results + * + * partial_mats an array of partial_dyn_mat_t* + * n_partials number of partial mats + * mmap_dir Where to host the mmap file + * result the full matrix, output parameters, this is initialized in the method so using ** + * + * The following error codes are returned: + * + * merge_okay : no problems + * incomplete_stripe_set : not all stripes needed to create a full matrix were foun + * sample_id_consistency : samples described by stripes are inconsistent + * square_mismatch : inconsistency on denotation of square matrix + */ +MergeStatus merge_partial_to_mmap_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp32_t** result); + + +// Find eigen values and vectors +// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. +// Original Paper: https://arxiv.org/abs/1007.5510 +// centered == n x n, must be symmetric, Note: will be used in-place as temp buffer +void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double **eigenvalues, double **eigenvectors); +void find_eigens_fast_p32(const uint32_t n_samples, const uint32_t n_dims,float * centered, float **eigenvalues, float **eigenvectors); + +// Perform Principal Coordinate Analysis +// mat - in, result of unifrac compute +// n_samples - in, size of the matrix (n x n) +// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. +// eigenvalues - out, alocated buffer of size n_dims +// samples - out, alocated buffer of size n_dims x n_samples +// proportion_explained - out, allocated buffer of size n_dims +void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, double **eigenvalues, double **samples, double **proportion_explained); +void pcoa_fp32(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained); +void pcoa_mixed(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained); + + +#ifdef __cplusplus +// TODO: only needed for testing, should be encased in a macro +void set_tasks(std::vector &tasks, + double alpha, + unsigned int n_samples, + unsigned int stripe_start, + unsigned int stripe_stop, + bool bypass_tips, + unsigned int nthreads); + +#endif diff --git a/R/unifrac_cpp/benchtest.sh b/R/unifrac_cpp/benchtest.sh new file mode 100644 index 000000000..ab9280b1d --- /dev/null +++ b/R/unifrac_cpp/benchtest.sh @@ -0,0 +1,17 @@ +set -e +set -x + +basedir=bench_tables_trees +resdir=$basedir/results +mkdir -p $resdir +for f in $basedir/*.biom +do + bench=${basedir}/$(basename $f .biom) + res=${resdir}/$(basename $f .biom) + for method in {unweighted,weighted_normalized,weighted_unnormalized} + do + /usr/bin/time -l ./su ${bench}.tre ${bench}.biom $method > ${res}.${method}.su.dm 2> ${res}.${method}.su.stats + /usr/bin/time -l ./sk ${bench}.tre ${bench}.biom $method > ${res}.${method}.sk.dm 2> ${res}.${method}.sk.stats + python compare_dms.py ${res}.${method}.sk.dm ${res}.${method}.su.dm + done +done diff --git a/R/unifrac_cpp/biom.cpp b/R/unifrac_cpp/biom.cpp new file mode 100644 index 000000000..a62a9e27f --- /dev/null +++ b/R/unifrac_cpp/biom.cpp @@ -0,0 +1,324 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include +#include +#include +#include "biom.hpp" + +using namespace H5; +using namespace su; + +/* datasets defined by the BIOM 2.x spec */ +const std::string OBS_INDPTR = std::string("/observation/matrix/indptr"); +const std::string OBS_INDICES = std::string("/observation/matrix/indices"); +const std::string OBS_DATA = std::string("/observation/matrix/data"); +const std::string OBS_IDS = std::string("/observation/ids"); + +const std::string SAMPLE_INDPTR = std::string("/sample/matrix/indptr"); +const std::string SAMPLE_INDICES = std::string("/sample/matrix/indices"); +const std::string SAMPLE_DATA = std::string("/sample/matrix/data"); +const std::string SAMPLE_IDS = std::string("/sample/ids"); + +biom::biom(std::string filename) { + file = H5File(filename.c_str(), H5F_ACC_RDONLY); + + /* establish the datasets */ + obs_indices = file.openDataSet(OBS_INDICES.c_str()); + obs_data = file.openDataSet(OBS_DATA.c_str()); + sample_indices = file.openDataSet(SAMPLE_INDICES.c_str()); + sample_data = file.openDataSet(SAMPLE_DATA.c_str()); + + /* cache IDs and indptr */ + sample_ids = std::vector(); + obs_ids = std::vector(); + sample_indptr = std::vector(); + obs_indptr = std::vector(); + + load_ids(OBS_IDS.c_str(), obs_ids); + load_ids(SAMPLE_IDS.c_str(), sample_ids); + load_indptr(OBS_INDPTR.c_str(), obs_indptr); + load_indptr(SAMPLE_INDPTR.c_str(), sample_indptr); + + /* cache shape and nnz info */ + n_samples = sample_ids.size(); + n_obs = obs_ids.size(); + set_nnz(); + + /* define a mapping between an ID and its corresponding offset */ + obs_id_index = std::unordered_map(); + sample_id_index = std::unordered_map(); + + create_id_index(obs_ids, obs_id_index); + create_id_index(sample_ids, sample_id_index); + + /* load obs sparse data */ + obs_indices_resident = (uint32_t**)malloc(sizeof(uint32_t**) * n_obs); + if(obs_indices_resident == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(uint32_t**) * n_obs, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + obs_data_resident = (double**)malloc(sizeof(double**) * n_obs); + if(obs_data_resident == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(double**) * n_obs, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + obs_counts_resident = (unsigned int*)malloc(sizeof(unsigned int) * n_obs); + if(obs_counts_resident == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(unsigned int) * n_obs, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + + uint32_t *current_indices = NULL; + double *current_data = NULL; + for(unsigned int i = 0; i < obs_ids.size(); i++) { + std::string id_ = obs_ids[i]; + unsigned int n = get_obs_data_direct(id_, current_indices, current_data); + obs_counts_resident[i] = n; + obs_indices_resident[i] = current_indices; + obs_data_resident[i] = current_data; + } + sample_counts = get_sample_counts(); +} + +biom::~biom() { + for(unsigned int i = 0; i < n_obs; i++) { + free(obs_indices_resident[i]); + free(obs_data_resident[i]); + } + free(obs_indices_resident); + free(obs_data_resident); + free(obs_counts_resident); +} + +void biom::set_nnz() { + // should these be cached? + DataType dtype = obs_data.getDataType(); + DataSpace dataspace = obs_data.getSpace(); + + hsize_t dims[1]; + dataspace.getSimpleExtentDims(dims, NULL); + nnz = dims[0]; +} + +void biom::load_ids(const char *path, std::vector &ids) { + DataSet ds_ids = file.openDataSet(path); + DataType dtype = ds_ids.getDataType(); + DataSpace dataspace = ds_ids.getSpace(); + + hsize_t dims[1]; + dataspace.getSimpleExtentDims(dims, NULL); + + /* the IDs are a dataset of variable length strings */ + char **dataout = (char**)malloc(sizeof(char*) * dims[0]); + if(dataout == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(char*) * dims[0], __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + ds_ids.read((void*)dataout, dtype); + + ids.reserve(dims[0]); + for(unsigned int i = 0; i < dims[0]; i++) { + ids.push_back(dataout[i]); + } + + for(unsigned int i = 0; i < dims[0]; i++) + free(dataout[i]); + free(dataout); +} + +void biom::load_indptr(const char *path, std::vector &indptr) { + DataSet ds = file.openDataSet(path); + DataType dtype = ds.getDataType(); + DataSpace dataspace = ds.getSpace(); + + hsize_t dims[1]; + dataspace.getSimpleExtentDims(dims, NULL); + + uint32_t *dataout = (uint32_t*)malloc(sizeof(uint32_t) * dims[0]); + if(dataout == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(uint32_t) * dims[0], __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + ds.read((void*)dataout, dtype); + + indptr.reserve(dims[0]); + for(unsigned int i = 0; i < dims[0]; i++) + indptr.push_back(dataout[i]); + free(dataout); +} + +void biom::create_id_index(std::vector &ids, + std::unordered_map &map) { + uint32_t count = 0; + map.reserve(ids.size()); + for(auto i = ids.begin(); i != ids.end(); i++, count++) { + map[*i] = count; + } +} + +unsigned int biom::get_obs_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out) { + uint32_t idx = obs_id_index.at(id); + uint32_t start = obs_indptr[idx]; + uint32_t end = obs_indptr[idx + 1]; + + hsize_t count[1] = {end - start}; + hsize_t offset[1] = {start}; + + DataType indices_dtype = obs_indices.getDataType(); + DataType data_dtype = obs_data.getDataType(); + + DataSpace indices_dataspace = obs_indices.getSpace(); + DataSpace data_dataspace = obs_data.getSpace(); + + DataSpace indices_memspace(1, count, NULL); + DataSpace data_memspace(1, count, NULL); + + indices_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); + data_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); + + current_indices_out = (uint32_t*)malloc(sizeof(uint32_t) * count[0]); + if(current_indices_out == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(uint32_t) * count[0], __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + current_data_out = (double*)malloc(sizeof(double) * count[0]); + if(current_data_out == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(double) * count[0], __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + + obs_indices.read((void*)current_indices_out, indices_dtype, indices_memspace, indices_dataspace); + obs_data.read((void*)current_data_out, data_dtype, data_memspace, data_dataspace); + + return count[0]; +} + +template +void biom::get_obs_data_TT(const std::string &id, TFloat* out) const { + uint32_t idx = obs_id_index.at(id); + unsigned int count = obs_counts_resident[idx]; + const uint32_t * const indices = obs_indices_resident[idx]; + const double * const data = obs_data_resident[idx]; + + // reset our output buffer + for(unsigned int i = 0; i < n_samples; i++) + out[i] = 0.0; + + for(unsigned int i = 0; i < count; i++) { + out[indices[i]] = data[i]; + } +} + +void biom::get_obs_data(const std::string &id, double* out) const { + biom::get_obs_data_TT(id,out); +} + +void biom::get_obs_data(const std::string &id, float* out) const { + biom::get_obs_data_TT(id,out); +} + + +// note: out is supposed to be fully filled, i.e. out[start:end] +template +void biom::get_obs_data_range_TT(const std::string &id, unsigned int start, unsigned int end, bool normalize, TFloat* out) const { + uint32_t idx = obs_id_index.at(id); + unsigned int count = obs_counts_resident[idx]; + const uint32_t * const indices = obs_indices_resident[idx]; + const double * const data = obs_data_resident[idx]; + + // reset our output buffer + for(unsigned int i = start; i < end; i++) + out[i-start] = 0.0; + + if (normalize) { + for(unsigned int i = 0; i < count; i++) { + const int32_t j = indices[i]; + if ((j>=start)&&(j=start)&&(j +#include +#include +#include + +#include "biom_interface.hpp" + +namespace su { + class biom : public biom_interface { + public: + /* default constructor + * + * @param filename The path to the BIOM table to read + */ + biom(std::string filename); + + /* default destructor + * + * Temporary arrays are freed + */ + virtual ~biom(); + + /* get a dense vector of observation data + * + * @param id The observation ID to fetch + * @param out An allocated array of at least size n_samples. + * Values of an index position [0, n_samples) which do not + * have data will be zero'd. + */ + void get_obs_data(const std::string &id, double* out) const; + void get_obs_data(const std::string &id, float* out) const; + + /* get a dense vector of a range of observation data + * + * @param id The observation ID to fetc + * @param start Initial index + * @param end First index past the end + * @param normalize If set, divide by sample_counts + * @param out An allocated array of at least size (end-start). First element will corrrectpoint to index start. + * Values of an index position [0, (end-start)) which do not + * have data will be zero'd. + */ + void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, double* out) const; + void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, float* out) const; + + private: + /* retain DataSet handles within the HDF5 file */ + H5::DataSet obs_indices; + H5::DataSet sample_indices; + H5::DataSet obs_data; + H5::DataSet sample_data; + H5::H5File file; + uint32_t **obs_indices_resident; + double **obs_data_resident; + unsigned int *obs_counts_resident; + + unsigned int get_obs_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out); + unsigned int get_sample_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out); + double* get_sample_counts(); + + /* At construction, lookups mapping IDs -> index position within an + * axis are defined + */ + std::unordered_map obs_id_index; + std::unordered_map sample_id_index; + + /* load ids from an axis + * + * @param path The dataset path to the ID dataset to load + * @param ids The variable representing the IDs to load into + */ + void load_ids(const char *path, std::vector &ids); + + /* load the index pointer for an axis + * + * @param path The dataset path to the index pointer to load + * @param indptr The vector to load the data into + */ + void load_indptr(const char *path, std::vector &indptr); + + /* count the number of nonzero values and set nnz */ + void set_nnz(); + + /* create an index mapping an ID to its corresponding index + * position. + * + * @param ids A vector of IDs to index + * @param map A hash table to populate + */ + void create_id_index(std::vector &ids, + std::unordered_map &map); + + + // templatized version + template void get_obs_data_TT(const std::string &id, TFloat* out) const; + template void get_obs_data_range_TT(const std::string &id, unsigned int start, unsigned int end, bool normalize, TFloat* out) const; + }; +} + +#endif /* _UNIFRAC_BIOM_H */ + diff --git a/R/unifrac_cpp/biom_interface.hpp b/R/unifrac_cpp/biom_interface.hpp new file mode 100644 index 000000000..bbfc80e4d --- /dev/null +++ b/R/unifrac_cpp/biom_interface.hpp @@ -0,0 +1,72 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2021-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + + +#ifndef _UNIFRAC_BIOM_INTERFACE_H +#define _UNIFRAC_BIOM_INTERFACE_H + +#include +#include + +namespace su { + class biom_interface { + public: + // cache the IDs contained within the table + std::vector sample_ids; + std::vector obs_ids; + + // cache both index pointers into both CSC and CSR representations + std::vector sample_indptr; + std::vector obs_indptr; + + uint32_t n_samples; // the number of samples + uint32_t n_obs; // the number of observations + uint32_t nnz; // the total number of nonzero entries + double *sample_counts; + + /* default constructor + * + * Automatically create the needed objects. + * All other initialization happens in children constructors. + */ + biom_interface() {} + + /* default destructor + * + * Automatically destroy the objects. + * All other cleanup must have been performed by the children constructors. + */ + virtual ~biom_interface() {} + + /* get a dense vector of observation data + * + * @param id The observation ID to fetch + * @param out An allocated array of at least size n_samples. + * Values of an index position [0, n_samples) which do not + * have data will be zero'd. + */ + virtual void get_obs_data(const std::string &id, double* out) const = 0; + virtual void get_obs_data(const std::string &id, float* out) const = 0; + + /* get a dense vector of a range of observation data + * + * @param id The observation ID to fetc + * @param start Initial index + * @param end First index past the end + * @param normalize If set, divide by sample_counts + * @param out An allocated array of at least size (end-start). First element will corrrectpoint to index start. + * Values of an index position [0, (end-start)) which do not + * have data will be zero'd. + */ + virtual void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, double* out) const = 0; + virtual void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, float* out) const = 0; + }; +} + +#endif /* _UNIFRAC_BIOOM_INTERFACE_H */ diff --git a/R/unifrac_cpp/capi_test.c b/R/unifrac_cpp/capi_test.c new file mode 100644 index 000000000..c1981f048 --- /dev/null +++ b/R/unifrac_cpp/capi_test.c @@ -0,0 +1,71 @@ +#include +#include +#include +#include +#include "api.hpp" + +#ifndef bool +#define bool char +#define true 1 +#define false 0 +#endif + +void err(bool condition, const char* msg) { + if(condition) { + fprintf(stderr, "%s\n", msg); + exit(1); + } +} + +void test_su(int num_cores){ + mat_t* result = NULL; + const char* table = "test.biom"; + const char* tree = "test.tre"; + const char* method = "unweighted"; + double exp[] = {0.2, 0.57142857, 0.6, 0.5, 0.2, 0.42857143, 0.66666667, 0.6, 0.33333333, 0.71428571, 0.85714286, 0.42857143, 0.33333333, 0.4, 0.6}; + + ComputeStatus status; + status = one_off(table, tree, method, + false, 1.0, false, num_cores, &result); + + err(status != okay, "Compute failed"); + err(result == NULL, "Empty result"); + err(result->n_samples != 6, "Wrong number of samples"); + err(result->cf_size != 15, "Wrong condensed form size"); + err(!result->is_upper_triangle, "Result is not squaure"); + + for(unsigned int i = 0; i < result->cf_size; i++) + err(fabs(exp[i] - result->condensed_form[i]) > 0.00001, "Result is wrong"); + +} + +void test_faith_pd(){ + r_vec* result = NULL; + const char* table = "test.biom"; + const char* tree = "test.tre"; + double exp[] = {4, 5, 6, 3, 2, 5}; + + ComputeStatus status; + status = faith_pd_one_off(table, tree, &result); + + err(status != okay, "Compute failed"); + err(result == NULL, "Empty result"); + err(result->n_samples != 6, "Wrong number of samples"); + + for(unsigned int i = 0; i < result->n_samples; i++) + err(fabs(exp[i] - result->values[i]) > 0.00001, "Result is wrong"); + +} + +int main(int argc, char** argv) { + int num_cores = strtol(argv[1], NULL, 10); + + printf("Testing Striped UniFrac...\n"); + test_su(num_cores); + printf("Tests passed.\n"); + printf("Testing Faith's PD...\n"); + test_faith_pd(); + printf("Tests passed.\n"); + return 0; +} + diff --git a/R/unifrac_cpp/cmd.cpp b/R/unifrac_cpp/cmd.cpp new file mode 100644 index 000000000..4cfd31083 --- /dev/null +++ b/R/unifrac_cpp/cmd.cpp @@ -0,0 +1 @@ +#include "cmd.hpp" diff --git a/R/unifrac_cpp/cmd.hpp b/R/unifrac_cpp/cmd.hpp new file mode 100644 index 000000000..408a169e3 --- /dev/null +++ b/R/unifrac_cpp/cmd.hpp @@ -0,0 +1,32 @@ +#include +#include +#include +#include + +class InputParser{ + /* this object was shamelessly adapted from + http://stackoverflow.com/a/868894 + */ + public: + InputParser (int &argc, char **argv){ + for (int i=1; i < argc; ++i) + this->tokens.push_back(std::string(argv[i])); + } + /// @author iain + const std::string& getCmdOption(const std::string &option) const{ + std::vector::const_iterator itr; + itr = std::find(this->tokens.begin(), this->tokens.end(), option); + if (itr != this->tokens.end() && ++itr != this->tokens.end()){ + return *itr; + } + return empty; + } + /// @author iain + bool cmdOptionExists(const std::string &option) const{ + return std::find(this->tokens.begin(), this->tokens.end(), option) + != this->tokens.end(); + } + private: + std::vector tokens; + const std::string empty; +}; diff --git a/R/unifrac_cpp/faithpd.cpp b/R/unifrac_cpp/faithpd.cpp new file mode 100644 index 000000000..206749653 --- /dev/null +++ b/R/unifrac_cpp/faithpd.cpp @@ -0,0 +1,84 @@ +#include +#include +#include +#include +#include "api.hpp" +#include "cmd.hpp" +#include "tree.hpp" +#include "biom.hpp" +#include "unifrac.hpp" + + +void usage() { + std::cout << "usage: faithpd -i -t -o " << std::endl; + std::cout << std::endl; + std::cout << " -i\t\tThe input BIOM table." << std::endl; + std::cout << " -t\t\tThe input phylogeny in newick." << std::endl; + std::cout << " -o\t\tThe output series." << std::endl; + std::cout << std::endl; + std::cout << "Citations: " << std::endl; + std::cout << " For Faith's PD, please see:" << std::endl; + std::cout << " Faith Biological Conservation 1992; DOI: 10.1016/0006-3207(92)91201-3" << std::endl; + std::cout << std::endl; + +} + +const char* compute_status_messages[7] = {"No error.", + "The tree file cannot be found.", + "The table file cannot be found.", + "The table file contains an empty table.", + "An unknown method was requested.", + "Table observation IDs are not a subset of the tree tips. This error can also be triggered if a node name contains a single quote (this is unlikely).", + "Error creating the output."}; + +void err(std::string msg) { + std::cerr << "ERROR: " << msg << std::endl << std::endl; + usage(); +} + +int faith_cli_one_off(std::string table_filename, std::string tree_filename, + std::string output_filename) { + if(output_filename.empty()) { + err("output filename missing"); + return EXIT_FAILURE; + } + + if(table_filename.empty()) { + err("table filename missing"); + return EXIT_FAILURE; + } + + if(tree_filename.empty()) { + err("tree filename missing"); + return EXIT_FAILURE; + } + + r_vec *result = NULL; + compute_status status; + status = faith_pd_one_off(table_filename.c_str(), tree_filename.c_str(), &result); + if(status != okay || result == NULL) { + fprintf(stderr, "Compute failed in faith_pd_one_off: %s\n", compute_status_messages[status]); + exit(EXIT_FAILURE); + } + + write_vec(output_filename.c_str(), result); + destroy_results_vec(&result); + + return EXIT_SUCCESS; +} + +int main(int argc, char **argv){ + InputParser input(argc, argv); + if(input.cmdOptionExists("-h") || input.cmdOptionExists("--help") || argc == 1) { + usage(); + return EXIT_SUCCESS; + } + + const std::string &table_filename = input.getCmdOption("-i"); + const std::string &tree_filename = input.getCmdOption("-t"); + const std::string &output_filename = input.getCmdOption("-o"); + + faith_cli_one_off(table_filename, tree_filename, output_filename); + + return EXIT_SUCCESS; +} diff --git a/R/unifrac_cpp/skbio_alt.cpp b/R/unifrac_cpp/skbio_alt.cpp new file mode 100644 index 000000000..464f6f78b --- /dev/null +++ b/R/unifrac_cpp/skbio_alt.cpp @@ -0,0 +1,617 @@ +/* + * Classes, methods and unction that provide skbio-like unctionality + */ + +#include "skbio_alt.hpp" +#include + +#include + +// Not using anything mkl specific, but this is what we get from Conda +#include +#include + +// Compute the E_matrix with means +// centered must be pre-allocated and same size as mat (n_samples*n_samples)...will work even if centered==mat +// row_means must be pre-allocated and n_samples in size +template +inline void E_matrix_means(const TRealIn * mat, const uint32_t n_samples, // IN + TReal * centered, TReal * row_means, TReal &global_mean) { // OUT + /* + Compute E matrix from a distance matrix and store in temp centered matrix. + + Squares and divides by -2 the input elementwise. Eq. 9.20 in + Legendre & Legendre 1998. + + Compute sum of the rows at the same time. + */ + + TReal global_sum = 0.0; + +#pragma omp parallel for shared(mat,centered,row_means) reduction(+: global_sum) + for (uint32_t row=0; row +inline void F_matrix_inplace(const TReal * __restrict__ row_means, const TReal global_mean, TReal * __restrict__ centered, const uint32_t n_samples) { + /* + Compute F matrix from E matrix. + + Centring step: for each element, the mean of the corresponding + row and column are substracted, and the mean of the whole + matrix is added. Eq. 9.21 in Legendre & Legendre 1998. + Pseudo-code: + row_means = E_matrix.mean(axis=1, keepdims=True) + col_means = Transpose(row_means) + matrix_mean = E_matrix.mean() + return E_matrix - row_means - col_means + matrix_mean + */ + + // use a tiled pattern to maximize locality of row_means +#pragma omp parallel for shared(centered,row_means) + for (uint32_t trow=0; trow +inline void mat_to_centered_T(const TRealIn * mat, const uint32_t n_samples, TReal * centered) { + + TReal global_mean; + TReal *row_means = (TReal *) malloc(uint64_t(n_samples)*sizeof(TReal)); + E_matrix_means(mat, n_samples, centered, row_means, global_mean); + F_matrix_inplace(row_means, global_mean, centered, n_samples); + free(row_means); +} + +void su::mat_to_centered(const double * mat, const uint32_t n_samples, double * centered) { + mat_to_centered_T(mat,n_samples,centered); +} + +void su::mat_to_centered(const float * mat, const uint32_t n_samples, float * centered) { + mat_to_centered_T(mat,n_samples,centered); +} + +void su::mat_to_centered(const double * mat, const uint32_t n_samples, float * centered) { + mat_to_centered_T(mat,n_samples,centered); +} + +// Matrix dot multiplication +// Expects FORTRAN-style ColOrder +// mat must be cols x rows +// other must be cols x rows (ColOrder... rows elements together) +template +inline void mat_dot_T(const TReal *mat, const TReal *other, const uint32_t rows, const uint32_t cols, TReal *out); + +template<> +inline void mat_dot_T(const double *mat, const double *other, const uint32_t rows, const uint32_t cols, double *out) +{ + cblas_dgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, other, rows, 0.0, out, rows); +} + +template<> +inline void mat_dot_T(const float *mat, const float *other, const uint32_t rows, const uint32_t cols, float *out) +{ + cblas_sgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, other, rows, 0.0, out, rows); +} + +// Expects FORTRAN-style ColOrder +// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. +// Original Paper: https://arxiv.org/abs/1007.5510 +// Step 1 +// centered == n x n +// randomized = k*2 x n (ColOrder... n elements together) +template +inline void centered_randomize_T(const TReal * centered, const uint32_t n_samples, const uint32_t k, TReal * randomized) { + uint64_t matrix_els = uint64_t(n_samples)*uint64_t(k); + TReal * tmp = (TReal *) malloc(matrix_els*sizeof(TReal)); + + // Form a real n x k matrix whose entries are independent, identically + // distributed Gaussian random variables of zero mean and unit variance + TReal *G = tmp; + { + std::default_random_engine generator; + std::normal_distribution distribution; + for (uint64_t i=0; i(centered,G,n_samples,k,randomized); + + // power method... single iteration.. store in 2nd part of output + // Reusing tmp buffer for intermediate storage + mat_dot_T(centered,randomized,n_samples,k,tmp); + mat_dot_T(centered,tmp,n_samples,k,randomized+matrix_els); + + free(tmp); +} + +// templated LAPACKE wrapper + +// Compute QR +// H is in,overwritten by Q on out +// H is (r x c), Q is (r x qc), with rc<=c +template +inline int qr_i_T(const uint32_t rows, const uint32_t cols, TReal *H, uint32_t &qcols); + +template<> +inline int qr_i_T(const uint32_t rows, const uint32_t cols, double *H, uint32_t &qcols) { + qcols= std::min(rows,cols); + double *tau= new double[qcols]; + int rc = LAPACKE_dgeqrf(LAPACK_COL_MAJOR, rows, cols, H, rows, tau); + if (rc==0) { + qcols= std::min(rows,cols); + rc = LAPACKE_dorgqr(LAPACK_COL_MAJOR, rows, qcols, qcols, H, rows, tau); + } + delete[] tau; + return rc; +} + +template<> +inline int qr_i_T(const uint32_t rows, const uint32_t cols, float *H, uint32_t &qcols) { + qcols= std::min(rows,cols); + float *tau= new float[qcols]; + int rc = LAPACKE_sgeqrf(LAPACK_COL_MAJOR, rows, cols, H, rows, tau); + if (rc==0) { + qcols= std::min(rows,cols); + rc = LAPACKE_sorgqr(LAPACK_COL_MAJOR, rows, qcols, qcols, H, rows, tau); + } + delete[] tau; + return rc; +} + +namespace su { + +// helper class, since QR ops are multi function +template +class QR { + public: + uint32_t rows; + uint32_t cols; + + TReal *Q; + + // will take ownership of _H + QR(const uint32_t _rows, const uint32_t _cols, TReal *_H) + : rows(_rows) + , Q(_H) + { + int rc = qr_i_T(_rows, _cols, Q, cols); + if (rc!=0) { + fprintf(stderr, "qr_i_T(_rows,_cols, H, cols) failed with %i\n", rc); + exit(1); // should never fail + } + } + + ~QR() { + free(Q); + } + + // res = mat * Q + // mat must be rows x rows + // res will be rows * cols + void qdot_r_sq(const TReal *mat, TReal *res); + + // res = Q * mat + // mat must be cols * cols + // res will be rows * cols + void qdot_l_sq(const TReal *mat, TReal *res); + +}; + +} + +template<> +inline void su::QR::qdot_r_sq(const double *mat, double *res) { + cblas_dgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, Q, rows, 0.0, res, rows); +} + +template<> +inline void su::QR::qdot_r_sq(const float *mat, float *res) { + cblas_sgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, Q, rows, 0.0, res, rows); +} + +template<> +inline void su::QR::qdot_l_sq(const double *mat, double *res) { + cblas_dgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, cols, 1.0, Q, rows, mat, cols, 0.0, res, rows); +} + +template<> +inline void su::QR::qdot_l_sq(const float *mat, float *res) { + cblas_sgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, cols, 1.0, Q, rows, mat, cols, 0.0, res, rows); +} + +// compute svd, and return S and V +// T = input +// S output +// T is Vt on output +template +inline int svd_it_T(const uint32_t rows, const uint32_t cols, TReal *T, TReal *S); + +template<> +inline int svd_it_T(const uint32_t rows, const uint32_t cols, double *T, double *S) { + double *superb = (double *) malloc(sizeof(double)*rows); + int res =LAPACKE_dgesvd(LAPACK_COL_MAJOR, 'N', 'O', rows, cols, T, rows, S, NULL, rows, NULL, cols, superb); + free(superb); + + return res; +} + +template<> +inline int svd_it_T(const uint32_t rows, const uint32_t cols, float *T, float *S) { + float *superb = (float *) malloc(sizeof(float)*rows); + int res =LAPACKE_sgesvd(LAPACK_COL_MAJOR, 'N', 'O', rows, cols, T, rows, S, NULL, rows, NULL, cols, superb); + free(superb); + + return res; +} + +// square matrix transpose, with org not alingned +template +inline void transpose_sq_st_T(const uint64_t n, const uint64_t stride, const TReal *in, TReal *out) { + // n expected to be small, so simple single-thread perfect + // org_n>=n guaranteed + for (uint64_t i=0; i +inline void transpose_T(const uint64_t rows, const uint64_t cols, const TReal *in, TReal *out) { + // To be optimizedc + for (uint64_t i=0; i +inline void find_eigens_fast_T(const uint32_t n_samples, const uint32_t n_dims, TReal * centered, TReal * &eigenvalues, TReal * &eigenvectors) { + const uint32_t k = n_dims+2; + + int rc; + + TReal *S = (TReal *) malloc(uint64_t(n_samples)*sizeof(TReal)); // take worst case size as a start + TReal *Ut = NULL; + + { + TReal *H = (TReal *) malloc(sizeof(TReal)*uint64_t(n_samples)*uint64_t(k)*2); + + // step 1 + centered_randomize_T(centered, n_samples, k, H); + + // step 2 + // QR decomposition of H + + su::QR qr_obj(n_samples, k*2, H); // H is now owned by qr_obj, as Q + + // step 3 + // T = centered * Q (since centered^T == centered, due to being symmetric) + // centered = n x n + // T = n x ref + + TReal *T = (TReal *) malloc(sizeof(TReal)*uint64_t(qr_obj.rows)*uint64_t(qr_obj.cols)); + qr_obj.qdot_r_sq(centered,T); + + // step 4 + // compute svd + // update T in-place, Wt on output (Vt according to the LAPACK nomenclature) + rc=svd_it_T(qr_obj.rows,qr_obj.cols, T, S); + if (rc!=0) { + fprintf(stderr, "svd_it_T(n_samples, T, S) failed with %i\n",rc); + exit(1); // should never fail + } + + // step 5 + // Compute U = Q*Wt^t + { + // transpose Wt -> W, Wt uses n_samples strides + TReal * W = (TReal *) malloc(sizeof(TReal)*uint64_t(qr_obj.cols)*uint64_t(qr_obj.cols)); + transpose_sq_st_T(qr_obj.cols, qr_obj.rows, T, W); // Wt == T on input + + Ut = T; // Ut takes ownership of the T buffer + qr_obj.qdot_l_sq(W, Ut); + + free(W); + } + + } // we don't need qr_obj anymore, release memory + + // step 6 + // get the interesting subset, and return + + // simply truncate the values, since it is a vector + eigenvalues = (TReal *) realloc(S, sizeof(TReal)*n_dims); + + // *eigenvectors = U = Vt + // use only the truncated part of W, then transpose + TReal *U = (TReal *) malloc(uint64_t(n_samples)*uint64_t(n_dims)*sizeof(TReal)); + + transpose_T(n_samples, n_dims, Ut, U); + eigenvectors = U; + + free(Ut); +} + +void su::find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double * &eigenvalues, double * &eigenvectors) { + find_eigens_fast_T(n_samples, n_dims, centered, eigenvalues, eigenvectors); +} + +void su::find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, float * centered, float * &eigenvalues, float * &eigenvectors) { + find_eigens_fast_T(n_samples, n_dims, centered, eigenvalues, eigenvectors); +} + +// helper class + +namespace su { + +template +class NewCentered { +private: + const uint32_t n_samples; + const uint32_t n_dims; + TReal * centered_buf; +public: + NewCentered(const uint32_t _n_samples, const uint32_t _n_dims) + : n_samples(_n_samples) + , n_dims(_n_dims) + , centered_buf(NULL) + {} + + TReal * get_buf() { + if (centered_buf==NULL) centered_buf = (TReal *) malloc(sizeof(TReal)*uint64_t(n_samples)*uint64_t(n_samples)); + return centered_buf; + } + + void release_buf() { + if (centered_buf!=NULL) free(centered_buf); + centered_buf=NULL; + } + + ~NewCentered() { + if (centered_buf!=NULL) release_buf(); + } + +private: + NewCentered(const NewCentered &other) = delete; + NewCentered& operator=(const NewCentered &other) = delete; +}; + +template +class InPlaceCentered { +private: + TReal * mat; +public: + InPlaceCentered(TReal * _mat) + : mat(_mat) + {} + + TReal * get_buf() { return mat; } + + void release_buf() {} + + ~InPlaceCentered() {} +}; + +} + +/* + Perform Principal Coordinate Analysis. + + Principal Coordinate Analysis (PCoA) is a method similar + to Principal Components Analysis (PCA) with the difference that PCoA + operates on distance matrices, typically with non-euclidian and thus + ecologically meaningful distances like UniFrac in microbiome research. + + In ecology, the euclidean distance preserved by Principal + Component Analysis (PCA) is often not a good choice because it + deals poorly with double zeros (Species have unimodal + distributions along environmental gradients, so if a species is + absent from two sites at the same site, it can't be known if an + environmental variable is too high in one of them and too low in + the other, or too low in both, etc. On the other hand, if an + species is present in two sites, that means that the sites are + similar.). + + Note that the returned eigenvectors are not normalized to unit length. +*/ + +// mat - in, result of unifrac compute +// inplace - in, if true, use mat as a work buffer +// n_samples - in, size of the matrix (n x n) +// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. +// eigenvalues - out, alocated buffer of size n_dims +// samples - out, alocated buffer of size n_dims x n_samples +// proportion_explained - out, allocated buffer of size n_dims + +template +inline void pcoa_T(TRealIn * mat, TCenter ¢er_obj, const uint32_t n_samples, const uint32_t n_dims, TReal * &eigenvalues, TReal * &samples,TReal * &proportion_explained) { + proportion_explained = (TReal *) malloc(sizeof(TReal)*n_dims); + + TReal diag_sum = 0.0; + TReal *eigenvectors = NULL; + + { + TReal *centered = center_obj.get_buf(); + + // First must center the matrix + mat_to_centered_T(mat,n_samples,centered); + + // get the sum of the diagonal, needed later + // and centered will be updated in-place in find_eigen + for (uint32_t i=0; i(n_samples,n_dims,centered,eigenvalues,eigenvectors); + + center_obj.release_buf(); + } + + // expects eigenvalues to be ordered and non-negative + // The above unction guarantees that + + + // Scale eigenvalues to have length = sqrt(eigenvalue). This + // works because np.linalg.eigh returns normalized + // eigenvectors. Each row contains the coordinates of the + // objects in the space of principal coordinates. Note that at + // least one eigenvalue is zero because only n-1 axes are + // needed to represent n points in a euclidean space. + // samples = eigvecs * np.sqrt(eigvals) + // we will just update in place and pass out + samples = eigenvectors; + + // use proportion_explained as tmp buffer here + { + TReal *sqvals = proportion_explained; + for (uint32_t i=0; i cobj(n_samples, n_dims); + pcoa_T(mat, cobj , n_samples, n_dims, eigenvalues, samples, proportion_explained); +} + +void su::pcoa(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained) { + su::NewCentered cobj(n_samples, n_dims); + pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); +} + +void su::pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained) { + su::NewCentered cobj(n_samples, n_dims); + pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); +} + +void su::pcoa_inplace(double * mat, const uint32_t n_samples, const uint32_t n_dims, double * &eigenvalues, double * &samples, double * &proportion_explained) { + su::InPlaceCentered cobj(mat); + pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); +} + +void su::pcoa_inplace(float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained) { + su::InPlaceCentered cobj(mat); + pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); +} + diff --git a/R/unifrac_cpp/skbio_alt.hpp b/R/unifrac_cpp/skbio_alt.hpp new file mode 100644 index 000000000..284f0ea6e --- /dev/null +++ b/R/unifrac_cpp/skbio_alt.hpp @@ -0,0 +1,44 @@ +/* + * Classes, methods and unction that provide skbio-like unctionality + */ + +#ifndef UNIFRAC_SKBIO_ALT_H +#define UNIFRAC_SKBIO_ALT_H + +#include + +namespace su { + +// Center the matrix +// mat and center must be nxn and symmetric +// centered must be pre-allocated and same size as mat...will work even if centered==mat +void mat_to_centered(const double * mat, const uint32_t n_samples, double * centered); +void mat_to_centered(const float * mat, const uint32_t n_samples, float * centered); +void mat_to_centered(const double * mat, const uint32_t n_samples, float * centered); + +// Find eigen values and vectors +// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. +// Original Paper: https://arxiv.org/abs/1007.5510 +// centered == n x n, must be symmetric, Note: will be used in-place as temp buffer +void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double * &eigenvalues, double * &eigenvectors); +void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, float * centered, float * &eigenvalues, float * &eigenvectors); + +// Perform Principal Coordinate Analysis +// mat - in, result of unifrac compute +// n_samples - in, size of the matrix (n x n) +// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. +// eigenvalues - out, alocated buffer of size n_dims +// samples - out, alocated buffer of size n_dims x n_samples +// proportion_explained - out, allocated buffer of size n_dims +void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, double * &eigenvalues, double * &samples, double * &proportion_explained); +void pcoa(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained); +void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained); + +// in-place version, will use mat as temp buffer internally +void pcoa_inplace(double * mat, const uint32_t n_samples, const uint32_t n_dims, double * &eigenvalues, double * &samples, double * &proportion_explained); +void pcoa_inplace(float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained); + + +} + +#endif diff --git a/R/unifrac_cpp/su.cpp b/R/unifrac_cpp/su.cpp new file mode 100644 index 000000000..25c238738 --- /dev/null +++ b/R/unifrac_cpp/su.cpp @@ -0,0 +1,492 @@ +#include +#include +#include +#include +#include +#include +#include "api.hpp" +#include "cmd.hpp" +#include "tree.hpp" +#include "biom.hpp" +#include "unifrac.hpp" + +enum Format {format_invalid,format_ascii, format_hdf5_fp32, format_hdf5_fp64}; + +void usage() { + std::cout << "usage: ssu -i -o -m [METHOD] -t [-n threads] [-a alpha] [-f] [--vaw]" << std::endl; + std::cout << " [--mode [MODE]] [--start starting-stripe] [--stop stopping-stripe] [--partial-pattern ]" << std::endl; + std::cout << " [--n-partials number_of_partitions] [--report-bare] [--format|-r out-mode]" << std::endl; + std::cout << std::endl; + std::cout << " -i\t\tThe input BIOM table." << std::endl; + std::cout << " -t\t\tThe input phylogeny in newick." << std::endl; + std::cout << " -m\t\tThe method, [unweighted | weighted_normalized | weighted_unnormalized | generalized | unweighted_fp32 | weighted_normalized_fp32 | weighted_unnormalized_fp32 | generalized_fp32]." << std::endl; + std::cout << " -o\t\tThe output distance matrix." << std::endl; + std::cout << " -n\t\t[OPTIONAL] The number of threads, default is 1." << std::endl; + std::cout << " -a\t\t[OPTIONAL] Generalized UniFrac alpha, default is 1." << std::endl; + std::cout << " -f\t\t[OPTIONAL] Bypass tips, reduces compute by about 50%." << std::endl; + std::cout << " --vaw\t[OPTIONAL] Variance adjusted, default is to not adjust for variance." << std::endl; + std::cout << " --mode\t[OPTIONAL] Mode of operation:" << std::endl; + std::cout << " \t\t one-off : [DEFAULT] compute UniFrac." << std::endl; + std::cout << " \t\t partial : Compute UniFrac over a subset of stripes." << std::endl; + std::cout << " \t\t partial-report : Start and stop suggestions for partial compute." << std::endl; + std::cout << " \t\t merge-partial : Merge partial UniFrac results." << std::endl; + std::cout << " --start\t[OPTIONAL] If mode==partial, the starting stripe." << std::endl; + std::cout << " --stop\t[OPTIONAL] If mode==partial, the stopping stripe." << std::endl; + std::cout << " --partial-pattern\t[OPTIONAL] If mode==merge-partial, a glob pattern for partial outputs to merge." << std::endl; + std::cout << " --n-partials\t[OPTIONAL] If mode==partial-report, the number of partitions to compute." << std::endl; + std::cout << " --report-bare\t[OPTIONAL] If mode==partial-report, produce barebones output." << std::endl; + std::cout << " --format|-r\t[OPTIONAL] Output format:" << std::endl; + std::cout << " \t\t ascii : [DEFAULT] Original ASCII format." << std::endl; + std::cout << " \t\t hfd5 : HFD5 format. May be fp32 or fp64, depending on method." << std::endl; + std::cout << " \t\t hdf5_fp32 : HFD5 format, using fp32 precision." << std::endl; + std::cout << " \t\t hdf5_fp64 : HFD5 format, using fp64 precision." << std::endl; + std::cout << " --pcoa\t[OPTIONAL] Number of PCoA dimensions to compute (default: 10, do not compute if 0)" << std::endl; + std::cout << " --diskbuf\t[OPTIONAL] Use a disk buffer to reduce memory footprint. Provide path to a fast partition (ideally NVMe)." << std::endl; + std::cout << std::endl; + std::cout << "Citations: " << std::endl; + std::cout << " For UniFrac, please see:" << std::endl; + std::cout << " McDonald et al. Nature Methods 2018; DOI: 10.1038/s41592-018-0187-8" << std::endl; + std::cout << " Lozupone and Knight Appl Environ Microbiol 2005; DOI: 10.1128/AEM.71.12.8228-8235.2005" << std::endl; + std::cout << " Lozupone et al. Appl Environ Microbiol 2007; DOI: 10.1128/AEM.01996-06" << std::endl; + std::cout << " Hamady et al. ISME 2010; DOI: 10.1038/ismej.2009.97" << std::endl; + std::cout << " Lozupone et al. ISME 2011; DOI: 10.1038/ismej.2010.133" << std::endl; + std::cout << " For Generalized UniFrac, please see: " << std::endl; + std::cout << " Chen et al. Bioinformatics 2012; DOI: 10.1093/bioinformatics/bts342" << std::endl; + std::cout << " For Variance Adjusted UniFrac, please see: " << std::endl; + std::cout << " Chang et al. BMC Bioinformatics 2011; DOI: 10.1186/1471-2105-12-118" << std::endl; + std::cout << std::endl; + std::cout << "Runtime progress can be obtained by issuing a SIGUSR1 signal. If running with " << std::endl; + std::cout << "multiple threads, this signal will only be honored if issued to the master PID. " << std::endl; + std::cout << "The report will yield the following information: " << std::endl; + std::cout << std::endl; + std::cout << "tid: start: stop: k: total:" << std::endl; + std::cout << std::endl; + std::cout << "The proportion of the tree that has been evaluated can be determined from (k / total)." << std::endl; + std::cout << std::endl; +} + +const char* compute_status_messages[7] = {"No error.", + "The tree file cannot be found.", + "The table file cannot be found.", + "The table file contains an empty table.", + "An unknown method was requested.", + "Table observation IDs are not a subset of the tree tips. This error can also be triggered if a node name contains a single quote (this is unlikely).", + "Error creating the output."}; + + +// https://stackoverflow.com/questions/8401777/simple-glob-in-c-on-unix-system +inline std::vector glob(const std::string& pat){ + using namespace std; + glob_t glob_result; + glob(pat.c_str(),GLOB_TILDE,NULL,&glob_result); + vector ret; + for(unsigned int i=0;i partials = glob(partial_pattern); + partial_dyn_mat_t** partial_mats = (partial_dyn_mat_t**)malloc(sizeof(partial_dyn_mat_t*) * partials.size()); + for(size_t i = 0; i < partials.size(); i++) { + IOStatus io_err = read_partial_header(partials[i].c_str(), &partial_mats[i]); + if(io_err != read_okay) { + std::ostringstream msg; + msg << "Unable to parse file (" << partials[i] << "); err " << io_err; + err(msg.str()); + return EXIT_FAILURE; + } + } + + const char * mmap_dir_c = mmap_dir.empty() ? NULL : mmap_dir.c_str(); + + int status; + if (format_val==format_hdf5_fp64) { + status = mode_merge_partial_fp64(output_filename.c_str(), format_val, pcoa_dims, partials.size(), partial_mats, mmap_dir_c); + } else if (format_val==format_hdf5_fp32) { + status = mode_merge_partial_fp32(output_filename.c_str(), format_val, pcoa_dims, partials.size(), partial_mats, mmap_dir_c); + } else { + status = mode_merge_partial_fp64(output_filename.c_str(), format_val, pcoa_dims, partials.size(), partial_mats, mmap_dir_c); + } + + for(size_t i = 0; i < partials.size(); i++) { + destroy_partial_dyn_mat(&partial_mats[i]); + } + + return status; +} + +int mode_partial(std::string table_filename, std::string tree_filename, + std::string output_filename, std::string method_string, + bool vaw, double g_unifrac_alpha, bool bypass_tips, + unsigned int nthreads, int start_stripe, int stop_stripe) { + if(output_filename.empty()) { + err("output filename missing"); + return EXIT_FAILURE; + } + + if(table_filename.empty()) { + err("table filename missing"); + return EXIT_FAILURE; + } + + if(tree_filename.empty()) { + err("tree filename missing"); + return EXIT_FAILURE; + } + + if(method_string.empty()) { + err("method missing"); + return EXIT_FAILURE; + } + + if(start_stripe < 0) { + err("Starting stripe must be >= 0"); + return EXIT_FAILURE; + } + if(stop_stripe <= start_stripe) { + err("In '--mode partial', the stop and start stripes must be specified, and the stop stripe must be > start stripe"); + return EXIT_FAILURE; + } + + partial_mat_t *result = NULL; + compute_status status; + status = partial(table_filename.c_str(), tree_filename.c_str(), method_string.c_str(), + vaw, g_unifrac_alpha, bypass_tips, nthreads, start_stripe, stop_stripe, &result); + if(status != okay || result == NULL) { + fprintf(stderr, "Compute failed in partial: %s\n", compute_status_messages[status]); + exit(EXIT_FAILURE); + } + + io_status err = write_partial(output_filename.c_str(), result); + destroy_partial_mat(&result); + + if(err != write_okay){ + fprintf(stderr, "Write failed: %s\n", err == open_error ? "could not open output" : "unknown error"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +int mode_one_off(const std::string &table_filename, const std::string &tree_filename, + const std::string &output_filename, const std::string &format_str, Format format_val, + const std::string &method_string, unsigned int pcoa_dims, + bool vaw, double g_unifrac_alpha, bool bypass_tips, + unsigned int nthreads, const std::string &mmap_dir) { + if(output_filename.empty()) { + err("output filename missing"); + return EXIT_FAILURE; + } + + if(table_filename.empty()) { + err("table filename missing"); + return EXIT_FAILURE; + } + + if(tree_filename.empty()) { + err("tree filename missing"); + return EXIT_FAILURE; + } + + if(method_string.empty()) { + err("method missing"); + return EXIT_FAILURE; + } + + compute_status status = okay; + if (format_val==format_ascii) { + mat_t *result = NULL; + + status = one_off(table_filename.c_str(), tree_filename.c_str(), method_string.c_str(), + vaw, g_unifrac_alpha, bypass_tips, nthreads, &result); + if(status != okay || result == NULL) { + fprintf(stderr, "Compute failed in one_off: %s\n", compute_status_messages[status]); + exit(EXIT_FAILURE); + } + + IOStatus iostatus = write_mat(output_filename.c_str(), result); + destroy_mat(&result); + + if(iostatus!=write_okay) { + err("Failed to write output file."); + status = output_error; + } + + } else { + const char * mmap_dir_c = mmap_dir.empty() ? NULL : mmap_dir.c_str(); + + status = unifrac_to_file(table_filename.c_str(), tree_filename.c_str(), output_filename.c_str(), + method_string.c_str(), vaw, g_unifrac_alpha, bypass_tips, nthreads, format_str.c_str(), + pcoa_dims, mmap_dir_c); + + if (status != okay) { + fprintf(stderr, "Compute failed in one_off: %s\n", compute_status_messages[status]); + } + } + + return (status==okay) ? EXIT_SUCCESS : EXIT_FAILURE; +} + +void ssu_sig_handler(int signo) { + if (signo == SIGUSR1) { + printf("Status cannot be reported.\n"); + } +} + +Format get_format(const std::string &format_string, const std::string &method_string) { + Format format_val = format_invalid; + if (format_string.empty()) { + format_val = format_ascii; + } else if (format_string == "ascii") { + format_val = format_ascii; + } else if (format_string == "hdf5_fp32") { + format_val = format_hdf5_fp32; + } else if (format_string == "hdf5_fp64") { + format_val = format_hdf5_fp64; + } else if (format_string == "hdf5") { + if ((method_string=="unweighted_fp32") || (method_string=="weighted_normalized_fp32") || (method_string=="weighted_unnormalized_fp32") || (method_string=="generalized_fp32")) + format_val = format_hdf5_fp32; + else + format_val = format_hdf5_fp64; + } + + return format_val; +} + +int main(int argc, char **argv){ + signal(SIGUSR1, ssu_sig_handler); + InputParser input(argc, argv); + if(input.cmdOptionExists("-h") || input.cmdOptionExists("--help") || argc == 1) { + usage(); + return EXIT_SUCCESS; + } + + unsigned int nthreads; + std::string table_filename = input.getCmdOption("-i"); + std::string tree_filename = input.getCmdOption("-t"); + std::string output_filename = input.getCmdOption("-o"); + std::string method_string = input.getCmdOption("-m"); + std::string nthreads_arg = input.getCmdOption("-n"); + std::string gunifrac_arg = input.getCmdOption("-a"); + std::string mode_arg = input.getCmdOption("--mode"); + std::string start_arg = input.getCmdOption("--start"); + std::string stop_arg = input.getCmdOption("--stop"); + std::string partial_pattern = input.getCmdOption("--partial-pattern"); + std::string npartials = input.getCmdOption("--n-partials"); + std::string report_bare = input.getCmdOption("--report-bare"); + std::string format_arg = input.getCmdOption("--format"); + std::string sformat_arg = input.getCmdOption("-r"); + std::string pcoa_arg = input.getCmdOption("--pcoa"); + std::string diskbuf_arg = input.getCmdOption("--diskbuf"); + + if(nthreads_arg.empty()) { + nthreads = 1; + } else { + nthreads = atoi(nthreads_arg.c_str()); + } + + bool vaw = input.cmdOptionExists("--vaw"); + bool bare = input.cmdOptionExists("--report-bare"); + bool bypass_tips = input.cmdOptionExists("-f"); + double g_unifrac_alpha; + + if(gunifrac_arg.empty()) { + g_unifrac_alpha = 1.0; + } else { + g_unifrac_alpha = atof(gunifrac_arg.c_str()); + } + + int start_stripe; + if(start_arg.empty()) + start_stripe = 0; + else + start_stripe = atoi(start_arg.c_str()); + + int stop_stripe; + if(stop_arg.empty()) + stop_stripe = 0; + else + stop_stripe = atoi(stop_arg.c_str()); + + int n_partials; + if(npartials.empty()) + n_partials = 1; + else + n_partials = atoi(npartials.c_str()); + + if(n_partials<1) { + err("--n-partials cannot be < 1"); + return EXIT_FAILURE; + } + if(n_partials>1000000000) { + err("--n-partials cannot be > 1G"); + return EXIT_FAILURE; + } + + Format format_val = format_invalid; + if(!format_arg.empty()) { + format_val = get_format(format_arg,method_string); + } else { + format_val = get_format(sformat_arg,method_string); + format_arg=sformat_arg; // easier to use a single variable + } + if(format_val==format_invalid) { + err("Invalid format, must be one of ascii|hdf5|hdf5_fp32|hdf5_fp64"); + return EXIT_FAILURE; + } + + unsigned int pcoa_dims; + if(pcoa_arg.empty()) + pcoa_dims = 10; + else + pcoa_dims = atoi(pcoa_arg.c_str()); + + + if(mode_arg.empty() || mode_arg == "one-off") + return mode_one_off(table_filename, tree_filename, output_filename, format_arg, format_val, method_string, pcoa_dims, vaw, g_unifrac_alpha, bypass_tips, nthreads, diskbuf_arg); + else if(mode_arg == "partial") + return mode_partial(table_filename, tree_filename, output_filename, method_string, vaw, g_unifrac_alpha, bypass_tips, nthreads, start_stripe, stop_stripe); + else if(mode_arg == "merge-partial") + return mode_merge_partial(output_filename, format_val, pcoa_dims, partial_pattern, diskbuf_arg); + else if(mode_arg == "partial-report") + return mode_partial_report(table_filename, uint32_t(n_partials), bare); + else + err("Unknown mode. Valid options are: one-off, partial, merge-partial, partial-report"); + + return EXIT_SUCCESS; +} + diff --git a/R/unifrac_cpp/su_R.cpp b/R/unifrac_cpp/su_R.cpp new file mode 100644 index 000000000..0e2b595ef --- /dev/null +++ b/R/unifrac_cpp/su_R.cpp @@ -0,0 +1,44 @@ +#include +#include +#include +#include "api.hpp" + +using namespace std; +using namespace Rcpp; + + +// [[Rcpp::export]] +Rcpp::List unifrac(const char* table, const char* tree, int nthreads){ + mat_t* result = NULL; + const char* method = "unweighted"; + ComputeStatus status; + status = one_off(table, tree, method, false, 1.0, false, nthreads, &result); + vector cf; + //push result->condensed_form into a vector becuase R doesn't like double* + for(int i=0; icf_size; i++){ + cf.push_back(result->condensed_form[i]); + } + + return Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, + Rcpp::Named("is_upper_triangle") = result->is_upper_triangle, + Rcpp::Named("cf_size") = result->cf_size, + Rcpp::Named("c_form") = cf); + +} + +// [[Rcpp::export]] +Rcpp::List faith_pd(const char* table, const char* tree){ + r_vec* result = NULL; + ComputeStatus status; + status = faith_pd_one_off(table, tree, &result); + vector values; + for(int i = 0; i < result->n_samples; i++){ + values.push_back(result->values[i]); + } + + return Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, + Rcpp::Named("faith_pd") = values); + +} + + diff --git a/R/unifrac_cpp/task_parameters.hpp b/R/unifrac_cpp/task_parameters.hpp new file mode 100644 index 000000000..8f3d53277 --- /dev/null +++ b/R/unifrac_cpp/task_parameters.hpp @@ -0,0 +1,35 @@ +#include +#include + +#ifndef __su_task_parameters + #ifdef __cplusplus + namespace su { + #endif + + /* task specific compute parameters + * + * n_samples the number of samples being processed + * start the first stride to process + * stop the last stride to process + * tid the thread identifier + * bypass_tips ignore tips on compute, reduces compute by ~50% + * g_unifrac_alpha an alpha value for generalized unifrac + */ + struct task_parameters { + uint32_t n_samples; // number of samples + unsigned int start; // starting stripe + unsigned int stop; // stopping stripe + unsigned int tid; // thread ID + bool bypass_tips; // avoid compute at tips + + // task specific arguments below + double g_unifrac_alpha; // generalized unifrac alpha + }; + + #ifdef __cplusplus + } + #endif + +#define __su_task_parameters +#endif + diff --git a/R/unifrac_cpp/test_api.cpp b/R/unifrac_cpp/test_api.cpp new file mode 100644 index 000000000..ac87716cc --- /dev/null +++ b/R/unifrac_cpp/test_api.cpp @@ -0,0 +1,743 @@ +#include +#include "api.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * test harness adapted from + * https://github.com/noporpoise/BitArray/blob/master/dev/bit_array_test.c + */ +const char *suite_name; +char suite_pass; +int suites_run = 0, suites_failed = 0, suites_empty = 0; +int tests_in_suite = 0, tests_run = 0, tests_failed = 0; + +#define QUOTE(str) #str +#define ASSERT(x) {tests_run++; tests_in_suite++; if(!(x)) \ + { fprintf(stderr, "failed assert [%s:%i] %s\n", __FILE__, __LINE__, QUOTE(x)); \ + suite_pass = 0; tests_failed++; }} + +void SUITE_START(const char *name) { + suite_pass = 1; + suite_name = name; + suites_run++; + tests_in_suite = 0; +} + +void SUITE_END() { + printf("Testing %s ", suite_name); + size_t suite_i; + for(suite_i = strlen(suite_name); suite_i < 80-8-5; suite_i++) printf("."); + printf("%s\n", suite_pass ? " pass" : " fail"); + if(!suite_pass) suites_failed++; + if(!tests_in_suite) suites_empty++; +} +/* + * End adapted code + */ + + +//void test_write_mat() { +// SUITE_START("test write mat_t"); +// SUITE_END(); +//} +// +//void test_read_mat() { +// SUITE_START("test read mat_t"); +// SUITE_END(); +//} +// + +template +void fill_test_pm(TMat* pm, int case_id) { + pm->n_samples = 6; + pm->sample_ids = (char**)malloc(sizeof(char*) * 6); + pm->sample_ids[0] = (char*)malloc(sizeof(char) * 2); + pm->sample_ids[0][0] = 'A'; pm->sample_ids[0][1] = '\0'; + pm->sample_ids[1] = (char*)malloc(sizeof(char) * 2); + pm->sample_ids[1][0] = 'B'; pm->sample_ids[1][1] = '\0'; + pm->sample_ids[2] = (char*)malloc(sizeof(char) * 3); + pm->sample_ids[2][0] = 'C'; pm->sample_ids[2][1] = 'x'; pm->sample_ids[2][2] = '\0'; + pm->sample_ids[3] = (char*)malloc(sizeof(char) * 2); + pm->sample_ids[3][0] = 'D'; pm->sample_ids[3][1] = '\0'; + pm->sample_ids[4] = (char*)malloc(sizeof(char) * 2); + pm->sample_ids[4][0] = 'E'; pm->sample_ids[4][1] = '\0'; + pm->sample_ids[5] = (char*)malloc(sizeof(char) * 2); + pm->sample_ids[5][0] = 'F'; pm->sample_ids[5][1] = '\0'; + + if (case_id==0) { + pm->stripe_start = 0; + pm->stripe_stop = 3; + pm->stripe_total = 3; + pm->stripes = (TReal**)malloc(sizeof(TReal*) * 3); + pm->stripes[0] = (TReal*)malloc(sizeof(TReal) * 6); + pm->stripes[0][0] = 1; pm->stripes[0][1] = 2; pm->stripes[0][2] = 3; pm->stripes[0][3] = 4; pm->stripes[0][4] = 5; pm->stripes[0][5] = 6; + pm->stripes[1] = (TReal*)malloc(sizeof(TReal) * 6); + pm->stripes[1][0] = 7; pm->stripes[1][1] = 8; pm->stripes[1][2] = 9; pm->stripes[1][3] = 10; pm->stripes[1][4] = 11; pm->stripes[1][5] = 12; + pm->stripes[2] = (TReal*)malloc(sizeof(TReal) * 6); + pm->stripes[2][0] = 13; pm->stripes[2][1] = 14; pm->stripes[2][2] = 15; pm->stripes[2][3] = 16; pm->stripes[2][4] = 17; pm->stripes[2][5] = 18; + } else if (case_id==1) { + pm->stripe_start = 0; + pm->stripe_stop = 2; + pm->stripe_total = 3; + pm->stripes = (TReal**)malloc(sizeof(TReal*) * 2); + pm->stripes[0] = (TReal*)malloc(sizeof(TReal) * 6); + pm->stripes[0][0] = 1; pm->stripes[0][1] = 2; pm->stripes[0][2] = 3; pm->stripes[0][3] = 4; pm->stripes[0][4] = 5; pm->stripes[0][5] = 6; + pm->stripes[1] = (TReal*)malloc(sizeof(TReal) * 6); + pm->stripes[1][0] = 7; pm->stripes[1][1] = 8; pm->stripes[1][2] = 9; pm->stripes[1][3] = 10; pm->stripes[1][4] = 11; pm->stripes[1][5] = 12; + } else { // assume 2 + pm->stripe_start = 2; + pm->stripe_stop = 3; + pm->stripe_total = 3; + pm->stripes = (TReal**)malloc(sizeof(TReal*) * 1); + pm->stripes[0] = (TReal*)malloc(sizeof(TReal) * 6); + pm->stripes[0][0] = 16; pm->stripes[0][1] = 17; pm->stripes[0][2] = 18; pm->stripes[0][3] = 16; pm->stripes[0][4] = 17; pm->stripes[0][5] = 18; + } + pm->is_upper_triangle = true; +} + +partial_mat_t* make_test_pm(int case_id) { + partial_mat_t* pm = (partial_mat_t*)malloc(sizeof(partial_mat_t)); + + fill_test_pm(pm,case_id); + return pm; +} + +partial_dyn_mat_t* make_test_pdm(int case_id) { + partial_dyn_mat_t* pm = (partial_dyn_mat_t*)malloc(sizeof(partial_dyn_mat_t)); + fill_test_pm(pm,case_id); + pm->offsets = (uint64_t*)calloc(pm->stripe_stop-pm->stripe_start,sizeof(uint64_t)); + pm->filename = strdup("dummy"); + + return pm; +} + +mat_t* mat_three_rep() { + mat_t* res = (mat_t*)malloc(sizeof(mat_t)); + res->n_samples = 6; + res->cf_size = 15; + res->is_upper_triangle = true; + res->condensed_form = (double*)malloc(sizeof(double) * 15); + // using second half of third stripe. the last stripe when operating on even numbers of samples is normally redundant with the first half, + // but that was more annoying in to write up in the tests. + res->condensed_form[0] = 1; res->condensed_form[1] = 7; res->condensed_form[2] = 16; res->condensed_form[3] = 11; res->condensed_form[4] = 6; + res->condensed_form[5] = 2; res->condensed_form[6] = 8; res->condensed_form[7] = 17; res->condensed_form[8] = 12; + res->condensed_form[9] = 3; res->condensed_form[10] = 9; res->condensed_form[11] = 18; + res->condensed_form[12] = 4; res->condensed_form[13] = 10; + res->condensed_form[14] = 5; + res->sample_ids = (char**)malloc(sizeof(char*) * 6); + res->sample_ids[0] = (char*)malloc(sizeof(char) * 2); + res->sample_ids[0][0] = 'A'; res->sample_ids[0][1] = '\0'; + res->sample_ids[1] = (char*)malloc(sizeof(char) * 2); + res->sample_ids[1][0] = 'B'; res->sample_ids[1][1] = '\0'; + res->sample_ids[2] = (char*)malloc(sizeof(char) * 3); + res->sample_ids[2][0] = 'C'; res->sample_ids[2][1] = 'x'; res->sample_ids[2][2] = '\0'; + res->sample_ids[3] = (char*)malloc(sizeof(char) * 2); + res->sample_ids[3][0] = 'D'; res->sample_ids[3][1] = '\0'; + res->sample_ids[4] = (char*)malloc(sizeof(char) * 2); + res->sample_ids[4][0] = 'E'; res->sample_ids[4][1] = '\0'; + res->sample_ids[5] = (char*)malloc(sizeof(char) * 2); + res->sample_ids[5][0] = 'F'; res->sample_ids[5][1] = '\0'; + + return res; +} + +template +TMat* mat_full_three_rep() { + TMat* res = (TMat*)malloc(sizeof(TMat)); + res->n_samples = 6; + res->flags=0; + res->matrix = (TReal*)malloc(sizeof(TReal) * 36); + TReal * m=res->matrix ; + m[ 0] = 0; m[ 1] = 1; m[ 2] = 7; m[ 3] = 16; m[ 4] = 11; m[ 5] = 6; + m[ 6] = 1; m[ 7] = 0; m[ 8] = 2; m[ 9] = 8; m[10] = 17; m[11] = 12; + m[12] = 7; m[13] = 2; m[14] = 0; m[15] = 3; m[16] = 9; m[17] = 18; + m[18] = 16; m[19] = 8; m[20] = 3; m[21] = 0; m[22] = 4; m[23] = 10; + m[24] = 11; m[25] = 17; m[26] = 9; m[27] = 4; m[28] = 0; m[29] = 5; + m[30] = 6; m[31] = 12; m[32] = 18; m[33] = 10; m[34] = 5; m[35] = 0; + + res->sample_ids = (char**)malloc(sizeof(char*) * 6); + res->sample_ids[0] = (char*)malloc(sizeof(char) * 2); + res->sample_ids[0][0] = 'A'; res->sample_ids[0][1] = '\0'; + res->sample_ids[1] = (char*)malloc(sizeof(char) * 2); + res->sample_ids[1][0] = 'B'; res->sample_ids[1][1] = '\0'; + res->sample_ids[2] = (char*)malloc(sizeof(char) * 3); + res->sample_ids[2][0] = 'C'; res->sample_ids[2][1] = 'x'; res->sample_ids[2][2] = '\0'; + res->sample_ids[3] = (char*)malloc(sizeof(char) * 2); + res->sample_ids[3][0] = 'D'; res->sample_ids[3][1] = '\0'; + res->sample_ids[4] = (char*)malloc(sizeof(char) * 2); + res->sample_ids[4][0] = 'E'; res->sample_ids[4][1] = '\0'; + res->sample_ids[5] = (char*)malloc(sizeof(char) * 2); + res->sample_ids[5][0] = 'F'; res->sample_ids[5][1] = '\0'; + + return res; +} + +void test_read_write_partial_mat() { + SUITE_START("test read/write partial_mat_t"); + + partial_mat_t* pm = make_test_pm(0); + + io_status err = write_partial("/tmp/ssu_io.dat", pm); + ASSERT(err == write_okay); + + { + partial_mat_t *obs = NULL; + err = read_partial("/tmp/ssu_io.dat", &obs); + + ASSERT(err == read_okay); + ASSERT(obs->n_samples == 6); + ASSERT(obs->stripe_start == 0); + ASSERT(obs->stripe_stop == 3); + ASSERT(obs->stripe_total == 3); + ASSERT(strcmp(obs->sample_ids[0], "A") == 0); + ASSERT(strcmp(obs->sample_ids[1], "B") == 0); + ASSERT(strcmp(obs->sample_ids[2], "Cx") == 0); + ASSERT(strcmp(obs->sample_ids[3], "D") == 0); + ASSERT(strcmp(obs->sample_ids[4], "E") == 0); + ASSERT(strcmp(obs->sample_ids[5], "F") == 0); + + for(int i = 0; i < 3; i++) { + for(int j = 0; j < 6; j++) { + ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); + } + } + + destroy_partial_mat(&obs); + } + + { + partial_dyn_mat_t *obs = NULL; + err = read_partial_header("/tmp/ssu_io.dat", &obs); + + ASSERT(err == read_okay); + ASSERT(obs->n_samples == 6); + ASSERT(obs->stripe_start == 0); + ASSERT(obs->stripe_stop == 3); + ASSERT(obs->stripe_total == 3); + ASSERT(strcmp(obs->sample_ids[0], "A") == 0); + ASSERT(strcmp(obs->sample_ids[1], "B") == 0); + ASSERT(strcmp(obs->sample_ids[2], "Cx") == 0); + ASSERT(strcmp(obs->sample_ids[3], "D") == 0); + ASSERT(strcmp(obs->sample_ids[4], "E") == 0); + ASSERT(strcmp(obs->sample_ids[5], "F") == 0); + + for(int i = 0; i < 3; i++) { + ASSERT(obs->stripes[i]==NULL); + } + + err = read_partial_one_stripe(obs,1); + ASSERT(err == read_okay); + + ASSERT(obs->stripes[0]==NULL); + ASSERT(obs->stripes[1]!=NULL); + ASSERT(obs->stripes[2]==NULL); + + { + const int i = 1; + for(int j = 0; j < 6; j++) { + ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); + } + } + + err = read_partial_one_stripe(obs,0); + ASSERT(err == read_okay); + + ASSERT(obs->stripes[0]!=NULL); + ASSERT(obs->stripes[1]!=NULL); + ASSERT(obs->stripes[2]==NULL); + + { + const int i = 0; + for(int j = 0; j < 6; j++) { + ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); + } + } + + err = read_partial_one_stripe(obs,2); + ASSERT(err == read_okay); + + + for(int i = 0; i < 3; i++) { + ASSERT(obs->stripes[i]!=NULL); + for(int j = 0; j < 6; j++) { + ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); + } + } + + destroy_partial_dyn_mat(&obs); + } + + unlink("/tmp/ssu_io.dat"); + + SUITE_END(); +} + +void test_merge_partial_mat() { + SUITE_START("test merge partial_mat_t"); + + // the easy test + partial_mat_t* pm1 = make_test_pm(1); + partial_mat_t* pm2 = make_test_pm(2); + + mat_t* exp = mat_three_rep(); + + partial_mat_t* pms[2]; + pms[0] = pm1; + pms[1] = pm2; + + mat_t* obs = NULL; + merge_status err = merge_partial(pms, 2, 1, &obs); + ASSERT(err == merge_okay); + ASSERT(obs->cf_size == exp->cf_size); + ASSERT(obs->n_samples == exp->n_samples); + ASSERT(obs->is_upper_triangle == exp->is_upper_triangle); + for(unsigned int i = 0; i < obs->cf_size; i++) { + ASSERT(obs->condensed_form[i] == exp->condensed_form[i]); + } + for(unsigned int i = 0; i < obs->n_samples; i++) + ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); + + // out of order test + + pms[0] = pm2; + pms[1] = pm1; + + obs = NULL; + err = merge_partial(pms, 2, 1, &obs); + ASSERT(err == merge_okay); + ASSERT(obs->cf_size == exp->cf_size); + ASSERT(obs->n_samples == exp->n_samples); + ASSERT(obs->is_upper_triangle == exp->is_upper_triangle); + for(unsigned int i = 0; i < obs->cf_size; i++) { + ASSERT(obs->condensed_form[i] == exp->condensed_form[i]); + } + for(unsigned int i = 0; i < obs->n_samples; i++) + ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); + + // error checking + pm1->stripe_start = 0; + pm1->stripe_stop = 3; + pm1->stripe_total = 9; + pm1->is_upper_triangle = true; + + pm2->stripe_start = 3; + pm2->stripe_stop = 5; + pm2->stripe_total = 9; + pm2->is_upper_triangle = true; + + partial_mat_t* pm3 = make_test_pm(0); + pm3->stripe_start = 6; + pm3->stripe_stop = 9; + pm3->stripe_total = 9; + + partial_mat_t* pms_err[3]; + + pms_err[2] = pm1; + pms_err[0] = pm2; + pms_err[1] = pm3; + + err = merge_partial(pms_err, 3, 1, &obs); + ASSERT(err == incomplete_stripe_set); + + pm2->stripe_start = 2; + pm2->stripe_stop = 6; + err = merge_partial(pms_err, 3, 1, &obs); + ASSERT(err == stripes_overlap); + + pm2->stripe_start = 3; + pm2->sample_ids[2][0] = 'X'; + err = merge_partial(pms_err, 3, 1, &obs); + ASSERT(err == sample_id_consistency); + + pm2->sample_ids[2][0] = 'C'; + pm3->n_samples = 2; + err = merge_partial(pms_err, 3, 1, &obs); + ASSERT(err == partials_mismatch); + + pm3->n_samples = 6; + pm3->stripe_total = 12; + err = merge_partial(pms_err, 3, 1, &obs); + ASSERT(err == partials_mismatch); + + pm3->is_upper_triangle = false; + pm3->stripe_total = 9; + err = merge_partial(pms_err, 3, 1, &obs); + ASSERT(err == square_mismatch); + + SUITE_END(); +} + +void test_merge_partial_dyn_mat() { + SUITE_START("test merge partial_dyn_mat_t"); + + // the easy test + partial_dyn_mat_t* pm1 = make_test_pdm(1); + partial_dyn_mat_t* pm2 = make_test_pdm(2); + + mat_full_fp64_t* exp = mat_full_three_rep(); + + partial_dyn_mat_t* pms[2]; + pms[0] = pm1; + pms[1] = pm2; + + mat_full_fp64_t* obs = NULL; + merge_status err = merge_partial_to_matrix(pms, 2, &obs); + ASSERT(err == merge_okay); + ASSERT(obs->n_samples == exp->n_samples); + for(unsigned int i = 0; i < (obs->n_samples*obs->n_samples); i++) { + ASSERT(obs->matrix[i] == exp->matrix[i]); + } + for(unsigned int i = 0; i < obs->n_samples; i++) { + ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); + } + // out of order test + + // recreate deallocated stripes + ASSERT(pm1->stripes[0]==NULL); + ASSERT(pm1->stripes[1]==NULL); + ASSERT(pm2->stripes[0]==NULL); + destroy_partial_dyn_mat(&pm1); + destroy_partial_dyn_mat(&pm2); + + + pm1 = make_test_pdm(1); + pm2 = make_test_pdm(2); + + pms[0] = pm2; + pms[1] = pm1; + + mat_full_fp32_t* exp2 = mat_full_three_rep(); + mat_full_fp32_t *obs2 = NULL; + err = merge_partial_to_matrix_fp32(pms, 2, &obs2); + ASSERT(err == merge_okay); + ASSERT(obs2->n_samples == exp2->n_samples); + for(unsigned int i = 0; i < (obs2->n_samples*obs2->n_samples); i++) { + ASSERT(obs2->matrix[i] == exp2->matrix[i]); + } + for(unsigned int i = 0; i < obs2->n_samples; i++) + ASSERT(strcmp(obs2->sample_ids[i], exp2->sample_ids[i]) == 0); + + + ASSERT(pm2->stripes[0]==NULL); + ASSERT(pm1->stripes[0]==NULL); + ASSERT(pm1->stripes[1]==NULL); + destroy_partial_dyn_mat(&pm1); + destroy_partial_dyn_mat(&pm2); + + + pm1 = make_test_pdm(1); + pm2 = make_test_pdm(2); + + + // error checking + pm1->stripe_start = 0; + pm1->stripe_stop = 3; + pm1->stripe_total = 9; + pm1->is_upper_triangle = true; + + pm2->stripe_start = 3; + pm2->stripe_stop = 5; + pm2->stripe_total = 9; + pm2->is_upper_triangle = true; + + partial_dyn_mat_t* pm3 = make_test_pdm(0); + pm3->stripe_start = 6; + pm3->stripe_stop = 9; + pm3->stripe_total = 9; + + partial_dyn_mat_t* pms_err[3]; + + pms_err[2] = pm1; + pms_err[0] = pm2; + pms_err[1] = pm3; + + err = merge_partial_to_matrix(pms_err, 3, &obs); + ASSERT(err == incomplete_stripe_set); + + pm2->stripe_start = 2; + pm2->stripe_stop = 6; + err = merge_partial_to_matrix(pms_err, 3, &obs); + ASSERT(err == stripes_overlap); + + pm2->stripe_start = 3; + pm2->sample_ids[2][0] = 'X'; + err = merge_partial_to_matrix(pms_err, 3, &obs); + ASSERT(err == sample_id_consistency); + + pm2->sample_ids[2][0] = 'C'; + pm3->n_samples = 2; + err = merge_partial_to_matrix(pms_err, 3, &obs); + ASSERT(err == partials_mismatch); + + pm3->n_samples = 6; + pm3->stripe_total = 12; + err = merge_partial_to_matrix(pms_err, 3, &obs); + ASSERT(err == partials_mismatch); + + /* + * Disable for now... not dealing properly with is_upper_triangle == false + + pm3->is_upper_triangle = false; + pm3->stripe_total = 9; + err = merge_partial_to_matrix(pms_err, 3, &obs); + ASSERT(err == square_mismatch); + */ + + destroy_mat_full_fp64(&obs); + // note, we cannot cleanly destroy the partial_dyn_mat_t structures that have been hacked by hand + + SUITE_END(); +} + +void test_merge_partial_io() { + SUITE_START("test merge partial_io"); + + // the easy test + partial_mat_t* s1 = make_test_pm(1); + partial_mat_t* s2 = make_test_pm(2); + + io_status ierr; + + ierr = write_partial("/tmp/ssu_io_1.dat", s1); + ASSERT(ierr == write_okay); + + ierr = write_partial("/tmp/ssu_io_2.dat", s2); + ASSERT(ierr == write_okay); + + partial_dyn_mat_t* pm1 = NULL; + partial_dyn_mat_t* pm2 = NULL; + + ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1); + ASSERT(ierr == read_okay); + + ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2); + ASSERT(ierr == read_okay); + + mat_full_fp64_t* exp = mat_full_three_rep(); + + partial_dyn_mat_t* pms[2]; + pms[0] = pm1; + pms[1] = pm2; + + mat_full_fp64_t* obs = NULL; + merge_status err = merge_partial_to_matrix(pms, 2, &obs); + ASSERT(err == merge_okay); + ASSERT(obs->n_samples == exp->n_samples); + for(unsigned int i = 0; i < (obs->n_samples*obs->n_samples); i++) { + ASSERT(obs->matrix[i] == exp->matrix[i]); + } + for(unsigned int i = 0; i < obs->n_samples; i++) { + ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); + } + ASSERT(pm1->stripes[0]==NULL); + ASSERT(pm1->stripes[1]==NULL); + ASSERT(pm2->stripes[0]==NULL); + + destroy_mat_full_fp64(&obs); + destroy_partial_dyn_mat(&pm1); + destroy_partial_dyn_mat(&pm2); + + // out of order test + partial_dyn_mat_t* pm1b = NULL; + partial_dyn_mat_t* pm2b = NULL; + + ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1b); + ASSERT(ierr == read_okay); + + ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2b); + ASSERT(ierr == read_okay); + + pms[0] = pm2b; + pms[1] = pm1b; + + mat_full_fp32_t* exp2 = mat_full_three_rep(); + mat_full_fp32_t *obs2 = NULL; + err = merge_partial_to_matrix_fp32(pms, 2, &obs2); + ASSERT(err == merge_okay); + ASSERT(obs2->n_samples == exp2->n_samples); + for(unsigned int i = 0; i < (obs2->n_samples*obs2->n_samples); i++) { + ASSERT(obs2->matrix[i] == exp2->matrix[i]); + } + for(unsigned int i = 0; i < obs2->n_samples; i++) + ASSERT(strcmp(obs2->sample_ids[i], exp2->sample_ids[i]) == 0); + + + ASSERT(pm2b->stripes[0]==NULL); + ASSERT(pm1b->stripes[0]==NULL); + ASSERT(pm1b->stripes[1]==NULL); + + destroy_mat_full_fp32(&obs2); + destroy_partial_dyn_mat(&pm1b); + destroy_partial_dyn_mat(&pm2b); + + unlink("/tmp/ssu_io_1.dat"); + unlink("/tmp/ssu_io_2.dat"); + + SUITE_END(); +} + +void test_merge_partial_mmap() { + SUITE_START("test merge partial_mmap"); + + // the easy test + partial_mat_t* s1 = make_test_pm(1); + partial_mat_t* s2 = make_test_pm(2); + + io_status ierr; + + ierr = write_partial("/tmp/ssu_io_1.dat", s1); + ASSERT(ierr == write_okay); + + ierr = write_partial("/tmp/ssu_io_2.dat", s2); + ASSERT(ierr == write_okay); + + partial_dyn_mat_t* pm1 = NULL; + partial_dyn_mat_t* pm2 = NULL; + + ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1); + ASSERT(ierr == read_okay); + + ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2); + ASSERT(ierr == read_okay); + + mat_full_fp64_t* exp = mat_full_three_rep(); + + partial_dyn_mat_t* pms[2]; + pms[0] = pm1; + pms[1] = pm2; + + mat_full_fp32_t* obs = NULL; + merge_status err = merge_partial_to_mmap_matrix_fp32(pms, 2, "/tmp", &obs); + ASSERT(err == merge_okay); + ASSERT(obs->n_samples == exp->n_samples); + ASSERT(obs->flags != 0); + for(unsigned int i = 0; i < (obs->n_samples*obs->n_samples); i++) { + ASSERT(obs->matrix[i] == exp->matrix[i]); + } + for(unsigned int i = 0; i < obs->n_samples; i++) { + ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); + } + ASSERT(pm1->stripes[0]==NULL); + ASSERT(pm1->stripes[1]==NULL); + ASSERT(pm2->stripes[0]==NULL); + + destroy_mat_full_fp32(&obs); + destroy_partial_dyn_mat(&pm1); + destroy_partial_dyn_mat(&pm2); + + // test failure due to FS problems + + ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1); + ASSERT(ierr == read_okay); + + ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2); + ASSERT(ierr == read_okay); + + pms[0] = pm1; + pms[1] = pm2; + + + err = merge_partial_to_mmap_matrix_fp32(pms, 2, "/santa/goes/skiing", &obs); + ASSERT(err != merge_okay); + destroy_partial_dyn_mat(&pm1); + destroy_partial_dyn_mat(&pm2); + + destroy_partial_mat(&s1); + destroy_partial_mat(&s2); + + unlink("/tmp/ssu_io_1.dat"); + unlink("/tmp/ssu_io_2.dat"); + + + SUITE_END(); +} + +void test_to_file_one(const char *method) { + + static const char h5name[]="/tmp/ssu_t1.h5"; + struct stat sbuf; + + ComputeStatus urc; + int frc; + + // ensure file does not already exist + frc=stat(h5name,&sbuf); + if (frc == 0) { + unlink(h5name); + frc=stat(h5name,&sbuf); + } + ASSERT(frc != 0); + + urc=unifrac_to_file("test.biom","test.tre",h5name,method,false,1.0,false,1,"hdf5",0,NULL); + ASSERT(urc == okay); + + // first, we check it does exist + frc=stat(h5name,&sbuf); + ASSERT(frc == 0); + + { + try { + H5::H5File file(h5name, H5F_ACC_RDONLY); + H5::DataSet mds(file.openDataSet("matrix")); + H5::DataSpace dataspace(mds.getSpace()); + + ASSERT(dataspace.isSimple() == true); + ASSERT(dataspace.getSimpleExtentNdims() == 2); + + hsize_t dims[2]; + dataspace.getSimpleExtentDims(dims, NULL); + ASSERT(dims[0] == 6); + ASSERT(dims[1] == 6); + } catch(...) { + int rc=1; + ASSERT(rc == 0); // if we get here is always an error, just to get a nice message + } + } + + unlink(h5name); + +} + +void test_to_file() { + SUITE_START("test unifrac_to_file"); + + test_to_file_one("unweighted"); + test_to_file_one("unweighted_fp32"); + test_to_file_one("weighted_normalized"); + test_to_file_one("weighted_normalized_fp32"); + test_to_file_one("weighted_unnormalized"); + test_to_file_one("weighted_unnormalized_fp32"); + test_to_file_one("generalized"); + test_to_file_one("generalized_fp32"); + + SUITE_END(); +} + +int main(int argc, char** argv) { + /* one_off and partial are executed as integration tests */ + + //test_write_mat(); + //test_read_mat(); + test_read_write_partial_mat(); + test_merge_partial_mat(); + test_merge_partial_dyn_mat(); + test_merge_partial_io(); + test_merge_partial_mmap(); + test_to_file(); + + printf("\n"); + printf(" %i / %i suites failed\n", suites_failed, suites_run); + printf(" %i / %i suites empty\n", suites_empty, suites_run); + printf(" %i / %i tests failed\n", tests_failed, tests_run); + + printf("\n THE END.\n"); + + return tests_failed ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/R/unifrac_cpp/test_ska.cpp b/R/unifrac_cpp/test_ska.cpp new file mode 100644 index 000000000..949c9d669 --- /dev/null +++ b/R/unifrac_cpp/test_ska.cpp @@ -0,0 +1,516 @@ +#include +#include "skbio_alt.hpp" +#include +#include +#include + +#include "api.hpp" + +/* + * test harness adapted from + * https://github.com/noporpoise/BitArray/blob/master/dev/bit_array_test.c + */ +const char *suite_name; +char suite_pass; +int suites_run = 0, suites_failed = 0, suites_empty = 0; +int tests_in_suite = 0, tests_run = 0, tests_failed = 0; + +#define QUOTE(str) #str +#define ASSERT(x) {tests_run++; tests_in_suite++; if(!(x)) \ + { fprintf(stderr, "failed assert [%s:%i] %s\n", __FILE__, __LINE__, QUOTE(x)); \ + suite_pass = 0; tests_failed++; }} + +void SUITE_START(const char *name) { + suite_pass = 1; + suite_name = name; + suites_run++; + tests_in_suite = 0; +} + +void SUITE_END() { + printf("Testing %s ", suite_name); + size_t suite_i; + for(suite_i = strlen(suite_name); suite_i < 80-8-5; suite_i++) printf("."); + printf("%s\n", suite_pass ? " pass" : " fail"); + if(!suite_pass) suites_failed++; + if(!tests_in_suite) suites_empty++; +} +/* + * End adapted code + */ + + +void test_center_mat() { + SUITE_START("test center mat"); + + // unweighted unifrac of test.biom + double matrix[] = { + 0.0000000000, 0.2000000000, 0.5714285714, 0.6000000000, 0.5000000000, 0.2000000000, + 0.2000000000, 0.0000000000, 0.4285714286 ,0.6666666667, 0.6000000000, 0.3333333333, + 0.5714285714, 0.4285714286, 0.0000000000, 0.7142857143, 0.8571428571, 0.4285714286, + 0.6000000000, 0.6666666667, 0.7142857143, 0.0000000000, 0.3333333333, 0.4000000000, + 0.5000000000, 0.6000000000, 0.8571428571, 0.3333333333, 0.0000000000, 0.6000000000, + 0.2000000000, 0.3333333333, 0.4285714286, 0.4000000000, 0.6000000000, 0.0000000000}; + + const uint32_t n_samples = 6; + + double exp[] = { 0.05343726, 0.04366213, -0.0329743 , -0.07912698, -0.00495654, 0.01995843, + 0.04366213, 0.073887 , 0.04867914, -0.11112434, -0.04973167,-0.00537226, + -0.0329743 , 0.04867914, 0.20714475, -0.07737528, -0.17044974, 0.02497543, + -0.07912698, -0.11112434, -0.07737528, 0.14830877, 0.11192366, 0.00739418, + -0.00495654, -0.04973167, -0.17044974, 0.11192366, 0.18664966,-0.07343537, + 0.01995843, -0.00537226, 0.02497543, 0.00739418, -0.07343537, 0.02647959 }; + + { + double *centered = (double *) malloc(6*6*sizeof(double)); + + su::mat_to_centered(matrix, n_samples, centered); + + for(int i = 0; i < (6*6); i++) { + //printf("%i %f %f\n",i,float(centered[i]),float(exp[i])); + ASSERT(fabs(centered[i] - exp[i]) < 0.000001); + } + + free(centered); + } + + float *matrix_fp32 = (float *) malloc(6*6*sizeof(float)); + for(int i = 0; i < (6*6); i++) matrix_fp32[i] = matrix[i]; + + + { + float *centered_fp32 = (float *) malloc(6*6*sizeof(float)); + + su::mat_to_centered(matrix_fp32, n_samples, centered_fp32); + + for(int i = 0; i < (6*6); i++) { + //printf("%i %f %f\n",i,float(centered_fp32[i]),float(exp[i])); + ASSERT(fabs(centered_fp32[i] - exp[i]) < 0.000001); + } + + free(centered_fp32); + } + + free(matrix_fp32); + + + SUITE_END(); +} + +void test_pcoa() { + SUITE_START("test pcoa"); + + // unweighted unifrac of crawford.biom + double matrix[] = { + 0. , 0.71836067 , 0.71317361 , 0.69746044 , 0.62587207 , 0.72826674 + , 0.72065895 , 0.72640581 , 0.73606053, + 0.71836067 , 0. , 0.70302967 , 0.73407301 , 0.6548042 , 0.71547381 + , 0.78397813 , 0.72318399 , 0.76138933, + 0.71317361 , 0.70302967 , 0. , 0.61041275 , 0.62331299 , 0.71848305 + , 0.70416337 , 0.75258475 , 0.79249029, + 0.69746044 , 0.73407301 , 0.61041275 , 0. , 0.6439278 , 0.70052733 + , 0.69832716 , 0.77818938 , 0.72959894, + 0.62587207 , 0.6548042 , 0.62331299 , 0.6439278 , 0. , 0.75782689 + , 0.71005144 , 0.75065046 , 0.78944369, + 0.72826674 , 0.71547381 , 0.71848305 , 0.70052733 , 0.75782689 , 0. + , 0.63593642 , 0.71283615 , 0.58314638, + 0.72065895 , 0.78397813 , 0.70416337 , 0.69832716 , 0.71005144 , 0.63593642 + , 0. , 0.69200762 , 0.68972056, + 0.72640581 , 0.72318399 , 0.75258475 , 0.77818938 , 0.75065046 , 0.71283615 + , 0.69200762 , 0. , 0.71514083, + 0.73606053 , 0.76138933 , 0.79249029 , 0.72959894 , 0.78944369 , 0.58314638 + , 0.68972056 , 0.71514083 , 0. }; + + + const uint32_t n_samples = 9; + + // Test centering + + double exp2[] = { + 0.22225336 , -0.025481 , -0.03491711 , -0.02614685 , 0.01899659 , -0.05103589 + , -0.03971812 , -0.02699392 , -0.03695706, + -0.025481 , 0.24282669 , -0.01744751 , -0.04206624 , 0.0107569 , -0.03151438 + , -0.07706765 , -0.0143721 , -0.0456347, + -0.03491711 , -0.01744751 , 0.21652901 , 0.02791465 , 0.0177328 , -0.04682078 + , -0.03082866 , -0.04921529 , -0.08294711, + -0.02614685 , -0.04206624 , 0.02791465 , 0.21190401 , 0.00235833 , -0.03639361 + , -0.02904855 , -0.07112525 , -0.03739649, + 0.01899659 , 0.0107569 , 0.0177328 , 0.00235833 , 0.20745566 , -0.08039931 + , -0.03952883 , -0.05229812 , -0.08507403, + -0.05103589 , -0.03151438 , -0.04682078 , -0.03639361 , -0.08039931 , 0.20604732 + , 0.00964595 , -0.02533192 , 0.05580262, + -0.03971812 , -0.07706765 , -0.03082866 , -0.02904855 , -0.03952883 , 0.00964595 + , 0.21765972 , -0.00489531 , -0.00621856, + -0.02699392 , -0.0143721 , -0.04921529 , -0.07112525 , -0.05229812 , -0.02533192 + , -0.00489531 , 0.2514242 , -0.00719229, + -0.03695706 , -0.0456347 , -0.08294711 , -0.03739649 , -0.08507403 , 0.05580262 + , -0.00621856 , -0.00719229 , 0.24561762 }; + + double *centered = (double *) malloc(9*9*sizeof(double)); + + su::mat_to_centered(matrix, n_samples, centered); + + for(int i = 0; i < (9*9); i++) { + //printf("%i %f %f\n",i,float(centered[i]),float(exp[i])); + ASSERT(fabs(centered[i] - exp2[i]) < 0.000001); + } + + // Test eigens + + double exp3a[] = {0.45752162, 0.3260088 , 0.2791141 , 0.26296948, 0.20924533}; + double exp3b[] = { + -0.17316152, 0.17579996, 0.23301609, -0.74519625, -0.05194624, + -0.19959264, 0.53235665, -0.53370018, 0.2173474 , 0.26736004, + -0.35794942, -0.27956624, 0.01114096, 0.40488848, -0.13121464, + -0.2296467 , -0.47494333, -0.12571292, 0.02313551, -0.46916459, + -0.44501584, 0.05597451, 0.07717711, -0.15881922, 0.24594442, + 0.40335552, -0.16290597, -0.30327343, 0.03778646, 0.21664806, + 0.23769142, -0.29034629, 0.46813757, 0.13858945, 0.58624834, + 0.2407584 , 0.51300752, 0.48211607, 0.34422672, -0.42416046, + 0.52356078, -0.0693768 , -0.30890127, -0.26195855, -0.23971493}; + + { + double *eigenvalues; + double *eigenvectors; + su::find_eigens_fast(n_samples, 5, centered, eigenvalues, eigenvectors); + + for(int i = 0; i < 5; i++) { + //printf("%i %f %f\n",i,float(eigenvalues[i]),float(exp3a[i])); + ASSERT(fabs(eigenvalues[i] - exp3a[i]) < 0.000001); + } + + // signs may flip, that's normal + for(int i = 0; i < (5*9); i++) { + //printf("%i %f %f %f\n",i,float(eigenvectors[i]),float(exp3b[i]),float(fabs(eigenvectors[i]) - fabs(exp3b[i]))); + ASSERT( fabs(fabs(eigenvectors[i]) - fabs(exp3b[i])) < 0.000001); + } + + free(eigenvectors); + free(eigenvalues); + } + free(centered); + + + // Test PCoA (incudes the above calls + + double *exp4a = exp3a; // same eigenvals; + double exp4b[] = { + -0.11712705, 0.10037682, -0.12310531, -0.38214073, -0.02376195, + -0.13500515, 0.30396064, 0.28196047, 0.11145694, 0.12229942, + -0.24211822, -0.15962444, -0.00588591, 0.20762904, -0.06002196, + -0.15533382, -0.27117925, 0.06641571, 0.01186402, -0.21461156, + -0.30101024, 0.03195987, -0.04077363, -0.08144337, 0.1125032 , + 0.27283106, -0.09301471, 0.16022314, 0.0193771 , 0.09910206, + 0.16077529, -0.16577955, -0.24732293, 0.07106943, 0.26816958, + 0.16284981, 0.29291283, -0.25470794, 0.17652136, -0.19402517, + 0.35413832, -0.0396122 , 0.1631964 , -0.13433378, -0.10965362}; + double exp4c[] = {0.22630343, 0.16125338, 0.13805791, 0.13007231, 0.10349879}; + + { + double *eigenvalues; + double *samples; + double *proportion_explained; + + su::pcoa(matrix, n_samples, 5, eigenvalues, samples, proportion_explained); + + for(int i = 0; i < 5; i++) { + //printf("%i %f %f\n",i,float(eigenvalues[i]),float(exp4a[i])); + ASSERT(fabs(eigenvalues[i] - exp4a[i]) < 0.000001); + } + + // signs may flip, that's normal + for(int i = 0; i < (5*9); i++) { + //printf("%i %f %f %f\n",i,float(samples[i]),float(exp4b[i]),float(fabs(samples[i]) - fabs(exp4b[i]))); + ASSERT( fabs(fabs(samples[i]) - fabs(exp4b[i])) < 0.000001); + } + + for(int i = 0; i < 5; i++) { + //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp4c[i])); + ASSERT(fabs(proportion_explained[i] - exp4c[i]) < 0.000001); + } + + free(eigenvalues); + free(samples); + free(proportion_explained); + } + + // Test PCoA mixed mode + { + float *eigenvalues_fp32; + float *samples_fp32; + float *proportion_explained_fp32; + + su::pcoa(matrix, n_samples, 5, eigenvalues_fp32, samples_fp32, proportion_explained_fp32); + + for(int i = 0; i < 5; i++) { + //printf("%i %f %f\n",i,float(eigenvalues_fp32[i]),float(exp4a[i])); + ASSERT(fabs(eigenvalues_fp32[i] - exp4a[i]) < 0.000001); + } + + // signs may flip, that's normal + for(int i = 0; i < (5*9); i++) { + //printf("%i %f %f %f\n",i,float(samples_fp32[i]),float(exp4b[i]),float(fabs(samples_fp32[i]) - fabs(exp4b[i]))); + ASSERT( fabs(fabs(samples_fp32[i]) - fabs(exp4b[i])) < 0.000001); + } + + for(int i = 0; i < 5; i++) { + //printf("%i %f %f\n",i,float(proportion_explained_fp32[i]),float(exp4c[i])); + ASSERT(fabs(proportion_explained_fp32[i] - exp4c[i]) < 0.000001); + } + + free(eigenvalues_fp32); + free(samples_fp32); + free(proportion_explained_fp32); + } + + // test in-place + { + double *eigenvalues; + double *samples; + double *proportion_explained; + + su::pcoa_inplace(matrix, n_samples, 5, eigenvalues, samples, proportion_explained); + // Note: matrix content has been destroyed + + for(int i = 0; i < 5; i++) { + //printf("%i %f %f\n",i,float(eigenvalues[i]),float(exp4a[i])); + ASSERT(fabs(eigenvalues[i] - exp4a[i]) < 0.000001); + } + + // signs may flip, that's normal + for(int i = 0; i < (5*9); i++) { + //printf("%i %f %f %f\n",i,float(samples[i]),float(exp4b[i]),float(fabs(samples[i]) - fabs(exp4b[i]))); + ASSERT( fabs(fabs(samples[i]) - fabs(exp4b[i])) < 0.000001); + } + + for(int i = 0; i < 5; i++) { + //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp4c[i])); + ASSERT(fabs(proportion_explained[i] - exp4c[i]) < 0.000001); + } + + free(eigenvalues); + free(samples); + free(proportion_explained); + } + + SUITE_END(); +} + +void test_pcoa_big() { + SUITE_START("test pcoa big"); + + //too big to inline, use support file + FILE *fptr = fopen("test_ska_pcoa_big.dat", "r"); + ASSERT(fptr != NULL) + + unsigned int n_samples = 0; + fscanf(fptr,"# unifrac %u\n",&n_samples); + ASSERT(n_samples == 57); + + + // first 57 rows/cols of unweighted unifrac of EMP + double matrix[57*57]; + for (unsigned int i=0; i<(57*57); i++) + fscanf(fptr,"%lf\n",&(matrix[i])); + + unsigned int n_dims = 0; + fscanf(fptr,"# pcoa %u\n",&n_dims); + ASSERT(n_dims == 7); + + double exp1[7]; + for (unsigned int i=0; i<(7); i++) + fscanf(fptr,"%lf\n",&(exp1[i])); + + double exp2[7*57]; + for (unsigned int i=0; i<(7*57); i++) + fscanf(fptr,"%lf\n",&(exp2[i])); + + double exp3[7]; + for (unsigned int i=0; i<(7); i++) + fscanf(fptr,"%lf\n",&(exp3[i])); + + fclose(fptr); + + { + double *eigenvalues; + double *samples; + double *proportion_explained; + + su::pcoa(matrix, n_samples, n_dims, eigenvalues, samples, proportion_explained); + + // last three eignes are very close to each other and could come back in reverse order + + for(unsigned int i = 0; i < n_dims ; i++) { + //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); + const double max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo + ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); + } + + // signs may flip, that's normal + for(unsigned int i = 0; i < (n_samples*n_dims); i++) { + if ((i%n_dims)<4) { + //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); + ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.05) + } else { + // any of the 3 will do + unsigned int ibase = (i/n_dims)*n_dims; + //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); + ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); + } + } + + for(unsigned int i = 0; i < n_dims; i++) { + //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); + const double max_err = (i<4) ? 0.001 : 0.01; + ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); + } + + free(proportion_explained); + free(samples); + free(eigenvalues); + } + + { + float *eigenvalues; + float *samples; + float *proportion_explained; + + su::pcoa(matrix, n_samples, n_dims, eigenvalues, samples, proportion_explained); + + // last three eignes are very close to each other and could come back in reverse order + + for(unsigned int i = 0; i < n_dims ; i++) { + //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); + const float max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo + ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); + } + + // signs may flip, that's normal + for(unsigned int i = 0; i < (n_samples*n_dims); i++) { + if ((i%n_dims)<4) { + //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); + ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.1) + } else { + // any of the 3 will do + unsigned int ibase = (i/n_dims)*n_dims; + //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); + ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); + } + } + + for(unsigned int i = 0; i < n_dims; i++) { + //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); + const float max_err = (i<4) ? 0.001 : 0.01; + ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); + } + + free(proportion_explained); + free(samples); + free(eigenvalues); + } + + { + float matrix_fp32[57*57]; + for (unsigned int i=0; i<(57*57); i++) + matrix_fp32[i] = matrix[i]; + + { + float *eigenvalues; + float *samples; + float *proportion_explained; + + su::pcoa(matrix_fp32, n_samples, n_dims, eigenvalues, samples, proportion_explained); + + // last three eignes are very close to each other and could come back in reverse order + + for(unsigned int i = 0; i < n_dims ; i++) { + //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); + const float max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo + ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); + } + + // signs may flip, that's normal + for(unsigned int i = 0; i < (n_samples*n_dims); i++) { + if ((i%n_dims)<4) { + //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); + ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.1) + } else { + // any of the 3 will do + unsigned int ibase = (i/n_dims)*n_dims; + //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); + ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); + } + } + + for(unsigned int i = 0; i < n_dims; i++) { + //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); + const float max_err = (i<4) ? 0.001 : 0.01; + ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); + } + + free(proportion_explained); + free(samples); + free(eigenvalues); + } + + { + float *eigenvalues; + float *samples; + float *proportion_explained; + + su::pcoa_inplace(matrix_fp32, n_samples, n_dims, eigenvalues, samples, proportion_explained); + // Note: content of matrix_fp32 has been destroyed + + // last three eignes are very close to each other and could come back in reverse order + + for(unsigned int i = 0; i < n_dims ; i++) { + //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); + const float max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo + ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); + } + + // signs may flip, that's normal + for(unsigned int i = 0; i < (n_samples*n_dims); i++) { + if ((i%n_dims)<4) { + //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); + ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.1) + } else { + // any of the 3 will do + unsigned int ibase = (i/n_dims)*n_dims; + //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); + ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); + } + } + + for(unsigned int i = 0; i < n_dims; i++) { + //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); + const float max_err = (i<4) ? 0.001 : 0.01; + ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); + } + + free(proportion_explained); + free(samples); + free(eigenvalues); + } + + } + + + + SUITE_END(); +} + +int main(int argc, char** argv) { + test_center_mat(); + test_pcoa(); + test_pcoa_big(); + + printf("\n"); + printf(" %i / %i suites failed\n", suites_failed, suites_run); + printf(" %i / %i suites empty\n", suites_empty, suites_run); + printf(" %i / %i tests failed\n", tests_failed, tests_run); + + printf("\n THE END.\n"); + + return tests_failed ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/R/unifrac_cpp/test_su.cpp b/R/unifrac_cpp/test_su.cpp new file mode 100644 index 000000000..9ef3d3341 --- /dev/null +++ b/R/unifrac_cpp/test_su.cpp @@ -0,0 +1,1872 @@ +#include +#include "api.hpp" +#include "tree.hpp" +#include "biom.hpp" +#include "unifrac.hpp" +#include "unifrac_internal.hpp" +#include +#include +#include + +/* + * test harness adapted from + * https://github.com/noporpoise/BitArray/blob/master/dev/bit_array_test.c + */ +const char *suite_name; +char suite_pass; +int suites_run = 0, suites_failed = 0, suites_empty = 0; +int tests_in_suite = 0, tests_run = 0, tests_failed = 0; + +#define QUOTE(str) #str +#define ASSERT(x) {tests_run++; tests_in_suite++; if(!(x)) \ + { fprintf(stderr, "failed assert [%s:%i] %s\n", __FILE__, __LINE__, QUOTE(x)); \ + suite_pass = 0; tests_failed++; }} + +void SUITE_START(const char *name) { + suite_pass = 1; + suite_name = name; + suites_run++; + tests_in_suite = 0; +} + +void SUITE_END() { + printf("Testing %s ", suite_name); + size_t suite_i; + for(suite_i = strlen(suite_name); suite_i < 80-8-5; suite_i++) printf("."); + printf("%s\n", suite_pass ? " pass" : " fail"); + if(!suite_pass) suites_failed++; + if(!tests_in_suite) suites_empty++; +} +/* + * End adapted code + */ + +std::vector _bool_array_to_vector(bool *arr, unsigned int n) { + std::vector vec; + + for(unsigned int i = 0; i < n; i++) + vec.push_back(arr[i]); + + return vec; +} + +std::vector _uint32_array_to_vector(uint32_t *arr, unsigned int n) { + std::vector vec; + + for(unsigned int i = 0; i < n; i++) + vec.push_back(arr[i]); + + return vec; +} + +std::vector _double_array_to_vector(double *arr, unsigned int n) { + std::vector vec; + + for(unsigned int i = 0; i < n; i++) + vec.push_back(arr[i]); + + return vec; +} + +std::vector _string_array_to_vector(std::string *arr, unsigned int n) { + std::vector vec; + + for(unsigned int i = 0; i < n; i++) + vec.push_back(arr[i]); + + return vec; +} + +bool vec_almost_equal(std::vector a, std::vector b) { + if(a.size() != b.size()) { + return false; + } + for(unsigned int i = 0; i < a.size(); i++) { + if(!(fabs(a[i] - b[i]) < 0.000001)) { // sufficient given the tests + return false; + } + } + return true; +} + + +void test_bptree_constructor_simple() { + SUITE_START("bptree constructor simple"); + //01234567 + //11101000 + su::BPTree tree = su::BPTree("(('123:foo; bar':1,b:2)c);"); + + unsigned int exp_nparens = 8; + + std::vector exp_structure; + exp_structure.push_back(true); + exp_structure.push_back(true); + exp_structure.push_back(true); + exp_structure.push_back(false); + exp_structure.push_back(true); + exp_structure.push_back(false); + exp_structure.push_back(false); + exp_structure.push_back(false); + + std::vector exp_openclose; + exp_openclose.push_back(7); + exp_openclose.push_back(6); + exp_openclose.push_back(3); + exp_openclose.push_back(2); + exp_openclose.push_back(5); + exp_openclose.push_back(4); + exp_openclose.push_back(1); + exp_openclose.push_back(0); + + std::vector exp_names; + exp_names.push_back(std::string()); + exp_names.push_back(std::string("c")); + exp_names.push_back(std::string("123:foo; bar")); + exp_names.push_back(std::string()); + exp_names.push_back(std::string("b")); + exp_names.push_back(std::string()); + exp_names.push_back(std::string()); + exp_names.push_back(std::string()); + + std::vector exp_lengths; + exp_lengths.push_back(0.0); + exp_lengths.push_back(0.0); + exp_lengths.push_back(1.0); + exp_lengths.push_back(0.0); + exp_lengths.push_back(2.0); + exp_lengths.push_back(0.0); + exp_lengths.push_back(0.0); + exp_lengths.push_back(0.0); + + ASSERT(tree.nparens == exp_nparens); + ASSERT(tree.get_structure() == exp_structure); + ASSERT(tree.get_openclose() == exp_openclose); + ASSERT(tree.lengths == exp_lengths); + ASSERT(tree.names == exp_names); + + SUITE_END(); +} + +void test_bptree_constructor_from_existing() { + SUITE_START("bptree constructor from_existing"); + //01234567 + //11101000 + su::BPTree existing = su::BPTree("(('123:foo; bar':1,b:2)c);"); + su::BPTree tree = su::BPTree(existing.get_structure(), existing.lengths, existing.names); + + unsigned int exp_nparens = 8; + + std::vector exp_structure; + exp_structure.push_back(true); + exp_structure.push_back(true); + exp_structure.push_back(true); + exp_structure.push_back(false); + exp_structure.push_back(true); + exp_structure.push_back(false); + exp_structure.push_back(false); + exp_structure.push_back(false); + + std::vector exp_openclose; + exp_openclose.push_back(7); + exp_openclose.push_back(6); + exp_openclose.push_back(3); + exp_openclose.push_back(2); + exp_openclose.push_back(5); + exp_openclose.push_back(4); + exp_openclose.push_back(1); + exp_openclose.push_back(0); + + std::vector exp_names; + exp_names.push_back(std::string()); + exp_names.push_back(std::string("c")); + exp_names.push_back(std::string("123:foo; bar")); + exp_names.push_back(std::string()); + exp_names.push_back(std::string("b")); + exp_names.push_back(std::string()); + exp_names.push_back(std::string()); + exp_names.push_back(std::string()); + + std::vector exp_lengths; + exp_lengths.push_back(0.0); + exp_lengths.push_back(0.0); + exp_lengths.push_back(1.0); + exp_lengths.push_back(0.0); + exp_lengths.push_back(2.0); + exp_lengths.push_back(0.0); + exp_lengths.push_back(0.0); + exp_lengths.push_back(0.0); + + ASSERT(tree.nparens == exp_nparens); + ASSERT(tree.get_structure() == exp_structure); + ASSERT(tree.get_openclose() == exp_openclose); + ASSERT(tree.lengths == exp_lengths); + ASSERT(tree.names == exp_names); + + SUITE_END(); +} + +void test_bptree_mask() { + SUITE_START("bptree mask"); + //01234567 + //11101000 + //111000 + std::vector mask = {true, true, true, true, false, false, true, true}; + su::BPTree base = su::BPTree("(('123:foo; bar':1,b:2)c);"); + su::BPTree tree = base.mask(mask, base.lengths); + unsigned int exp_nparens = 6; + + std::vector exp_structure; + exp_structure.push_back(true); + exp_structure.push_back(true); + exp_structure.push_back(true); + exp_structure.push_back(false); + exp_structure.push_back(false); + exp_structure.push_back(false); + + std::vector exp_openclose; + exp_openclose.push_back(5); + exp_openclose.push_back(4); + exp_openclose.push_back(3); + exp_openclose.push_back(2); + exp_openclose.push_back(1); + exp_openclose.push_back(0); + + std::vector exp_names; + exp_names.push_back(std::string()); + exp_names.push_back(std::string("c")); + exp_names.push_back(std::string("123:foo; bar")); + exp_names.push_back(std::string()); + exp_names.push_back(std::string()); + exp_names.push_back(std::string()); + + std::vector exp_lengths; + exp_lengths.push_back(0.0); + exp_lengths.push_back(0.0); + exp_lengths.push_back(1.0); + exp_lengths.push_back(0.0); + exp_lengths.push_back(0.0); + exp_lengths.push_back(0.0); + + ASSERT(tree.nparens == exp_nparens); + ASSERT(tree.get_structure() == exp_structure); + ASSERT(tree.get_openclose() == exp_openclose); + ASSERT(tree.lengths == exp_lengths); + ASSERT(tree.names == exp_names); + + SUITE_END(); +} + +void test_bptree_constructor_single_descendent() { + SUITE_START("bptree constructor single descendent"); + + su::BPTree tree = su::BPTree("(((a)b)c,((d)e)f,g)r;"); + + unsigned int exp_nparens = 16; + + bool structure_arr[] = {1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0}; + std::vector exp_structure = _bool_array_to_vector(structure_arr, exp_nparens); + + double length_arr[] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; + std::vector exp_lengths = _double_array_to_vector(length_arr, exp_nparens); + + std::string names_arr[] = {"r", "c", "b", "a", "", "", "", "f", "e", "d", "", "", "", "g", "", ""}; + std::vector exp_names = _string_array_to_vector(names_arr, exp_nparens); + + ASSERT(tree.nparens == exp_nparens); + ASSERT(tree.get_structure() == exp_structure); + ASSERT(vec_almost_equal(tree.lengths, exp_lengths)); + ASSERT(tree.names == exp_names); + + SUITE_END(); +} + +void test_bptree_constructor_complex() { + SUITE_START("bp tree constructor complex"); + su::BPTree tree = su::BPTree("(((a:1,b:2.5)c:6,d:8,(e),(f,g,(h:1,i:2)j:1)k:1.2)l,m:2)r;"); + + unsigned int exp_nparens = 30; + + bool structure_arr[] = {1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0}; + std::vector exp_structure = _bool_array_to_vector(structure_arr, exp_nparens); + + double length_arr[] = {0, 0, 6, 1, 0, 2.5, 0, 0, 8, 0, 0, 0, 0, 0, 1.2, 0, 0, 0, 0, 1, 1, 0, 2, 0, 0, 0, 0, 2, 0, 0}; + std::vector exp_lengths = _double_array_to_vector(length_arr, exp_nparens); + + std::string names_arr[] = {"r", "l", "c", "a", "", "b", "", "", "d", "", "", "e", "", "", "k", "f", "", "g", "", "j", "h", "", "i", "", "", "", "", "m", "", ""}; + std::vector exp_names = _string_array_to_vector(names_arr, exp_nparens); + + ASSERT(tree.nparens == exp_nparens); + ASSERT(tree.get_structure() == exp_structure); + ASSERT(vec_almost_equal(tree.lengths, exp_lengths)); + ASSERT(tree.names == exp_names); + SUITE_END(); +} + +void test_bptree_constructor_semicolon() { + SUITE_START("bp tree constructor semicolon"); + su::BPTree tree = su::BPTree("((a,(b,c):5)'d','e; foo':10,((f))g)r;"); + + unsigned int exp_nparens = 20; + + bool structure_arr[] = {1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0}; + std::vector exp_structure = _bool_array_to_vector(structure_arr, exp_nparens); + + double length_arr[] = {0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0}; + std::vector exp_lengths = _double_array_to_vector(length_arr, exp_nparens); + + std::string names_arr[] = {"r", "d", "a", "", "", "b", "", "c", "", "", "", "e; foo", "", "g", "", "f", "", "", "", ""}; + std::vector exp_names = _string_array_to_vector(names_arr, exp_nparens); + + ASSERT(tree.nparens == exp_nparens); + ASSERT(tree.get_structure() == exp_structure); + ASSERT(vec_almost_equal(tree.lengths, exp_lengths)); + ASSERT(tree.names == exp_names); + SUITE_END(); +} + +void test_bptree_constructor_edgecases() { + SUITE_START("bp tree constructor edgecases"); + + su::BPTree tree1 = su::BPTree("((a,b));"); + bool structure_arr1[] = {1, 1, 1, 0, 1, 0, 0, 0}; + std::vector exp_structure1 = _bool_array_to_vector(structure_arr1, 8); + + su::BPTree tree2 = su::BPTree("(a);"); + bool structure_arr2[] = {1, 1, 0, 0}; + std::vector exp_structure2 = _bool_array_to_vector(structure_arr2, 4); + + su::BPTree tree3 = su::BPTree("();"); + bool structure_arr3[] = {1, 1, 0, 0}; + std::vector exp_structure3 = _bool_array_to_vector(structure_arr3, 4); + + su::BPTree tree4 = su::BPTree("((a,b),c);"); + bool structure_arr4[] = {1, 1, 1, 0, 1, 0, 0, 1, 0, 0}; + std::vector exp_structure4 = _bool_array_to_vector(structure_arr4, 10); + + su::BPTree tree5 = su::BPTree("(a,(b,c));"); + bool structure_arr5[] = {1, 1, 0, 1, 1, 0, 1, 0, 0, 0}; + std::vector exp_structure5 = _bool_array_to_vector(structure_arr5, 10); + + ASSERT(tree1.get_structure() == exp_structure1); + ASSERT(tree2.get_structure() == exp_structure2); + ASSERT(tree3.get_structure() == exp_structure3); + ASSERT(tree4.get_structure() == exp_structure4); + ASSERT(tree5.get_structure() == exp_structure5); + + SUITE_END(); +} + +void test_bptree_constructor_quoted_comma() { + SUITE_START("quoted comma bug"); + su::BPTree tree = su::BPTree("((3,'foo,bar')x,c)r;"); + std::vector exp_names = {"r", "x", "3", "", "foo,bar", "", "", "c", "", ""}; + ASSERT(exp_names.size() == tree.names.size()); + + for(unsigned int i = 0; i < tree.names.size(); i++) { + ASSERT(exp_names[i] == tree.names[i]); + } + SUITE_END(); +} + +void test_bptree_constructor_quoted_parens() { + SUITE_START("quoted parens"); + su::BPTree tree = su::BPTree("((3,'foo(b)ar')x,c)r;"); + std::vector exp_names = {"r", "x", "3", "", "foo(b)ar", "", "", "c", "", ""}; + ASSERT(exp_names.size() == tree.names.size()); + + for(unsigned int i = 0; i < tree.names.size(); i++) { + ASSERT(exp_names[i] == tree.names[i]); + } + SUITE_END(); +} +void test_bptree_postorder() { + SUITE_START("postorderselect"); + + // fig1 from https://www.dcc.uchile.cl/~gnavarro/ps/tcs16.2.pdf + su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); + uint32_t exp[] = {2, 4, 7, 6, 1, 11, 15, 17, 14, 13, 0}; + uint32_t obs[tree.nparens / 2]; + + for(unsigned int i = 0; i < (tree.nparens / 2); i++) + obs[i] = tree.postorderselect(i); + + std::vector exp_v = _uint32_array_to_vector(exp, tree.nparens / 2); + std::vector obs_v = _uint32_array_to_vector(obs, tree.nparens / 2); + + ASSERT(obs_v == exp_v); + SUITE_END(); +} + +void test_bptree_preorder() { + SUITE_START("preorderselect"); + + // fig1 from https://www.dcc.uchile.cl/~gnavarro/ps/tcs16.2.pdf + su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); + uint32_t exp[] = {0, 1, 2, 4, 6, 7, 11, 13, 14, 15, 17}; + uint32_t obs[tree.nparens / 2]; + + for(unsigned int i = 0; i < (tree.nparens / 2); i++) + obs[i] = tree.preorderselect(i); + + std::vector exp_v = _uint32_array_to_vector(exp, tree.nparens / 2); + std::vector obs_v = _uint32_array_to_vector(obs, tree.nparens / 2); + + ASSERT(obs_v == exp_v); + SUITE_END(); +} + +void test_bptree_parent() { + SUITE_START("parent"); + + // fig1 from https://www.dcc.uchile.cl/~gnavarro/ps/tcs16.2.pdf + su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); + uint32_t exp[] = {0, 1, 1, 1, 1, 1, 6, 6, 1, 0, 0, 0, 0, 13, 14, 14, 14, 14, 13, 0}; + + // all the -2 and +1 garbage is to avoid testing the root. + uint32_t obs[tree.nparens - 2]; + + for(int i = 0; i < (int(tree.nparens) - 2); i++) + obs[i] = tree.parent(i+1); + + std::vector exp_v = _uint32_array_to_vector(exp, tree.nparens - 2); + std::vector obs_v = _uint32_array_to_vector(obs, tree.nparens - 2); + + ASSERT(obs_v == exp_v); + SUITE_END(); +} + +void test_biom_constructor() { + SUITE_START("biom constructor"); + + su::biom table = su::biom("test.biom"); + uint32_t exp_n_samples = 6; + uint32_t exp_n_obs = 5; + + std::string sids[] = {"Sample1", "Sample2", "Sample3", "Sample4", "Sample5", "Sample6"}; + std::vector exp_sids = _string_array_to_vector(sids, exp_n_samples); + + std::string oids[] = {"GG_OTU_1", "GG_OTU_2","GG_OTU_3", "GG_OTU_4", "GG_OTU_5"}; + std::vector exp_oids = _string_array_to_vector(oids, exp_n_obs); + + uint32_t s_indptr[] = {0, 2, 5, 9, 11, 12, 15}; + std::vector exp_s_indptr = _uint32_array_to_vector(s_indptr, exp_n_samples + 1); + + uint32_t o_indptr[] = {0, 1, 6, 9, 13, 15}; + std::vector exp_o_indptr = _uint32_array_to_vector(o_indptr, exp_n_obs + 1); + + uint32_t exp_nnz = 15; + + ASSERT(table.n_samples == exp_n_samples); + ASSERT(table.n_obs == exp_n_obs); + ASSERT(table.nnz == exp_nnz); + ASSERT(table.sample_ids == exp_sids); + ASSERT(table.obs_ids == exp_oids); + ASSERT(table.sample_indptr == exp_s_indptr); + ASSERT(table.obs_indptr == exp_o_indptr); + + SUITE_END(); +} + +void test_biom_get_obs_data() { + SUITE_START("biom get obs data"); + + su::biom table = su::biom("test.biom"); + double exp0[] = {0.0, 0.0, 1.0, 0.0, 0.0, 0.0}; + std::vector exp0_vec = _double_array_to_vector(exp0, 6); + double exp1[] = {5.0, 1.0, 0.0, 2.0, 3.0, 1.0}; + std::vector exp1_vec = _double_array_to_vector(exp1, 6); + double exp2[] = {0.0, 0.0, 1.0, 4.0, 0.0, 2.0}; + std::vector exp2_vec = _double_array_to_vector(exp2, 6); + double exp3[] = {2.0, 1.0, 1.0, 0.0, 0.0, 1.0}; + std::vector exp3_vec = _double_array_to_vector(exp3, 6); + double exp4[] = {0.0, 1.0, 1.0, 0.0, 0.0, 0.0}; + std::vector exp4_vec = _double_array_to_vector(exp4, 6); + + double *out = (double*)malloc(sizeof(double) * 6); + std::vector obs_vec; + + table.get_obs_data(std::string("GG_OTU_1").c_str(), out); + obs_vec = _double_array_to_vector(out, 6); + ASSERT(vec_almost_equal(obs_vec, exp0_vec)); + + table.get_obs_data(std::string("GG_OTU_2").c_str(), out); + obs_vec = _double_array_to_vector(out, 6); + ASSERT(vec_almost_equal(obs_vec, exp1_vec)); + + table.get_obs_data(std::string("GG_OTU_3").c_str(), out); + obs_vec = _double_array_to_vector(out, 6); + ASSERT(vec_almost_equal(obs_vec, exp2_vec)); + + table.get_obs_data(std::string("GG_OTU_4").c_str(), out); + obs_vec = _double_array_to_vector(out, 6); + ASSERT(vec_almost_equal(obs_vec, exp3_vec)); + + table.get_obs_data(std::string("GG_OTU_5").c_str(), out); + obs_vec = _double_array_to_vector(out, 6); + ASSERT(vec_almost_equal(obs_vec, exp4_vec)); + + free(out); + SUITE_END(); +} + +void test_bptree_leftchild() { + SUITE_START("test bptree left child"); + su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); + + uint32_t exp[] = {1, 2, 0, 0, 7, 0, 0, 14, 15, 0, 0}; + std::vector structure = tree.get_structure(); + + uint32_t exp_pos = 0; + for(unsigned int i = 0; i < tree.nparens; i++) { + if(structure[i]) + ASSERT(tree.leftchild(i) == exp[exp_pos++]); + } + SUITE_END(); +} + +void test_bptree_rightchild() { + SUITE_START("test bptree right child"); + su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); + + uint32_t exp[] = {13, 6, 0, 0, 7, 0, 0, 14, 17, 0, 0}; + std::vector structure = tree.get_structure(); + + uint32_t exp_pos = 0; + for(unsigned int i = 0; i < tree.nparens; i++) { + if(structure[i]) + ASSERT(tree.rightchild(i) == exp[exp_pos++]); + } + SUITE_END(); +} + +void test_bptree_rightsibling() { + SUITE_START("test bptree rightsibling"); + su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); + + uint32_t exp[] = {0, 11, 4, 6, 0, 0, 13, 0, 0, 17, 0}; + std::vector structure = tree.get_structure(); + + uint32_t exp_pos = 0; + for(unsigned int i = 0; i < tree.nparens; i++) { + if(structure[i]) + ASSERT(tree.rightsibling(i) == exp[exp_pos++]); + } + SUITE_END(); +} + +void test_propstack_constructor() { + SUITE_START("test propstack constructor"); + su::PropStack ps(10); + // nothing to test directly... + SUITE_END(); +} + +void test_propstack_push_and_pop() { + SUITE_START("test propstack push and pop"); + su::PropStack ps(10); + + double *vec1 = ps.pop(1); + double *vec2 = ps.pop(2); + double *vec3 = ps.pop(3); + double *vec1_obs; + double *vec2_obs; + double *vec3_obs; + + ps.push(1); + ps.push(2); + ps.push(3); + + vec3_obs = ps.pop(4); + vec2_obs = ps.pop(5); + vec1_obs = ps.pop(6); + + ASSERT(vec1 == vec1_obs); + ASSERT(vec2 == vec2_obs); + ASSERT(vec3 == vec3_obs); + SUITE_END(); +} + +void test_propstack_get() { + SUITE_START("test propstack get"); + su::PropStack ps(10); + + double *vec1 = ps.pop(1); + double *vec2 = ps.pop(2); + double *vec3 = ps.pop(3); + + double *vec1_obs = ps.get(1); + double *vec2_obs = ps.get(2); + double *vec3_obs = ps.get(3); + + ASSERT(vec1 == vec1_obs); + ASSERT(vec2 == vec2_obs); + ASSERT(vec3 == vec3_obs); + SUITE_END(); +} + +void test_unifrac_set_proportions() { + SUITE_START("test unifrac set proportions"); + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + // ( ( ) ( ( ) ( ) ) ( ( ) ( ) ) ) + su::BPTree tree = su::BPTree("(GG_OTU_1,(GG_OTU_2,GG_OTU_3),(GG_OTU_5,GG_OTU_4));"); + su::biom table = su::biom("test.biom"); + su::PropStack ps(table.n_samples); + + double *obs = ps.pop(4); // GG_OTU_2 + double exp4[] = {0.714285714286, 0.333333333333, 0.0, 0.333333333333, 1.0, 0.25}; + set_proportions(obs, tree, 4, table, ps); + for(unsigned int i = 0; i < table.n_samples; i++) + ASSERT(fabs(obs[i] - exp4[i]) < 0.000001); + + obs = ps.pop(6); // GG_OTU_3 + double exp6[] = {0.0, 0.0, 0.25, 0.666666666667, 0.0, 0.5}; + set_proportions(obs, tree, 6, table, ps); + for(unsigned int i = 0; i < table.n_samples; i++) + ASSERT(fabs(obs[i] - exp6[i]) < 0.000001); + + obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 + double exp3[] = {0.71428571, 0.33333333, 0.25, 1.0, 1.0, 0.75}; + set_proportions(obs, tree, 3, table, ps); + for(unsigned int i = 0; i < table.n_samples; i++) + ASSERT(fabs(obs[i] - exp3[i]) < 0.000001); + SUITE_END(); +} + +void test_unifrac_set_proportions_range() { + SUITE_START("test unifrac set proportions range"); + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + // ( ( ) ( ( ) ( ) ) ( ( ) ( ) ) ) + su::BPTree tree = su::BPTree("(GG_OTU_1,(GG_OTU_2,GG_OTU_3),(GG_OTU_5,GG_OTU_4));"); + su::biom table = su::biom("test.biom"); + + const double exp4[] = {0.714285714286, 0.333333333333, 0.0, 0.333333333333, 1.0, 0.25}; + const double exp6[] = {0.0, 0.0, 0.25, 0.666666666667, 0.0, 0.5}; + const double exp3[] = {0.71428571, 0.33333333, 0.25, 1.0, 1.0, 0.75}; + + + // first the whole table + { + su::PropStack ps(table.n_samples); + + double *obs = ps.pop(4); // GG_OTU_2 + set_proportions_range(obs, tree, 4, table, 0, table.n_samples, ps); + for(unsigned int i = 0; i < table.n_samples; i++) + ASSERT(fabs(obs[i] - exp4[i]) < 0.000001); + + obs = ps.pop(6); // GG_OTU_3 + set_proportions_range(obs, tree, 6, table, 0, table.n_samples, ps); + for(unsigned int i = 0; i < table.n_samples; i++) + ASSERT(fabs(obs[i] - exp6[i]) < 0.000001); + + obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 + set_proportions_range(obs, tree, 3, table, 0, table.n_samples, ps); + for(unsigned int i = 0; i < table.n_samples; i++) + ASSERT(fabs(obs[i] - exp3[i]) < 0.000001); + } + + // beginning + { + su::PropStack ps(3); + + double *obs = ps.pop(4); // GG_OTU_2 + set_proportions_range(obs, tree, 4, table, 0, 3, ps); + for(unsigned int i = 0; i < 3; i++) + ASSERT(fabs(obs[i] - exp4[i]) < 0.000001); + + obs = ps.pop(6); // GG_OTU_3 + set_proportions_range(obs, tree, 6, table, 0, 3, ps); + for(unsigned int i = 0; i < 3; i++) + ASSERT(fabs(obs[i] - exp6[i]) < 0.000001); + + obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 + set_proportions_range(obs, tree, 3, table, 0, 3, ps); + for(unsigned int i = 0; i < 3; i++) + ASSERT(fabs(obs[i] - exp3[i]) < 0.000001); + } + + + // end + { + su::PropStack ps(4); + + double *obs = ps.pop(4); // GG_OTU_2 + set_proportions_range(obs, tree, 4, table, 2, table.n_samples, ps); + for(unsigned int i = 2; i < table.n_samples; i++) + ASSERT(fabs(obs[i-2] - exp4[i]) < 0.000001); + + obs = ps.pop(6); // GG_OTU_3 + set_proportions_range(obs, tree, 6, table, 2, table.n_samples, ps); + for(unsigned int i = 2; i < table.n_samples; i++) + ASSERT(fabs(obs[i-2] - exp6[i]) < 0.000001); + + obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 + set_proportions_range(obs, tree, 3, table, 2, table.n_samples, ps); + for(unsigned int i = 2; i < table.n_samples; i++) + ASSERT(fabs(obs[i-2] - exp3[i]) < 0.000001); + } + + + // middle + { + const unsigned int start = 1; + const unsigned int end = 4; + su::PropStack ps(end-start); + + double *obs = ps.pop(4); // GG_OTU_2 + set_proportions_range(obs, tree, 4, table, start, end, ps); + for(unsigned int i =start; i < end; i++) + ASSERT(fabs(obs[i-start] - exp4[i]) < 0.000001); + + obs = ps.pop(6); // GG_OTU_3 + set_proportions_range(obs, tree, 6, table, start, end, ps); + for(unsigned int i = start; i < end; i++) + ASSERT(fabs(obs[i-start] - exp6[i]) < 0.000001); + + obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 + set_proportions_range(obs, tree, 3, table, start, end, ps); + for(unsigned int i = start; i < end; i++) + ASSERT(fabs(obs[i-start] - exp3[i]) < 0.000001); + } + + SUITE_END(); +} + +void test_unifrac_set_proportions_range_float() { + SUITE_START("test unifrac set proportions range float"); + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + // ( ( ) ( ( ) ( ) ) ( ( ) ( ) ) ) + su::BPTree tree = su::BPTree("(GG_OTU_1,(GG_OTU_2,GG_OTU_3),(GG_OTU_5,GG_OTU_4));"); + su::biom table = su::biom("test.biom"); + + const float exp4[] = {0.714285714286, 0.333333333333, 0.0, 0.333333333333, 1.0, 0.25}; + const float exp6[] = {0.0, 0.0, 0.25, 0.666666666667, 0.0, 0.5}; + const float exp3[] = {0.71428571, 0.33333333, 0.25, 1.0, 1.0, 0.75}; + + // just midle + { + const unsigned int start = 1; + const unsigned int end = 4; + su::PropStack ps(end-start); + + float *obs = ps.pop(4); // GG_OTU_2 + set_proportions_range(obs, tree, 4, table, start, end, ps); + for(unsigned int i =start; i < end; i++) + ASSERT(fabs(obs[i-start] - exp4[i]) < 0.000001); + + obs = ps.pop(6); // GG_OTU_3 + set_proportions_range(obs, tree, 6, table, start, end, ps); + for(unsigned int i = start; i < end; i++) + ASSERT(fabs(obs[i-start] - exp6[i]) < 0.000001); + + obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 + set_proportions_range(obs, tree, 3, table, start, end, ps); + for(unsigned int i = start; i < end; i++) + ASSERT(fabs(obs[i-start] - exp3[i]) < 0.000001); + } + + SUITE_END(); +} + + + +void test_unifrac_deconvolute_stripes() { + SUITE_START("test deconvolute stripes"); + std::vector stripes; + double s1[] = {1, 1, 1, 1, 1, 1}; + double s2[] = {2, 2, 2, 2, 2, 2}; + double s3[] = {3, 3, 3, 3, 3, 3}; + stripes.push_back(s1); + stripes.push_back(s2); + stripes.push_back(s3); + + double exp[6][6] = { {0, 1, 2, 3, 2, 1}, + {1, 0, 1, 2, 3, 2}, + {2, 1, 0, 1, 2, 3}, + {3, 2, 1, 0, 1, 2}, + {2, 3, 2, 1, 0, 1}, + {1, 2, 3, 2, 1, 0} }; + double **obs = su::deconvolute_stripes(stripes, 6); + for(unsigned int i = 0; i < 6; i++) { + for(unsigned int j = 0; j < 6; j++) { + ASSERT(exp[i][j] == obs[i][j]); + } + } + free(obs); + SUITE_END(); +} + +void test_unifrac_stripes_to_condensed_form_even() { + SUITE_START("test stripes_to_condensed_form even samples"); + std::vector stripes; + double s1[] = {0, 9, 17, 24, 30, 35, 39, 42, 44, 8}; + double s2[] = {1, 10, 18, 25, 31, 36, 40, 43, 7, 16}; + double s3[] = {2, 11, 19, 26, 32, 37, 41, 6, 15, 23}; + double s4[] = {3, 12, 20, 27, 33, 38, 5, 14, 22, 29}; + double s5[] = {4, 13, 21, 28, 34, 4, 13, 21, 28, 34}; + stripes.push_back(s1); + stripes.push_back(s2); + stripes.push_back(s3); + stripes.push_back(s4); + stripes.push_back(s5); + + double exp[45] = {/* 0, */ 0, 1, 2, 3, 4, 5, 6, 7, 8, + /* *, 0, */ 9, 10, 11, 12, 13, 14, 15, 16, + /* *, *, 0, */ 17, 18, 19, 20, 21, 22, 23, + /* *, *, *, 0, */ 24, 25, 26, 27, 28, 29, + /* *, *, *, *, 0, */ 30, 31, 32, 33, 34, + /* *, *, *, *, *, 0, */ 35, 36, 37, 38, + /* *, *, *, *, *, *, 0, */ 39, 40, 41, + /* *, *, *, *, *, *, *, 0, */ 42, 43, + /* *, *, *, *, *, *, *, *, 0, */ 44}; + /* *, *, *, *, *, *, *, *, *, *, 0 */ + + double *obs = (double*)malloc(sizeof(double) * 45); + su::stripes_to_condensed_form(stripes, 10, obs, 0, 5); + for(unsigned int i = 0; i < 45; i++) { + ASSERT(exp[i] == obs[i]); + } + free(obs); + SUITE_END(); +} + +void test_unifrac_stripes_to_condensed_form_odd() { + SUITE_START("test stripes_to_condensed_form odd samples"); + std::vector stripes; + double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0}; + double s2[] = {20, 19, 18, 17, 16, 15, 14 ,13, 12, 11, 1}; + double s3[] = {21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 2}; + double s4[] = {40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 3}; + double s5[] = {41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 4}; + stripes.push_back(s1); + stripes.push_back(s2); + stripes.push_back(s3); + stripes.push_back(s4); + stripes.push_back(s5); + + double exp[55] = {/* 0, */ 1, 20, 21, 40, 41, 47, 33, 29, 11, 0, + /* 1, 0, */ 2, 19, 22, 39, 42, 48, 32, 30, 1, + /*20, 2, 0, */ 3, 18, 23, 38, 43, 49, 31, 2, + /*21, 19, 3, 0, */ 4, 17, 24, 37, 44, 50, 3, + /*40, 22, 18, 4, 0, */ 5, 16, 25 ,36, 45 , 4, + /*41, 39, 23, 17, 5, 0, */ 6, 15, 26, 35, 46, + /*47, 42, 38, 24, 16, 6, 0, */ 7, 14, 27, 34, + /*33, 48, 43, 37, 25, 15, 7, 0,*/ 8, 13, 28, + /*29, 32, 49, 44, 36, 26, 14, 8, 0, */ 9, 12, + /*11, 30, 31, 50, 45, 35, 27, 13, 9, 0,*/ 10}; + /* 0, 1, 2, 3, 4, 46, 34, 28, 12, 10, 0}; */ + double *obs = (double*)malloc(sizeof(double) * 55); + su::stripes_to_condensed_form(stripes, 11, obs, 0, 5); + for(unsigned int i = 0; i < 55; i++) { + ASSERT(exp[i] == obs[i]); + } + free(obs); + SUITE_END(); +} + +void test_unifrac_stripes_to_condensed_form_odd2() { + SUITE_START("test stripes_to_condensed_form odd(2) samples"); + std::vector stripes; + double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9}; + double s2[] = {18, 17, 16, 15, 14, 13, 12 ,11, 10}; + double s3[] = {19, 20, 21, 22, 23, 24, 25, 26, 27}; + double s4[] = {36, 35, 34, 33, 32, 31, 30, 29, 28}; + double s5[] = {31, 30, 29, 28, 36, 35, 34, 33, 32}; + stripes.push_back(s1); + stripes.push_back(s2); + stripes.push_back(s3); + stripes.push_back(s4); + stripes.push_back(s5); + + double exp[36] = {/* 0, */ 1, 18, 19, 36, 31, 25, 11, 9, + /* 1, 0, */ 2, 17, 20, 35, 30, 26, 10, + /*20, 2, 0, */ 3, 16, 21, 34, 29, 27, + /*21, 19, 3, 0, */ 4, 15, 22, 33, 28, + /*40, 22, 18, 4, 0, */ 5, 14, 23 ,32, + /*41, 39, 23, 17, 5, 0, */ 6, 13, 24, + /*47, 42, 38, 24, 16, 6, 0, */ 7, 12, + /*47, 42, 38, 24, 16, 6, 7, 0, */ 8}; + /* 0, 1, 2, 3, 4, 46, 34, 8, 8, 0}; */ + double *obs = (double*)malloc(sizeof(double) * 36); + su::stripes_to_condensed_form(stripes, 9, obs, 0, 5); + for(unsigned int i = 0; i < 36; i++) { + ASSERT(exp[i] == obs[i]); + } + free(obs); + SUITE_END(); +} + +class ValidatedMemoryStripes : public su::MemoryStripes { + private: + const uint32_t n_stripes; + mutable std::vector stripe_status; // 0 new, 1 allocated, 2 deallocated, 3 reallocated, 6 deallocate after rellocation + public: + ValidatedMemoryStripes(uint32_t _n_stripes, std::vector &_stripes) + : su::MemoryStripes(_stripes) + , n_stripes(_n_stripes) + , stripe_status(n_stripes) + { + for (uint32_t i=0; i2); + return out; + } + + +}; + + +void test_unifrac_stripes_to_matrix_even() { + SUITE_START("test stripes_to_matrix even samples"); + std::vector stripes; + double s1[] = {0, 9, 17, 24, 30, 35, 39, 42, 44, 8}; + double s2[] = {1, 10, 18, 25, 31, 36, 40, 43, 7, 16}; + double s3[] = {2, 11, 19, 26, 32, 37, 41, 6, 15, 23}; + double s4[] = {3, 12, 20, 27, 33, 38, 5, 14, 22, 29}; + double s5[] = {4, 13, 21, 28, 34, 4, 13, 21, 28, 34}; + stripes.push_back(s1); + stripes.push_back(s2); + stripes.push_back(s3); + stripes.push_back(s4); + stripes.push_back(s5); + + // test also double to float conversion + float exp[100] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, + 0, 0, 9, 10, 11, 12, 13, 14, 15, 16, + 1, 9, 0, 17, 18, 19, 20, 21, 22, 23, + 2, 10, 17, 0, 24, 25, 26, 27, 28, 29, + 3, 11, 18, 24, 0, 30, 31, 32, 33, 34, + 4, 12, 19, 25, 30, 0, 35, 36, 37, 38, + 5, 13, 20, 26, 31, 35, 0, 39, 40, 41, + 6, 14, 21, 27, 32, 36, 39, 0, 42, 43, + 7, 15, 22, 28, 33, 37, 40, 42, 0, 44, + 8, 16, 23, 29, 34, 38, 41, 43, 44, 0}; + { + float *obs = (float*)malloc(sizeof(float) * 100); + ValidatedMemoryStripes vs(5,stripes); + su::stripes_to_matrix_fp32(vs, 10, 5, obs); + for(unsigned int i = 0; i < 100; i++) { + ASSERT(exp[i] == obs[i]); + } + + ASSERT(vs.allInitialized() == true); + ASSERT(vs.allDealocated() == true); + ASSERT(vs.anyRealocated() == false); + + free(obs); + } + + { // small tiles + float *obs = (float*)malloc(sizeof(float) * 100); + ValidatedMemoryStripes vs(5,stripes); + su::stripes_to_matrix_fp32(vs, 10, 5, obs, 4); + for(unsigned int i = 0; i < 100; i++) { + ASSERT(exp[i] == obs[i]); + } + + ASSERT(vs.allInitialized() == true); + ASSERT(vs.allDealocated() == true); + ASSERT(vs.anyRealocated() == false); + + free(obs); + } + + { // large tiles + float *obs = (float*)malloc(sizeof(float) * 100); + ValidatedMemoryStripes vs(5,stripes); + su::stripes_to_matrix_fp32(vs, 10, 5, obs, 128); + for(unsigned int i = 0; i < 100; i++) { + ASSERT(exp[i] == obs[i]); + } + + ASSERT(vs.allInitialized() == true); + ASSERT(vs.allDealocated() == true); + ASSERT(vs.anyRealocated() == false); + + free(obs); + } + + + // test also intermediate, 2-step procedure + double *obsC = (double*)malloc(sizeof(double) * 45); + su::stripes_to_condensed_form(stripes, 10, obsC, 0, 5); + + float *obs2 = (float*)malloc(sizeof(float) * 100); + su::condensed_form_to_matrix_fp32(obsC, 10, obs2); + + for(unsigned int i = 0; i < 100; i++) { + ASSERT(exp[i] == obs2[i]); + } + + free(obs2); + free(obsC); + SUITE_END(); +} + +void test_unifrac_stripes_to_matrix_odd() { + SUITE_START("test stripes_to_matrix odd samples"); + std::vector stripes; + double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0}; + double s2[] = {20, 19, 18, 17, 16, 15, 14 ,13, 12, 11, 1}; + double s3[] = {21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 2}; + double s4[] = {40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 3}; + double s5[] = {41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 4}; + stripes.push_back(s1); + stripes.push_back(s2); + stripes.push_back(s3); + stripes.push_back(s4); + stripes.push_back(s5); + + double exp[121] = { 0, 1, 20, 21, 40, 41, 47, 33, 29, 11, 0, + 1, 0, 2, 19, 22, 39, 42, 48, 32, 30, 1, + 20, 2, 0, 3, 18, 23, 38, 43, 49, 31, 2, + 21, 19, 3, 0, 4, 17, 24, 37, 44, 50, 3, + 40, 22, 18, 4, 0, 5, 16, 25 ,36, 45 , 4, + 41, 39, 23, 17, 5, 0, 6, 15, 26, 35, 46, + 47, 42, 38, 24, 16, 6, 0, 7, 14, 27, 34, + 33, 48, 43, 37, 25, 15, 7, 0, 8, 13, 28, + 29, 32, 49, 44, 36, 26, 14, 8, 0, 9, 12, + 11, 30, 31, 50, 45, 35, 27, 13, 9, 0, 10, + 0, 1, 2, 3, 4, 46, 34, 28, 12, 10, 0}; + + { + double *obs = (double*)malloc(sizeof(double) * 121); + ValidatedMemoryStripes vs(5,stripes); + su::stripes_to_matrix(vs, 11, 5, obs); + for(unsigned int i = 0; i < 121; i++) { + ASSERT(exp[i] == obs[i]); + } + ASSERT(vs.allInitialized() == true); + ASSERT(vs.allDealocated() == true); + ASSERT(vs.anyRealocated() == false); + free(obs); + } + + { // small tiling + double *obs = (double*)malloc(sizeof(double) * 121); + ValidatedMemoryStripes vs(5,stripes); + su::stripes_to_matrix(vs, 11, 5, obs,4); + for(unsigned int i = 0; i < 121; i++) { + ASSERT(exp[i] == obs[i]); + } + ASSERT(vs.allInitialized() == true); + ASSERT(vs.allDealocated() == true); + ASSERT(vs.anyRealocated() == false); + free(obs); + } + + { // large tiling + double *obs = (double*)malloc(sizeof(double) * 121); + ValidatedMemoryStripes vs(5,stripes); + su::stripes_to_matrix(vs, 11, 5, obs,128); + for(unsigned int i = 0; i < 121; i++) { + ASSERT(exp[i] == obs[i]); + } + ASSERT(vs.allInitialized() == true); + ASSERT(vs.allDealocated() == true); + ASSERT(vs.anyRealocated() == false); + free(obs); + } + + + // test also intermediate, 2-step procedure + double *obsC = (double*)malloc(sizeof(double) * 55); + su::stripes_to_condensed_form(stripes, 11, obsC, 0, 5); + + double *obs2 = (double*)malloc(sizeof(double) * 121); + su::condensed_form_to_matrix(obsC, 11, obs2); + + for(unsigned int i = 0; i < 121; i++) { + ASSERT(exp[i] == obs2[i]); + } + + free(obs2); + free(obsC); + SUITE_END(); +} + +void test_unifrac_stripes_to_matrix_odd2() { + SUITE_START("test stripes_to_matrix odd(2) samples"); + std::vector stripes; + double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9}; + double s2[] = {18, 17, 16, 15, 14, 13, 12 ,11, 10}; + double s3[] = {19, 20, 21, 22, 23, 24, 25, 26, 27}; + double s4[] = {36, 35, 34, 33, 32, 31, 30, 29, 28}; + double s5[] = {31, 30, 29, 28, 36, 35, 34, 33, 32}; + stripes.push_back(s1); + stripes.push_back(s2); + stripes.push_back(s3); + stripes.push_back(s4); + stripes.push_back(s5); + + double exp[81] = { 0, 1, 18, 19, 36, 31, 25, 11, 9, + 1, 0, 2, 17, 20, 35, 30, 26, 10, + 18, 2, 0, 3, 16, 21, 34, 29, 27, + 19, 17, 3, 0, 4, 15, 22, 33, 28, + 36, 20, 16, 4, 0, 5, 14, 23 ,32, + 31, 35, 21, 15, 5, 0, 6, 13, 24, + 25, 30, 34, 22, 14, 6, 0, 7, 12, + 11, 26, 29, 33, 23, 13, 7, 0, 8, + 9, 10, 27, 28, 32, 24, 12, 8, 0}; + + { + double *obs = (double*)malloc(sizeof(double) * 81); + ValidatedMemoryStripes vs(5,stripes); + su::stripes_to_matrix(vs, 9, 5, obs); + for(unsigned int i = 0; i < 81; i++) { + ASSERT(exp[i] == obs[i]); + } + ASSERT(vs.allInitialized() == true); + ASSERT(vs.allDealocated() == true); + ASSERT(vs.anyRealocated() == false); + free(obs); + } + + { // small tile + double *obs = (double*)malloc(sizeof(double) * 81); + ValidatedMemoryStripes vs(5,stripes); + su::stripes_to_matrix(vs, 9, 5, obs,4); + for(unsigned int i = 0; i < 81; i++) { + ASSERT(exp[i] == obs[i]); + } + ASSERT(vs.allInitialized() == true); + ASSERT(vs.allDealocated() == true); + ASSERT(vs.anyRealocated() == false); + free(obs); + } + + { // large tile + double *obs = (double*)malloc(sizeof(double) * 81); + ValidatedMemoryStripes vs(5,stripes); + su::stripes_to_matrix(vs, 9, 5, obs,128); + for(unsigned int i = 0; i < 81; i++) { + ASSERT(exp[i] == obs[i]); + } + ASSERT(vs.allInitialized() == true); + ASSERT(vs.allDealocated() == true); + ASSERT(vs.anyRealocated() == false); + free(obs); + } + + // test also intermediate, 2-step procedure + double *obsC = (double*)malloc(sizeof(double) * 36); + su::stripes_to_condensed_form(stripes, 9, obsC, 0, 5); + + double *obs2 = (double*)malloc(sizeof(double) * 81); + su::condensed_form_to_matrix(obsC, 9, obs2); + + for(unsigned int i = 0; i < 81; i++) { + ASSERT(exp[i] == obs2[i]); + } + + free(obs2); + free(obsC); + SUITE_END(); +} + + +void test_unnormalized_weighted_unifrac() { + SUITE_START("test unnormalized weighted unifrac"); + + std::vector threads(1); + su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); + su::biom table = su::biom("test.biom"); + + std::vector exp; + double stride1[] = {1.52380952, 1.25, 2.75, 1.33333333, 2., 1.07142857}; + double stride2[] = {2.17857143, 2.66666667, 3.25, 1.0, 1.14285714, 1.83333333}; + double stride3[] = {1.9047619, 2.66666667, 1.75, 1.9047619, 2.66666667, 1.75}; + exp.push_back(stride1); + exp.push_back(stride2); + exp.push_back(stride3); + std::vector strides = su::make_strides(6); + std::vector strides_total = su::make_strides(6); + + su::task_parameters task_p; + task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = false; + + std::vector tasks; + tasks.push_back(task_p); + su::process_stripes(std::ref(table), + std::ref(tree), + su::weighted_unnormalized, + false, + std::ref(strides), + std::ref(strides_total), + std::ref(threads), + std::ref(tasks)); + + for(unsigned int i = 0; i < 3; i++) { + for(unsigned int j = 0; j < 6; j++) { + ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); + } + free(strides[i]); + } + SUITE_END(); +} + +void test_generalized_unifrac() { + SUITE_START("test generalized unifrac"); + + std::vector threads(1); + su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); + su::biom table = su::biom("test.biom"); + + // weighted normalized unifrac as computed above + std::vector w_exp; + double w_stride1[] = {0.38095238, 0.33333333, 0.73333333, 0.33333333, 0.5, 0.26785714}; + double w_stride2[] = {0.58095238, 0.66666667, 0.86666667, 0.25, 0.28571429, 0.45833333}; + double w_stride3[] = {0.47619048, 0.66666667, 0.46666667, 0.47619048, 0.66666667, 0.46666667}; + w_exp.push_back(w_stride1); + w_exp.push_back(w_stride2); + w_exp.push_back(w_stride3); + std::vector w_strides = su::make_strides(6); + std::vector w_strides_total = su::make_strides(6); + su::task_parameters w_task_p; + w_task_p.start = 0; w_task_p.stop = 3; w_task_p.tid = 0; w_task_p.n_samples = 6; w_task_p.bypass_tips = false; + w_task_p.g_unifrac_alpha = 1.0; + + std::vector tasks; + tasks.push_back(w_task_p); + su::process_stripes(std::ref(table), + std::ref(tree), + su::generalized, + false, + std::ref(w_strides), + std::ref(w_strides_total), + std::ref(threads), + std::ref(tasks)); + + // as computed by GUniFrac v1.0 + // Sample1 Sample2 Sample3 Sample4 Sample5 Sample6 + //Sample1 0.0000000 0.4408392 0.6886965 0.7060606 0.5833333 0.3278410 + //Sample2 0.4408392 0.0000000 0.5102041 0.7500000 0.8000000 0.5208125 + //Sample3 0.6886965 0.5102041 0.0000000 0.8649351 0.9428571 0.5952381 + //Sample4 0.7060606 0.7500000 0.8649351 0.0000000 0.5000000 0.4857143 + //Sample5 0.5833333 0.8000000 0.9428571 0.5000000 0.0000000 0.7485714 + //Sample6 0.3278410 0.5208125 0.5952381 0.4857143 0.7485714 0.0000000 + std::vector d0_exp; + double d0_stride1[] = {0.4408392, 0.5102041, 0.8649351, 0.5000000, 0.7485714, 0.3278410}; + double d0_stride2[] = {0.6886965, 0.7500000, 0.9428571, 0.4857143, 0.5833333, 0.5208125}; + double d0_stride3[] = {0.7060606, 0.8000000, 0.5952381, 0.7060606, 0.8000000, 0.5952381}; + d0_exp.push_back(d0_stride1); + d0_exp.push_back(d0_stride2); + d0_exp.push_back(d0_stride3); + std::vector d0_strides = su::make_strides(6); + std::vector d0_strides_total = su::make_strides(6); + su::task_parameters d0_task_p; + d0_task_p.start = 0; d0_task_p.stop = 3; d0_task_p.tid = 0; d0_task_p.n_samples = 6; d0_task_p.bypass_tips = false; + d0_task_p.g_unifrac_alpha = 0.0; + + tasks.clear(); + tasks.push_back(d0_task_p); + su::process_stripes(std::ref(table), + std::ref(tree), + su::generalized, + false, + std::ref(d0_strides), + std::ref(d0_strides_total), + std::ref(threads), + std::ref(tasks)); + + // as computed by GUniFrac v1.0 + // Sample1 Sample2 Sample3 Sample4 Sample5 Sample6 + //Sample1 0.0000000 0.4040518 0.6285560 0.5869439 0.4082483 0.2995673 + //Sample2 0.4040518 0.0000000 0.4160597 0.7071068 0.7302479 0.4860856 + //Sample3 0.6285560 0.4160597 0.0000000 0.8005220 0.9073159 0.5218198 + //Sample4 0.5869439 0.7071068 0.8005220 0.0000000 0.4117216 0.3485667 + //Sample5 0.4082483 0.7302479 0.9073159 0.4117216 0.0000000 0.6188282 + //Sample6 0.2995673 0.4860856 0.5218198 0.3485667 0.6188282 0.0000000 + std::vector d05_exp; + double d05_stride1[] = {0.4040518, 0.4160597, 0.8005220, 0.4117216, 0.6188282, 0.2995673}; + double d05_stride2[] = {0.6285560, 0.7071068, 0.9073159, 0.3485667, 0.4082483, 0.4860856}; + double d05_stride3[] = {0.5869439, 0.7302479, 0.5218198, 0.5869439, 0.7302479, 0.5218198}; + d05_exp.push_back(d05_stride1); + d05_exp.push_back(d05_stride2); + d05_exp.push_back(d05_stride3); + std::vector d05_strides = su::make_strides(6); + std::vector d05_strides_total = su::make_strides(6); + su::task_parameters d05_task_p; + d05_task_p.start = 0; d05_task_p.stop = 3; d05_task_p.tid = 0; d05_task_p.n_samples = 6; d05_task_p.bypass_tips = false; + d05_task_p.g_unifrac_alpha = 0.5; + + tasks.clear(); + tasks.push_back(d05_task_p); + su::process_stripes(std::ref(table), + std::ref(tree), + su::generalized, + false, + std::ref(d05_strides), + std::ref(d05_strides_total), + std::ref(threads), + std::ref(tasks)); + + for(unsigned int i = 0; i < 3; i++) { + for(unsigned int j = 0; j < 6; j++) { + ASSERT(fabs(w_strides[i][j] - w_exp[i][j]) < 0.000001); + ASSERT(fabs(d0_strides[i][j] - d0_exp[i][j]) < 0.000001); + ASSERT(fabs(d05_strides[i][j] - d05_exp[i][j]) < 0.000001); + } + free(w_strides[i]); + free(d0_strides[i]); + free(d05_strides[i]); + } + SUITE_END(); +} + +void test_vaw_unifrac_weighted_normalized() { + SUITE_START("test vaw weighted normalized unifrac"); + + std::vector threads(1); + su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); + su::biom table = su::biom("test.biom"); + + // as computed by GUniFrac, the original implementation of VAW-UniFrac + // could not be found. + // Sample1 Sample2 Sample3 Sample4 Sample5 Sample6 + //Sample1 0.0000000 0.4086040 0.6240185 0.4639481 0.2857143 0.2766318 + //Sample2 0.4086040 0.0000000 0.3798594 0.6884992 0.6807616 0.4735781 + //Sample3 0.6240185 0.3798594 0.0000000 0.7713254 0.8812897 0.5047114 + //Sample4 0.4639481 0.6884992 0.7713254 0.0000000 0.6666667 0.2709298 + //Sample5 0.2857143 0.6807616 0.8812897 0.6666667 0.0000000 0.4735991 + //Sample6 0.2766318 0.4735781 0.5047114 0.2709298 0.4735991 0.0000000 + // weighted normalized unifrac as computed above + + std::vector w_exp; + double w_stride1[] = {0.4086040, 0.3798594, 0.7713254, 0.6666667, 0.4735991, 0.2766318}; + double w_stride2[] = {0.6240185, 0.6884992, 0.8812897, 0.2709298, 0.2857143, 0.4735781}; + double w_stride3[] = {0.4639481, 0.6807616, 0.5047114, 0.4639481, 0.6807616, 0.5047114}; + w_exp.push_back(w_stride1); + w_exp.push_back(w_stride2); + w_exp.push_back(w_stride3); + std::vector w_strides = su::make_strides(6); + std::vector w_strides_total = su::make_strides(6); + su::task_parameters w_task_p; + w_task_p.start = 0; w_task_p.stop = 3; w_task_p.tid = 0; w_task_p.n_samples = 6; w_task_p.bypass_tips = false; + w_task_p.g_unifrac_alpha = 1.0; + + std::vector tasks; + tasks.push_back(w_task_p); + su::process_stripes(std::ref(table), + std::ref(tree), + su::weighted_normalized, + true, + std::ref(w_strides), + std::ref(w_strides_total), + std::ref(threads), + std::ref(tasks)); + + for(unsigned int i = 0; i < 3; i++) { + for(unsigned int j = 0; j < 6; j++) { + ASSERT(fabs(w_strides[i][j] - w_exp[i][j]) < 0.000001); + } + free(w_strides[i]); + } + SUITE_END(); +} + + +void test_make_strides() { + SUITE_START("test make stripes"); + std::vector exp; + double stride[] = {0., 0., 0.}; + exp.push_back(stride); + exp.push_back(stride); + exp.push_back(stride); + + std::vector obs = su::make_strides(3); + for(unsigned int i = 0; i < 3; i++) { + for(unsigned int j = 0; j < 6; j++) { + ASSERT(fabs(obs[i][j] - exp[i][j]) < 0.000001); + } + free(obs[i]); + } +} + +void test_faith_pd() { + SUITE_START("test faith PD"); + + // Note this tree is binary (opposed to example below) + su::BPTree tree = su::BPTree("((GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1):2,(GG_OTU_5:1,GG_OTU_4:1):1);"); + su::biom table = su::biom("test.biom"); + + // make vector of expectations from faith PD + double exp[6] = {6., 7., 8., 5., 4., 7.}; + + // run faith PD to get obs + double obs[6] = {0, 0, 0, 0, 0, 0}; + + su::faith_pd(table, tree, obs); + + // ASSERT that results = expectation + for (unsigned int i = 0; i < 6; i++){ + ASSERT(fabs(exp[i]-obs[i]) < 0.000001) + } + SUITE_END(); +} + +void test_faith_pd_shear(){ + SUITE_START("test faith PD extra OTUs in tree"); + + su::BPTree tree = su::BPTree("((GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1,GG_OTU_ex:9):1):2,(GG_OTU_5:1,GG_OTU_4:1,GG_OTU_ex2:12):1);"); + su::biom table = su::biom("test.biom"); + + // make vector of expectations from faith PD + double exp[6] = {6., 7., 8., 5., 4., 7.}; + + // run faith PD to get obs + double obs[6] = {0, 0, 0, 0, 0, 0}; + + std::unordered_set to_keep(table.obs_ids.begin(), \ + table.obs_ids.end()); \ + su::BPTree tree_sheared = tree.shear(to_keep).collapse(); + su::faith_pd(table, tree_sheared, obs); + + // ASSERT that results = expectation + for (unsigned int i = 0; i < 6; i++){ + ASSERT(fabs(exp[i]-obs[i]) < 0.000001) + } + SUITE_END(); +} + +void test_unweighted_unifrac() { + SUITE_START("test unweighted unifrac"); + std::vector threads(1); + su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); + su::biom table = su::biom("test.biom"); + + std::vector exp; + double stride1[] = {0.2, 0.42857143, 0.71428571, 0.33333333, 0.6, 0.2}; + double stride2[] = {0.57142857, 0.66666667, 0.85714286, 0.4, 0.5, 0.33333333}; + double stride3[] = {0.6, 0.6, 0.42857143, 0.6, 0.6, 0.42857143}; + exp.push_back(stride1); + exp.push_back(stride2); + exp.push_back(stride3); + std::vector strides = su::make_strides(6); + std::vector strides_total = su::make_strides(6); + + su::task_parameters task_p; + task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = false; + + std::vector tasks; + tasks.push_back(task_p); + su::process_stripes(std::ref(table), + std::ref(tree), + su::unweighted, + false, + std::ref(strides), + std::ref(strides_total), + std::ref(threads), + std::ref(tasks)); + + for(unsigned int i = 0; i < 3; i++) { + for(unsigned int j = 0; j < 6; j++) { + ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); + } + free(strides[i]); + } + SUITE_END(); +} + +void test_unweighted_unifrac_fast() { + SUITE_START("test unweighted unifrac no tips"); + std::vector threads(1); + su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); + su::biom table = su::biom("test.biom"); + + std::vector exp; + double stride1[] = {0., 0., 0.5, 0., 0.5, 0.}; + double stride2[] = {0., 0.5, 0.5, 0.5, 0.5, 0.}; + double stride3[] = {0.5, 0.5, 0., 0.5, 0.5, 0.}; + exp.push_back(stride1); + exp.push_back(stride2); + exp.push_back(stride3); + std::vector strides = su::make_strides(6); + std::vector strides_total = su::make_strides(6); + + su::task_parameters task_p; + task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = true; + + std::vector tasks; + tasks.push_back(task_p); + su::process_stripes(std::ref(table), + std::ref(tree), + su::unweighted, + false, + std::ref(strides), + std::ref(strides_total), + std::ref(threads), + std::ref(tasks)); + + for(unsigned int i = 0; i < 3; i++) { + for(unsigned int j = 0; j < 6; j++) { + ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); + } + free(strides[i]); + } + SUITE_END(); +} + +void test_normalized_weighted_unifrac() { + SUITE_START("test normalized weighted unifrac"); + std::vector threads(1); + su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); + su::biom table = su::biom("test.biom"); + + std::vector exp; + double stride1[] = {0.38095238, 0.33333333, 0.73333333, 0.33333333, 0.5, 0.26785714}; + double stride2[] = {0.58095238, 0.66666667, 0.86666667, 0.25, 0.28571429, 0.45833333}; + double stride3[] = {0.47619048, 0.66666667, 0.46666667, 0.47619048, 0.66666667, 0.46666667}; + exp.push_back(stride1); + exp.push_back(stride2); + exp.push_back(stride3); + std::vector strides = su::make_strides(6); + std::vector strides_total = su::make_strides(6); + + su::task_parameters task_p; + task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = false; + + + std::vector tasks; + tasks.push_back(task_p); + su::process_stripes(std::ref(table), + std::ref(tree), + su::weighted_normalized, + false, + std::ref(strides), + std::ref(strides_total), + std::ref(threads), + std::ref(tasks)); + + for(unsigned int i = 0; i < 3; i++) { + for(unsigned int j = 0; j < 6; j++) { + ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); + } + free(strides[i]); + } + SUITE_END(); +} + +void test_bptree_shear_simple() { + SUITE_START("test bptree shear simple"); + su::BPTree tree = su::BPTree("((3:2,4:3,(6:5)5:4)2:1,7:6,((10:9,11:10)9:8)8:7)r"); + + // simple + std::unordered_set to_keep = {"4", "6", "7", "10", "11"}; + + uint32_t exp_nparens = 20; + std::vector exp_structure = {true, true, true, false, true, true, false, false, false, true, + false, true, true, true, false, true, false, false, false, false}; + std::vector exp_names = {"r", "2", "4", "", "5", "6", "", "", "", "7", "", "8", "9", "10", "", + "11", "", "", "", ""}; + std::vector exp_lengths = {0, 1, 3, 0, 4, 5, 0, 0, 0, 6, 0, 7, 8, 9, 0, 10, 0, 0, 0, 0}; + + su::BPTree obs = tree.shear(to_keep); + ASSERT(obs.get_structure() == exp_structure); + ASSERT(exp_nparens == obs.nparens); + ASSERT(vec_almost_equal(exp_lengths, obs.lengths)); + ASSERT(obs.names == exp_names); + SUITE_END(); +} + +void test_bptree_shear_deep() { + SUITE_START("test bptree shear deep"); + su::BPTree tree = su::BPTree("((3:2,4:3,(6:5)5:4)2:1,7:6,((10:9,11:10)9:8)8:7)r"); + + // deep + std::unordered_set to_keep = {"10", "11"}; + + uint32_t exp_nparens = 10; + std::vector exp_structure = {true, true, true, true, false, true, false, false, false, false}; + std::vector exp_names = {"r", "8", "9", "10", "", "11", "", "", "", ""}; + std::vector exp_lengths = {0, 7, 8, 9, 0, 10, 0, 0, 0, 0}; + + su::BPTree obs = tree.shear(to_keep); + ASSERT(exp_nparens == obs.nparens); + ASSERT(obs.get_structure() == exp_structure); + ASSERT(vec_almost_equal(exp_lengths, obs.lengths)); + ASSERT(obs.names == exp_names); + SUITE_END(); +} + +void test_test_table_ids_are_subset_of_tree() { + SUITE_START("test test_table_ids_are_subset_of_tree"); + + su::BPTree tree = su::BPTree("(a:1,b:2)r;"); + su::biom table = su::biom("test.biom"); + std::string expected = "GG_OTU_1"; + std::string observed = su::test_table_ids_are_subset_of_tree(table, tree); + ASSERT(observed == expected); + + su::BPTree tree2 = su::BPTree("(GG_OTU_1,GG_OTU_5,GG_OTU_6,GG_OTU_2,GG_OTU_3,GG_OTU_4);"); + su::biom table2 = su::biom("test.biom"); + expected = ""; + observed = su::test_table_ids_are_subset_of_tree(table2, tree2); + ASSERT(observed == expected); + SUITE_END(); +} + + +void test_bptree_get_tip_names() { + SUITE_START("test bptree get_tip_names"); + su::BPTree tree = su::BPTree("((a:2,b:3,(c:5)d:4)e:1,f:6,((g:9,h:10)i:8)j:7)r"); + + std::unordered_set expected = {"a", "b", "c", "f", "g", "h"}; + std::unordered_set observed = tree.get_tip_names(); + ASSERT(observed == expected); + SUITE_END(); +} + +void test_bptree_collapse_simple() { + SUITE_START("test bptree collapse simple"); + su::BPTree tree = su::BPTree("((3:2,4:3,(6:5)5:4)2:1,7:6,((10:9,11:10)9:8)8:7)r"); + + uint32_t exp_nparens = 18; + std::vector exp_structure = {true, true, true, false, true, false, true, false, false, + true, false, true, true, false, true, false, false, false}; + std::vector exp_names = {"r", "2", "3", "", "4", "", "6", "", "", "7", "", "9", "10", "", "11", "", "", ""}; + std::vector exp_lengths = {0, 1, 2, 0, 3, 0, 9, 0, 0, 6, 0, 15, 9, 0, 10, 0, 0, 0}; + + su::BPTree obs = tree.collapse(); + + ASSERT(obs.get_structure() == exp_structure); + ASSERT(exp_nparens == obs.nparens); + ASSERT(vec_almost_equal(exp_lengths, obs.lengths)); + ASSERT(obs.names == exp_names); + SUITE_END(); +} + +void test_bptree_collapse_edge() { + SUITE_START("test bptree collapse edge case against root"); + + su::BPTree tree = su::BPTree("((a),b)r;"); + su::BPTree exp = su::BPTree("(a,b)r;"); + su::BPTree obs = tree.collapse(); + ASSERT(obs.get_structure() == exp.get_structure()); + ASSERT(obs.names == exp.names); + ASSERT(vec_almost_equal(obs.lengths, exp.lengths)); + + SUITE_END(); +} + +void test_unifrac_sample_counts() { + SUITE_START("test unifrac sample counts"); + su::biom table = su::biom("test.biom"); + double* obs = table.sample_counts; + double exp[] = {7, 3, 4, 6, 3, 4}; + for(unsigned int i = 0; i < 6; i++) + ASSERT(obs[i] == exp[i]); + SUITE_END(); +} + +void test_set_tasks() { + SUITE_START("test set tasks"); + std::vector obs(1); + std::vector exp(1); + + exp[0].g_unifrac_alpha = 1.0; + exp[0].n_samples = 100; + exp[0].bypass_tips = false; + exp[0].start = 0; + exp[0].stop = 100; + exp[0].tid = 0; + + set_tasks(obs, 1.0, 100, 0, 100, false, 1); + ASSERT(obs[0].g_unifrac_alpha == exp[0].g_unifrac_alpha); + ASSERT(obs[0].n_samples == exp[0].n_samples); + ASSERT(obs[0].start == exp[0].start); + ASSERT(obs[0].stop == exp[0].stop); + ASSERT(obs[0].tid == exp[0].tid); + + std::vector obs2(2); + std::vector exp2(2); + + exp2[0].g_unifrac_alpha = 1.0; + exp2[0].n_samples = 100; + exp2[0].bypass_tips = false; + exp2[0].start = 0; + exp2[0].stop = 50; + exp2[0].tid = 0; + exp2[1].g_unifrac_alpha = 1.0; + exp2[1].n_samples = 100; + exp2[1].bypass_tips = false; + exp2[1].start = 50; + exp2[1].stop = 100; + exp2[1].tid = 1; + + set_tasks(obs2, 1.0, 100, 0, 100, false, 2); + for(unsigned int i=0; i < 2; i++) { + ASSERT(obs2[i].g_unifrac_alpha == exp2[i].g_unifrac_alpha); + ASSERT(obs2[i].n_samples == exp2[i].n_samples); + ASSERT(obs2[i].start == exp2[i].start); + ASSERT(obs2[i].stop == exp2[i].stop); + ASSERT(obs2[i].tid == exp2[i].tid); + } + + std::vector obs3(3); + std::vector exp3(3); + + exp3[0].g_unifrac_alpha = 1.0; + exp3[0].n_samples = 100; + exp3[0].bypass_tips = false; + exp3[0].start = 25; + exp3[0].stop = 50; + exp3[0].tid = 0; + exp3[1].g_unifrac_alpha = 1.0; + exp3[1].n_samples = 100; + exp3[1].bypass_tips = false; + exp3[1].start = 50; + exp3[1].stop = 75; + exp3[1].tid = 1; + exp3[2].g_unifrac_alpha = 1.0; + exp3[2].n_samples = 100; + exp3[2].bypass_tips = false; + exp3[2].start = 75; + exp3[2].stop = 100; + exp3[2].tid = 2; + + set_tasks(obs3, 1.0, 100, 25, 100, false, 3); + for(unsigned int i=0; i < 3; i++) { + ASSERT(obs3[i].g_unifrac_alpha == exp3[i].g_unifrac_alpha); + ASSERT(obs3[i].n_samples == exp3[i].n_samples); + ASSERT(obs3[i].start == exp3[i].start); + ASSERT(obs3[i].stop == exp3[i].stop); + ASSERT(obs3[i].tid == exp3[i].tid); + } + + std::vector obs4(3); + std::vector exp4(3); + + exp4[0].g_unifrac_alpha = 1.0; + exp4[0].n_samples = 100; + exp4[0].bypass_tips = false; + exp4[0].start = 26; + exp4[0].stop = 51; + exp4[0].tid = 0; + exp4[1].g_unifrac_alpha = 1.0; + exp4[1].n_samples = 100; + exp4[1].bypass_tips = false; + exp4[1].start = 51; + exp4[1].stop = 76; + exp4[1].tid = 1; + exp4[2].g_unifrac_alpha = 1.0; + exp4[2].n_samples = 100; + exp4[2].bypass_tips = false; + exp4[2].start = 76; + exp4[2].stop = 100; + exp4[2].tid = 2; + + set_tasks(obs4, 1.0, 100, 26, 100, false, 3); + for(unsigned int i=0; i < 3; i++) { + ASSERT(obs4[i].g_unifrac_alpha == exp4[i].g_unifrac_alpha); + ASSERT(obs4[i].n_samples == exp4[i].n_samples); + ASSERT(obs4[i].start == exp4[i].start); + ASSERT(obs4[i].stop == exp4[i].stop); + ASSERT(obs4[i].tid == exp4[i].tid); + } + + // set_tasks boundary bug + std::vector obs16(16); + std::vector exp16(16); + set_tasks(obs16, 1.0, 9511, 0, 0, false, 16); + exp16[15].start = 4459; + exp16[15].stop = 4756; + ASSERT(obs16[15].start == exp16[15].start); + ASSERT(obs16[15].stop == exp16[15].stop); + SUITE_END(); +} + +void test_bptree_constructor_newline_bug() { + SUITE_START("test bptree constructor newline bug"); + su::BPTree tree = su::BPTree("((362be41f31fd26be95ae43a8769b91c0:0.116350803,(a16679d5a10caa9753f171977552d920:0.105836235,((a7acc2abb505c3ee177a12e514d3b994:0.008268754,(4e22aa3508b98813f52e1a12ffdb74ad:0.03144211,8139c4ac825dae48454fb4800fb87896:0.043622957)0.923:0.046588301)0.997:0.120902074,((2d3df7387323e2edcbbfcb6e56a02710:0.031543994,3f6752aabcc291b67a063fb6492fd107:0.091571442)0.759:0.016335166,((d599ebe277afb0dfd4ad3c2176afc50e:5e-09,84d0affc7243c7d6261f3a7d680b873f:0.010245188)0.883:0.048993011,51121722488d0c3da1388d1b117cd239:0.119447926)0.763:0.035660204)0.921:0.058191474)0.776:0.02854575)0.657:0.052060833)0.658:0.032547569,(99647b51f775c8ddde8ed36a7d60dbcd:0.173334268,(f18a9c8112372e2916a66a9778f3741b:0.194813398,(5833416522de0cca717a1abf720079ac:5e-09,(2bf1067d2cd4f09671e3ebe5500205ca:0.031692682,(b32621bcd86cb99e846d8f6fee7c9ab8:0.031330707,1016319c25196d73bdb3096d86a9df2f:5e-09)0.058:0.01028612)0.849:0.010284866)0.791:0.041353384)0.922:0.109470534):0.022169824000000005)root;\n\n"); + SUITE_END(); +} + +int main(int argc, char** argv) { + test_bptree_constructor_simple(); + test_bptree_constructor_newline_bug(); + test_bptree_constructor_from_existing(); + test_bptree_constructor_single_descendent(); + test_bptree_constructor_complex(); + test_bptree_constructor_semicolon(); + test_bptree_constructor_edgecases(); + test_bptree_constructor_quoted_comma(); + test_bptree_constructor_quoted_parens(); + test_bptree_postorder(); + test_bptree_preorder(); + test_bptree_parent(); + test_bptree_leftchild(); + test_bptree_rightchild(); + test_bptree_rightsibling(); + test_bptree_get_tip_names(); + test_bptree_mask(); + test_bptree_shear_simple(); + test_bptree_shear_deep(); + test_bptree_collapse_simple(); + test_bptree_collapse_edge(); + + test_biom_constructor(); + test_biom_get_obs_data(); + + test_propstack_constructor(); + test_propstack_push_and_pop(); + test_propstack_get(); + + test_unifrac_set_proportions(); + test_unifrac_set_proportions_range(); + test_unifrac_set_proportions_range_float(); + test_unifrac_deconvolute_stripes(); + test_unifrac_stripes_to_condensed_form_even(); + test_unifrac_stripes_to_condensed_form_odd(); + test_unifrac_stripes_to_condensed_form_odd2(); + test_unifrac_stripes_to_matrix_even(); + test_unifrac_stripes_to_matrix_odd(); + test_unifrac_stripes_to_matrix_odd2(); + test_unweighted_unifrac(); + test_unweighted_unifrac_fast(); + test_unnormalized_weighted_unifrac(); + test_normalized_weighted_unifrac(); + test_generalized_unifrac(); + test_vaw_unifrac_weighted_normalized(); + test_unifrac_sample_counts(); + test_set_tasks(); + test_test_table_ids_are_subset_of_tree(); + + test_faith_pd(); + test_faith_pd_shear(); + + printf("\n"); + printf(" %i / %i suites failed\n", suites_failed, suites_run); + printf(" %i / %i suites empty\n", suites_empty, suites_run); + printf(" %i / %i tests failed\n", tests_failed, tests_run); + + printf("\n THE END.\n"); + + return tests_failed ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/R/unifrac_cpp/tree.cpp b/R/unifrac_cpp/tree.cpp new file mode 100644 index 000000000..3dc6f98e7 --- /dev/null +++ b/R/unifrac_cpp/tree.cpp @@ -0,0 +1,462 @@ +#include "tree.hpp" +#include +#include + +using namespace su; + +BPTree::BPTree(std::string newick) { + openclose = std::vector(); + lengths = std::vector(); + names = std::vector(); + excess = std::vector(); + + select_0_index = std::vector(); + select_1_index = std::vector(); + structure = std::vector(); + structure.reserve(500000); // a fair sized tree... avoid reallocs, and its not _that_ much waste if this is wrong + + // three pass for parse. not ideal, but easier to map from IOW code + newick_to_bp(newick); + + // resize is correct here as we are not performing a push_back + openclose.resize(nparens); + lengths.resize(nparens); + names.resize(nparens); + select_0_index.resize(nparens / 2); + select_1_index.resize(nparens / 2); + excess.resize(nparens); + + structure_to_openclose(); + newick_to_metadata(newick); + index_and_cache(); +} + +BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names) { + structure = input_structure; + lengths = input_lengths; + names = input_names; + + nparens = structure.size(); + + openclose = std::vector(); + select_0_index = std::vector(); + select_1_index = std::vector(); + openclose.resize(nparens); + select_0_index.resize(nparens / 2); + select_1_index.resize(nparens / 2); + excess.resize(nparens); + + structure_to_openclose(); + index_and_cache(); +} + +BPTree BPTree::mask(std::vector topology_mask, std::vector in_lengths) { + + std::vector new_structure = std::vector(); + std::vector new_lengths = std::vector(); + std::vector new_names = std::vector(); + + uint32_t count = 0; + for(auto i = topology_mask.begin(); i != topology_mask.end(); i++) { + if(*i) + count++; + } + + new_structure.resize(count); + new_lengths.resize(count); + new_names.resize(count); + + auto mask_it = topology_mask.begin(); + auto base_it = this->structure.begin(); + uint32_t new_idx = 0; + uint32_t old_idx = 0; + for(; mask_it != topology_mask.end(); mask_it++, base_it++, old_idx++) { + if(*mask_it) { + new_structure[new_idx] = this->structure[old_idx]; + new_lengths[new_idx] = in_lengths[old_idx]; + new_names[new_idx] = this->names[old_idx]; + new_idx++; + } + } + + return BPTree(new_structure, new_lengths, new_names); +} + +std::unordered_set BPTree::get_tip_names() { + std::unordered_set observed; + + for(unsigned int i = 0; i < this->nparens; i++) { + if(this->isleaf(i)) { + observed.insert(this->names[i]); + } + } + + return observed; +} + +BPTree BPTree::shear(std::unordered_set to_keep) { + std::vector shearmask = std::vector(this->nparens); + int32_t p; + + for(unsigned int i = 0; i < this->nparens; i++) { + if(this->isleaf(i) && to_keep.count(this->names[i]) > 0) { + shearmask[i] = true; + shearmask[i+1] = true; + + p = this->parent(i); + while(p != -1 && !shearmask[p]) { + shearmask[p] = true; + shearmask[this->close(p)] = true; + p = this->parent(p); + } + } + } + return this->mask(shearmask, this->lengths); +} + +BPTree BPTree::collapse() { + std::vector collapsemask = std::vector(this->nparens); + std::vector new_lengths = std::vector(this->lengths); + + uint32_t current, first, last; + + for(uint32_t i = 0; i < this->nparens / 2; i++) { + current = this->preorderselect(i); + + if(this->isleaf(current) or (current == 0)) { // 0 == root + collapsemask[current] = true; + collapsemask[this->close(current)] = true; + } else { + first = this->leftchild(current); + last = this->rightchild(current); + + if(first == last) { + new_lengths[first] = new_lengths[first] + new_lengths[current]; + } else { + collapsemask[current] = true; + collapsemask[this->close(current)] = true; + } + } + } + + return this->mask(collapsemask, new_lengths); +} + /* + mask = bit_array_create(self.B.size) + bit_array_set_bit(mask, self.root()) + bit_array_set_bit(mask, self.close(self.root())) + + new_lengths = self._lengths.copy() + new_lengths_ptr = new_lengths.data + + with nogil: + for i in range(n): + current = self.preorderselect(i) + + if self.isleaf(current): + bit_array_set_bit(mask, current) + bit_array_set_bit(mask, self.close(current)) + else: + first = self.fchild(current) + last = self.lchild(current) + + if first == last: + new_lengths_ptr[first] = new_lengths_ptr[first] + \ + new_lengths_ptr[current] + else: + bit_array_set_bit(mask, current) + bit_array_set_bit(mask, self.close(current)) + + new_bp = self._mask_from_self(mask, new_lengths) + bit_array_free(mask) + return new_bp +*/ + + +BPTree::~BPTree() { +} + +void BPTree::index_and_cache() { + // should probably do the open/close in here too + unsigned int idx = 0; + auto i = structure.begin(); + auto k0 = select_0_index.begin(); + auto k1 = select_1_index.begin(); + auto e_it = excess.begin(); + unsigned int e = 0; + + for(; i != structure.end(); i++, idx++ ) { + if(*i) { + *(k1++) = idx; + *(e_it++) = ++e; + } + else { + *(k0++) = idx; + *(e_it++) = --e; + } + } +} + +uint32_t BPTree::postorderselect(uint32_t k) const { + return open(select_0_index[k]); +} + +uint32_t BPTree::preorderselect(uint32_t k) const { + return select_1_index[k]; +} + +inline uint32_t BPTree::open(uint32_t i) const { + return structure[i] ? i : openclose[i]; +} + +inline uint32_t BPTree::close(uint32_t i) const { + return structure[i] ? openclose[i] : i; +} + +bool BPTree::isleaf(unsigned int idx) const { + return (structure[idx] && !structure[idx + 1]); +} + +uint32_t BPTree::leftchild(uint32_t i) const { + // aka fchild + if(isleaf(i)) + return 0; // this is awkward, using 0 which is root, but a root cannot be a child. edge case + else + return i + 1; +} + +uint32_t BPTree::rightchild(uint32_t i) const { + // aka lchild + if(isleaf(i)) + return 0; // this is awkward, using 0 which is root, but a root cannot be a child. edge case + else + return open(close(i) - 1); +} + +uint32_t BPTree::rightsibling(uint32_t i) const { + // aka nsibling + uint32_t position = close(i) + 1; + if(position >= nparens) + return 0; // will return 0 if no sibling as root cannot have a sibling + else if(structure[position]) + return position; + else + return 0; +} + +int32_t BPTree::parent(uint32_t i) const { + return enclose(i); +} + +int32_t BPTree::enclose(uint32_t i) const { + if(structure[i]) + return bwd(i, -2) + 1; + else + return bwd(i - 1, -2) + 1; +} + +int32_t BPTree::bwd(uint32_t i, int d) const { + uint32_t target_excess = excess[i] + d; + for(int current_idx = i - 1; current_idx >= 0; current_idx--) { + if(excess[current_idx] == target_excess) + return current_idx; + } + return -1; +} + +void BPTree::newick_to_bp(std::string newick) { + char last_structure; + bool potential_single_descendent = false; + int count = 0; + bool in_quote = false; + for(auto c = newick.begin(); c != newick.end(); c++) { + if(*c == '\'') + in_quote = !in_quote; + + if(in_quote) + continue; + + switch(*c) { + case '(': + // opening of a node + count++; + structure.push_back(true); + last_structure = *c; + potential_single_descendent = true; + break; + case ')': + // closing of a node + if(potential_single_descendent || (last_structure == ',')) { + // we have a single descendent or a last child (i.e. ",)" scenario) + count += 3; + structure.push_back(true); + structure.push_back(false); + structure.push_back(false); + potential_single_descendent = false; + } else { + // it is possible still to have a single descendent in the case of + // multiple single descendents (e.g., (...()...) ) + count += 1; + structure.push_back(false); + } + last_structure = *c; + break; + case ',': + if(last_structure != ')') { + // we have a new tip + count += 2; + structure.push_back(true); + structure.push_back(false); + } + potential_single_descendent = false; + last_structure = *c; + break; + default: + break; + } + } + nparens = structure.size(); +} + + +void BPTree::structure_to_openclose() { + std::stack oc; + unsigned int open_idx; + unsigned int i = 0; + + for(auto it = structure.begin(); it != structure.end(); it++, i++) { + if(*it) { + oc.push(i); + } else { + open_idx = oc.top(); + oc.pop(); + openclose[i] = open_idx; + openclose[open_idx] = i; + } + } +} +// trim from end +// from http://stackoverflow.com/a/217605 +static inline std::string &rtrim(std::string &s) { + s.erase(std::find_if(s.rbegin(), s.rend(), + std::not1(std::ptr_fun(std::isspace))).base(), s.end()); + return s; +} + + +//// WEIRDNESS. THIS SOLVES IT WITH THE RTRIM. ISOLATE, MOVE TO CONSTRUCTOR. +void BPTree::newick_to_metadata(std::string newick) { + newick = rtrim(newick); + + std::string::iterator start = newick.begin(); + std::string::iterator end = newick.end(); + std::string token; + char last_structure = '\0'; + + unsigned int structure_idx = 0; + unsigned int lag = 0; + unsigned int open_idx; + + while(start != end) { + token = tokenize(start, end); + // this sucks. + if(token.length() == 1 && is_structure_character(token[0])) { + switch(token[0]) { + case '(': + structure_idx++; + break; + case ')': + case ',': + structure_idx++; + if(last_structure == ')') + lag++; + break; + } + } else { + // puts us on the corresponding closing parenthesis + structure_idx += lag; + lag = 0; + + open_idx = open(structure_idx); + set_node_metadata(open_idx, token); + // std::cout << structure_idx << " <-> " << open_idx << " " << token << std::endl; + // make sure to advance an extra position if we are a leaf as the + // as a leaf is by definition a 10, and doing a single advancement + // would put the structure to token mapping out of sync + if(isleaf(open_idx)) + structure_idx += 2; + else + structure_idx += 1; + + } + last_structure = token[0]; + } +} + +void BPTree::set_node_metadata(unsigned int open_idx, std::string &token) { + double length = 0.0; + std::string name = std::string(); + unsigned int colon_idx = token.find_last_of(':'); + + if(colon_idx == 0) + length = std::stof(token.substr(1)); + else if(colon_idx < token.length()) { + name = token.substr(0, colon_idx); + length = std::stof(token.substr(colon_idx + 1)); + } else + name = token; + + names[open_idx] = name; + lengths[open_idx] = length; +} + +inline bool BPTree::is_structure_character(char c) const { + return (c == '(' || c == ')' || c == ',' || c == ';'); +} + +std::string BPTree::tokenize(std::string::iterator &start, const std::string::iterator &end) { + bool inquote = false; + bool isquote = false; + char c; + std::string token; + + do { + c = *start; + start++; + + if(c == '\n') { + continue; + } + + isquote = c == '\''; + + if(inquote && isquote) { + inquote = false; + continue; + } else if(!inquote && isquote) { + inquote = true; + continue; + } + + if(is_structure_character(c) && !inquote) { + if(token.length() == 0) + token.push_back(c); + break; + } + + token.push_back(c); + + + } while(start != end); + + return token; +} + +std::vector BPTree::get_structure() { + return structure; +} + +std::vector BPTree::get_openclose() { + return openclose; +} + diff --git a/R/unifrac_cpp/tree.hpp b/R/unifrac_cpp/tree.hpp new file mode 100644 index 000000000..99f61dfec --- /dev/null +++ b/R/unifrac_cpp/tree.hpp @@ -0,0 +1,138 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#ifndef __UNIFRAC_TREE_H +#define __UNIFRAC_TREE_H 1 + +#include +#include +#include +#include +#include + +namespace su { + class BPTree { + public: + /* tracked attributes */ + std::vector lengths; + std::vector names; + + /* total number of parentheses */ + uint32_t nparens; + + /* default constructor + * + * @param newick A newick string + */ + BPTree(std::string newick); + + /* constructor from a defined topology + * + * @param input_structure A boolean vector defining the topology + * @param input_lengths A vector of double of the branch lengths + * @param input_names A vector of str of the vertex names + */ + BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names); + ~BPTree(); + + /* postorder tree traversal + * + * Get the index position of the ith node in a postorder tree + * traversal. + * + * @param i The ith node in a postorder traversal + */ + uint32_t postorderselect(uint32_t i)const ; + + /* preorder tree traversal + * + * Get the index position of the ith node in a preorder tree + * traversal. + * + * @param i The ith node in a preorder traversal + */ + uint32_t preorderselect(uint32_t i) const; + + /* Test if the node at an index position is a leaf + * + * @param i The node to evaluate + */ + bool isleaf(uint32_t i) const; + + /* Get the left child of a node + * + * @param i The node to obtain the left child from + */ + uint32_t leftchild(uint32_t i) const ; + + /* Get the right child of a node + * + * @param i The node to obtain the right child from + */ + uint32_t rightchild(uint32_t i) const; + + /* Get the right sibling of a node + * + * @param i The node to obtain the right sibling from + */ + uint32_t rightsibling(uint32_t i) const; + + /* Get the parent of a node + * + * @param i The node to obtain the parent of + */ + int32_t parent(uint32_t i) const; + + /* get the names at the tips of the tree */ + std::unordered_set get_tip_names(); + + /* public getters */ + std::vector get_structure(); + std::vector get_openclose(); + + /* serialize the structure as a sequence of 1s and 0s */ + void print() { + for(auto c = structure.begin(); c != structure.end(); c++) { + if(*c) + std::cout << "1"; + else + std::cout << "0"; + } + std::cout << std::endl; + } + BPTree mask(std::vector topology_mask, std::vector in_lengths); // mask self + + BPTree shear(std::unordered_set to_keep); + + BPTree collapse(); + + private: + std::vector structure; // the topology + std::vector openclose; // cache'd mapping between parentheses + std::vector select_0_index; // cache of select 0 + std::vector select_1_index; // cache of select 1 + std::vector excess; + + void index_and_cache(); // construct the select caches + void newick_to_bp(std::string newick); // convert a newick string to parentheses + void newick_to_metadata(std::string newick); // convert newick to attributes + void structure_to_openclose(); // set the cache mapping between parentheses pairs + void set_node_metadata(unsigned int open_idx, std::string &token); // set attributes for a node + bool is_structure_character(char c) const; // test if a character is a newick structure + inline uint32_t open(uint32_t i) const; // obtain the index of the opening for a given parenthesis + inline uint32_t close(uint32_t i) const; // obtain the index of the closing for a given parenthesis + std::string tokenize(std::string::iterator &start, const std::string::iterator &end); // newick -> tokens + + int32_t bwd(uint32_t i, int32_t d) const; + int32_t enclose(uint32_t i) const; + }; +} + +#endif /* UNIFRAC_TREE_H */ + diff --git a/R/unifrac_cpp/unifrac.cpp b/R/unifrac_cpp/unifrac.cpp new file mode 100644 index 000000000..809196ad3 --- /dev/null +++ b/R/unifrac_cpp/unifrac.cpp @@ -0,0 +1,494 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include "tree.hpp" +#include "biom_interface.hpp" +#include "unifrac.hpp" +#include "affinity.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unifrac_internal.hpp" + +// We will always have the CPU version +#define SUCMP_NM su_cpu +#include "unifrac_cmp.hpp" +#undef SUCMP_NM + +#ifdef UNIFRAC_ENABLE_ACC +#define SUCMP_NM su_acc +#include "unifrac_cmp.hpp" +#undef SUCMP_NM +#endif + +using namespace su; + +std::string su::test_table_ids_are_subset_of_tree(su::biom_interface &table, su::BPTree &tree) { + std::unordered_set tip_names = tree.get_tip_names(); + std::unordered_set::const_iterator hit; + std::string a_missing_name = ""; + + for(auto i : table.obs_ids) { + hit = tip_names.find(i); + if(hit == tip_names.end()) { + a_missing_name = i; + break; + } + } + + return a_missing_name; +} + +double** su::deconvolute_stripes(std::vector &stripes, uint32_t n) { + // would be better to just do striped_to_condensed_form + double **dm; + dm = (double**)malloc(sizeof(double*) * n); + if(dm == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(double*) * n, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + for(unsigned int i = 0; i < n; i++) { + dm[i] = (double*)malloc(sizeof(double) * n); + if(dm[i] == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(double) * n, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + dm[i][i] = 0; + } + + for(unsigned int i = 0; i < stripes.size(); i++) { + double *vec = stripes[i]; + unsigned int k = 0; + for(unsigned int row = 0, col = i + 1; row < n; row++, col++) { + if(col < n) { + dm[row][col] = vec[k]; + dm[col][row] = vec[k]; + } else { + dm[col % n][row] = vec[k]; + dm[row][col % n] = vec[k]; + } + k++; + } + } + return dm; +} + + +void su::stripes_to_condensed_form(std::vector &stripes, uint32_t n, double* cf, unsigned int start, unsigned int stop) { + // n must be >= 2, but that should be enforced upstream as that would imply + // computing unifrac on a single sample. + + uint64_t comb_N = comb_2(n); + for(unsigned int stripe = start; stripe < stop; stripe++) { + // compute the (i, j) position of each element in each stripe + uint64_t i = 0; + uint64_t j = stripe + 1; + for(uint64_t k = 0; k < n; k++, i++, j++) { + if(j == n) { + i = 0; + j = n - (stripe + 1); + } + // determine the position in the condensed form vector for a given (i, j) + // based off of + // https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.squareform.html + uint64_t comb_N_minus_i = comb_2(n - i); + cf[comb_N - comb_N_minus_i + (j - i - 1)] = stripes[stripe][k]; + } + } +} + + +// write in a 2D matrix +// also suitable for writing to disk +template +void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d) { + const uint64_t comb_N = su::comb_2(n); + for(uint64_t i = 0; i < n; i++) { + for(uint64_t j = 0; j < n; j++) { + TReal v; + if(i < j) { // upper triangle + const uint64_t comb_N_minus = su::comb_2(n - i); + v = cf[comb_N - comb_N_minus + (j - i - 1)]; + } else if (i > j) { // lower triangle + const uint64_t comb_N_minus = su::comb_2(n - j); + v = cf[comb_N - comb_N_minus + (i - j - 1)]; + } else { + v = 0.0; + } + buf2d[i*n+j] = v; + } + } +} + + +// make sure it is instantiated +template void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); +template void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); + +void su::condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d) { + su::condensed_form_to_matrix_T(cf,n,buf2d); +} + +void su::condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d) { + su::condensed_form_to_matrix_T(cf,n,buf2d); +} + +/* + * The stripes end up computing the following positions in the distance + * matrix. + * + * x A B C x x + * x x A B C x + * x x x A B C + * C x x x A B + * B C x x x A + * A B C x x x + * + * However, we store those stripes as vectors, ie + * [ A A A A A A ] + */ + + +// Helper class +// Will cache pointers and automatically release stripes when all elements are used +class OnceManagedStripes { + private: + const uint32_t n_samples; + const uint32_t n_stripes; + const ManagedStripes &stripes; + std::vector stripe_ptr; + std::vector stripe_accessed; + + const double *get_stripe(const uint32_t stripe) { + if (stripe_ptr[stripe]==0) stripe_ptr[stripe]=stripes.get_stripe(stripe); + return stripe_ptr[stripe]; + } + + void release_stripe(const uint32_t stripe) { + stripes.release_stripe(stripe); + stripe_ptr[stripe]=0; + } + + public: + OnceManagedStripes(const ManagedStripes &_stripes, const uint32_t _n_samples, const uint32_t _n_stripes) + : n_samples(_n_samples), n_stripes(_n_stripes) + , stripes(_stripes) + , stripe_ptr(n_stripes) + , stripe_accessed(n_stripes) + {} + + ~OnceManagedStripes() + { + for(uint32_t i = 0; i < n_stripes; i++) { + if (stripe_ptr[i]!=0) { + release_stripe(i); + } + } + } + + double get_val(const uint32_t stripe, const uint32_t el) + { + if (stripe_ptr[stripe]==0) stripe_ptr[stripe]=stripes.get_stripe(stripe); + const double *mystripe = stripe_ptr[stripe]; + double val = mystripe[el]; + + stripe_accessed[stripe]++; + if (stripe_accessed[stripe]==n_samples) release_stripe(stripe); // we will not use this stripe anymore + + return val; + } + + +}; + +// write in a 2D matrix +// also suitable for writing to disk +template +void su::stripes_to_matrix_T(const ManagedStripes &_stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size) { + // n_samples must be >= 2, but that should be enforced upstream as that would imply + // computing unifrac on a single sample. + + // tile for for better memory access pattern + const uint32_t TILE = (tile_size>0) ? tile_size : (128/sizeof(TReal)); + const uint32_t n_samples_tup = (n_samples+(TILE-1))/TILE; // round up + + OnceManagedStripes stripes(_stripes, n_samples, n_stripes); + + + for(uint32_t oi = 0; oi < n_samples_tup; oi++) { // off diagonal + // alternate between inner and outer off-diagonal, due to wrap around in stripes + const uint32_t o = ((oi%2)==0) ? \ + (oi/2)*TILE : /* close to diagonal */ \ + (n_samples_tup-(oi/2)-1)*TILE; /* far from diagonal */ + + for(uint32_t d = 0; d < (n_samples-o); d+=TILE) { // diagonal + + uint32_t iOut = d; + uint32_t jOut = d+o; + + uint32_t iMax = std::min(iOut+TILE,n_samples); + uint32_t jMax = std::min(jOut+TILE,n_samples); + + + if (iOut==jOut) { + // on diagonal + for(uint64_t i = iOut; i < iMax; i++) { + buf2d[i*n_samples+i] = 0.0; + + int64_t stripe=0; + + uint64_t j = i+1; + for(; (stripen_stripes) { + // ops, we overshoot... roll back + j-=(stripe-n_stripes); + stripe=n_stripes; + } + for(; (stripe(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size); +template void su::stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size); + +void su::stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size) { + return su::stripes_to_matrix_T(stripes, n_samples, n_stripes, buf2d, tile_size); +} + +void su::stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size) { + return su::stripes_to_matrix_T(stripes, n_samples, n_stripes, buf2d, tile_size); +} + + +void progressbar(float progress) { + // from http://stackoverflow.com/a/14539953 + // + // could encapsulate into a classs for displaying time elapsed etc + int barWidth = 70; + std::cout << "["; + int pos = barWidth * progress; + for (int i = 0; i < barWidth; ++i) { + if (i < pos) std::cout << "="; + else if (i == pos) std::cout << ">"; + else std::cout << " "; + } + std::cout << "] " << int(progress * 100.0) << " %\r"; + std::cout.flush(); +} + +// Computes Faith's PD for the samples in `table` over the phylogenetic +// tree given by `tree`. +// Assure that tree does not contain ids that are not in table +void su::faith_pd(biom_interface &table, + BPTree &tree, + double* result) { + PropStack propstack(table.n_samples); + + uint32_t node; + double *node_proportions; + double length; + + // for node in postorderselect + for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { + node = tree.postorderselect(k); + // get branch length + length = tree.lengths[node]; + + // get node proportions and set intermediate scores + node_proportions = propstack.pop(node); + set_proportions(node_proportions, tree, node, table, propstack); + + for (unsigned int sample = 0; sample < table.n_samples; sample++){ + // calculate contribution of node to score + result[sample] += (node_proportions[sample] > 0) * length; + } + } +} + + +#ifdef UNIFRAC_ENABLE_ACC + +// test only once, then use persistent value +static int proc_use_acc = -1; + +inline bool use_acc() { + if (proc_use_acc!=-1) return (proc_use_acc!=0); + int has_nvidia_gpu_rc = access("/proc/driver/nvidia/gpus", F_OK); + + bool print_info = false; + + if (const char* env_p = std::getenv("UNIFRAC_GPU_INFO")) { + print_info = true; + std::string env_s(env_p); + if ((env_s=="NO") || (env_s=="N") || (env_s=="no") || (env_s=="n") || + (env_s=="NEVER") || (env_s=="never")) { + print_info = false; + } + } + + + if (has_nvidia_gpu_rc != 0) { + if (print_info) printf("INFO (unifrac): GPU not found, using CPU\n"); + proc_use_acc=0; + return false; + } + + if (const char* env_p = std::getenv("UNIFRAC_USE_GPU")) { + std::string env_s(env_p); + if ((env_s=="NO") || (env_s=="N") || (env_s=="no") || (env_s=="n") || + (env_s=="NEVER") || (env_s=="never")) { + if (print_info) printf("INFO (unifrac): Use of GPU explicitly disabled, using CPU\n"); + proc_use_acc=0; + return false; + } + } + + if (print_info) printf("INFO (unifrac): Using GPU\n"); + proc_use_acc=1; + return true; +} +#endif + +void su::unifrac(biom_interface &table, + BPTree &tree, + Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const su::task_parameters* task_p) { +#ifdef UNIFRAC_ENABLE_ACC + if (use_acc()) { + su_acc::unifrac(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); + } else { +#else + if (true) { +#endif + su_cpu::unifrac(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); + } +} + + +void su::unifrac_vaw(biom_interface &table, + BPTree &tree, + Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const su::task_parameters* task_p) { +#ifdef UNIFRAC_ENABLE_ACC + if (use_acc()) { + su_acc::unifrac_vaw(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); + } else { +#else + if (true) { +#endif + su_cpu::unifrac_vaw(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); + } +} + + +void su::process_stripes(biom_interface &table, + BPTree &tree_sheared, + Method method, + bool variance_adjust, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + std::vector &threads, + std::vector &tasks) { + + // register a signal handler so we can ask the master thread for its + // progress + register_report_status(); + + // cannot use threading with openacc or openmp + for(unsigned int tid = 0; tid < threads.size(); tid++) { + if(variance_adjust) + su::unifrac_vaw( + std::ref(table), + std::ref(tree_sheared), + method, + std::ref(dm_stripes), + std::ref(dm_stripes_total), + &tasks[tid]); + else + su::unifrac( + std::ref(table), + std::ref(tree_sheared), + method, + std::ref(dm_stripes), + std::ref(dm_stripes_total), + &tasks[tid]); + } + + remove_report_status(); +} diff --git a/R/unifrac_cpp/unifrac.hpp b/R/unifrac_cpp/unifrac.hpp new file mode 100644 index 000000000..842c4aeb2 --- /dev/null +++ b/R/unifrac_cpp/unifrac.hpp @@ -0,0 +1,112 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include +#include +#include +#include +#include + +#ifndef __UNIFRAC + +#include "task_parameters.hpp" +#include "biom_interface.hpp" + + namespace su { + enum Method {unweighted, weighted_normalized, weighted_unnormalized, generalized, unweighted_fp32, weighted_normalized_fp32, weighted_unnormalized_fp32, generalized_fp32}; + + void faith_pd(biom_interface &table, BPTree &tree, double* result); + + std::string test_table_ids_are_subset_of_tree(biom_interface &table, BPTree &tree); + void unifrac(biom_interface &table, + BPTree &tree, + Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const task_parameters* task_p); + + void unifrac_vaw(biom_interface &table, + BPTree &tree, + Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const task_parameters* task_p); + + double** deconvolute_stripes(std::vector &stripes, uint32_t n); + + class ManagedStripes { + public: + virtual ~ManagedStripes() {} + virtual const double *get_stripe(uint32_t stripe) const = 0; + virtual void release_stripe(uint32_t stripe) const = 0; + }; + + class MemoryStripes : public ManagedStripes { + private: + const double * const * stripes; // just a pointer, not owned + public: + MemoryStripes(const double * const * _stripes) : stripes(_stripes) {} + MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} + MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} + MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} + MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} + + virtual const double *get_stripe(uint32_t stripe) const {return stripes[stripe];} + virtual void release_stripe(uint32_t stripe) const {}; + }; + + + void stripes_to_condensed_form(std::vector &stripes, uint32_t n, double* cf, unsigned int start, unsigned int stop); + + // tile_size==0 means memory optimized + template void stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size=0); + void stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size=0); + void stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size=0); + + + template void condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d); + void condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); + void condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); + + inline uint64_t comb_2(uint64_t N) { + // based off of _comb_int_long + // https://github.com/scipy/scipy/blob/v0.19.1/scipy/special/_comb.pyx + + // Compute binom(N, k) for integers. + // + // we're disregarding overflow as that practically should not + // happen unless the number of samples processed is in excess + // of 4 billion + uint64_t val, j, M, nterms; + uint64_t k = 2; + + M = N + 1; + nterms = k < (N - k) ? k : N - k; + + val = 1; + + for(j = 1; j < nterms + 1; j++) { + val *= M - j; + val /= j; + } + return val; + } + + // process the stripes described by tasks + void process_stripes(biom_interface &table, + BPTree &tree_sheared, + Method method, + bool variance_adjust, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + std::vector &threads, + std::vector &tasks); + } +#define __UNIFRAC 1 +#endif diff --git a/R/unifrac_cpp/unifrac_cmp.cpp b/R/unifrac_cpp/unifrac_cmp.cpp new file mode 100644 index 000000000..a2c22c91a --- /dev/null +++ b/R/unifrac_cpp/unifrac_cmp.cpp @@ -0,0 +1,395 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include "tree.hpp" +#include "biom_interface.hpp" +#include +#include +#include +#include + +#include "unifrac_internal.hpp" + +#include "unifrac_task.hpp" +// Note: unifrac_task.hpp defines SUCMP_NM, needed by unifrac_cmp.hpp +#include "unifrac_cmp.hpp" + +// embed in this file, to properly instantiate the templatized functions +#include "unifrac_task.cpp" + +using namespace SUCMP_NM; + +template +inline void initialize_sample_counts(TFloat*& _counts, const su::task_parameters* task_p, const su::biom_interface &table) { + const unsigned int n_samples = task_p->n_samples; + const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up + TFloat * counts = NULL; + int err = 0; + err = posix_memalign((void **)&counts, 4096, sizeof(TFloat) * n_samples_r); + if(counts == NULL || err != 0) { + fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", + sizeof(TFloat) * n_samples_r, err, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + for(unsigned int i = 0; i < n_samples; i++) { + counts[i] = table.sample_counts[i]; + } + // avoid NaNs + for(unsigned int i = n_samples; i < n_samples_r; i++) { + counts[i] = 0.0; + } + + _counts=counts; +} + +template +inline void unifracTT(const su::biom_interface &table, + const su::BPTree &tree, + const bool want_total, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const su::task_parameters* task_p) { + int err; + // no processor affinity whenusing openacc or openmp + + if(table.n_samples != task_p->n_samples) { + fprintf(stderr, "Task and table n_samples not equal\n"); + exit(EXIT_FAILURE); + } + const unsigned int n_samples = task_p->n_samples; + const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up + + + su::PropStackMulti propstack_multi(table.n_samples); + + const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; + + su::initialize_stripes(std::ref(dm_stripes), std::ref(dm_stripes_total), want_total, task_p); + + TaskT taskObj(std::ref(dm_stripes), std::ref(dm_stripes_total),max_emb,task_p); + + TFloat *lengths = NULL; + err = posix_memalign((void **)&lengths, 4096, sizeof(TFloat) * max_emb); + if(err != 0) { + fprintf(stderr, "posix_memalign(%d) failed: %d\n", sizeof(TFloat) * max_emb, err); + exit(EXIT_FAILURE); + } +#pragma acc enter data create(lengths[:max_emb]) + + /* + * The values in the example vectors correspond to index positions of an + * element in the resulting distance matrix. So, in the example below, + * the following can be interpreted: + * + * [0 1 2] + * [1 2 3] + * + * As comparing the sample for row 0 against the sample for col 1, the + * sample for row 1 against the sample for col 2, the sample for row 2 + * against the sample for col 3. + * + * In other words, we're computing stripes of a distance matrix. In the + * following example, we're computing over 6 samples requiring 3 + * stripes. + * + * A; stripe == 0 + * [0 1 2 3 4 5] + * [1 2 3 4 5 0] + * + * B; stripe == 1 + * [0 1 2 3 4 5] + * [2 3 4 5 0 1] + * + * C; stripe == 2 + * [0 1 2 3 4 5] + * [3 4 5 0 1 2] + * + * The stripes end up computing the following positions in the distance + * matrix. + * + * x A B C x x + * x x A B C x + * x x x A B C + * C x x x A B + * B C x x x A + * A B C x x x + * + * However, we store those stripes as vectors, ie + * [ A A A A A A ] + * + * We end up performing N / 2 redundant calculations on the last stripe + * (see C) but that is small over large N. + */ + + unsigned int k = 0; // index in tree + const unsigned int max_k = (tree.nparens / 2) - 1; + + const unsigned int num_prop_chunks = propstack_multi.get_num_stacks(); + while (k &propstack = propstack_multi.get_prop_stack(ck); + const unsigned int tstart = propstack_multi.get_start(ck); + const unsigned int tend = propstack_multi.get_end(ck); + unsigned int my_filled_emb = 0; + unsigned int my_k=k_start; + + while ((my_filled_embbypass_tips && tree.isleaf(node)) + continue; + + if (ck==0) { // they all do the same thing, so enough for the first to update the global state + lengths[filled_emb] = tree.lengths[node]; + filled_emb++; + } + taskObj.embed_proportions_range(node_proportions, tstart, tend, my_filled_emb); + my_filled_emb++; + } + if (ck==0) { // they all do the same thing, so enough for the first to update the global state + k=my_k; + } + } + + taskObj.sync_embedded_proportions(filled_emb); +#ifdef _OPENACC + // lengths may be still in use in async mode, wait +#pragma acc wait +#pragma acc update device(lengths[:filled_emb]) +#endif + taskObj._run(filled_emb,lengths); + filled_emb=0; + + su::try_report(task_p, k, max_k); + } + +#pragma acc wait + + if(want_total) { + const uint64_t start_idx = task_p->start; + const uint64_t stop_idx = task_p->stop; + + TFloat * const dm_stripes_buf = taskObj.dm_stripes.buf; + const TFloat * const dm_stripes_total_buf = taskObj.dm_stripes_total.buf; + +#pragma acc parallel loop collapse(2) present(dm_stripes_buf,dm_stripes_total_buf) + for(uint64_t i = start_idx; i < stop_idx; i++) + for(uint64_t j = 0; j < n_samples; j++) { + uint64_t idx = (i-start_idx)*n_samples_r+j; + dm_stripes_buf[idx]=dm_stripes_buf[idx]/dm_stripes_total_buf[idx]; + // taskObj.dm_stripes[i][j] = taskObj.dm_stripes[i][j] / taskObj.dm_stripes_total[i][j]; + } + + } + +#pragma acc exit data delete(lengths[:max_emb]) + free(lengths); +} + +void SUCMP_NM::unifrac(const su::biom_interface &table, + const su::BPTree &tree, + su::Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const su::task_parameters* task_p) { + switch(unifrac_method) { + case su::unweighted: + unifracTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_normalized: + unifracTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_unnormalized: + unifracTT,double>( table, tree, false, dm_stripes,dm_stripes_total,task_p); + break; + case su::generalized: + unifracTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::unweighted_fp32: + unifracTT,float>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_normalized_fp32: + unifracTT,float>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_unnormalized_fp32: + unifracTT,float>( table, tree, false, dm_stripes,dm_stripes_total,task_p); + break; + case su::generalized_fp32: + unifracTT,float>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + default: + fprintf(stderr, "Unknown unifrac task\n"); + exit(1); + break; + } +} + + +template +inline void unifrac_vawTT(const su::biom_interface &table, + const su::BPTree &tree, + const bool want_total, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const su::task_parameters* task_p) { + int err; + // no processor affinity whenusing openacc or openmp + + if(table.n_samples != task_p->n_samples) { + fprintf(stderr, "Task and table n_samples not equal\n"); + exit(EXIT_FAILURE); + } + const unsigned int n_samples = task_p->n_samples; + const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up + + su::PropStackMulti propstack_multi(table.n_samples); + su::PropStackMulti countstack_multi(table.n_samples); + + const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; + + TFloat *sample_total_counts; + + initialize_sample_counts(sample_total_counts, task_p, table); +#pragma acc enter data copyin(sample_total_counts[:n_samples_r]) + su::initialize_stripes(std::ref(dm_stripes), std::ref(dm_stripes_total), want_total, task_p); + + TaskT taskObj(std::ref(dm_stripes), std::ref(dm_stripes_total), sample_total_counts, max_emb, task_p); + + TFloat *lengths = NULL; + err = posix_memalign((void **)&lengths, 4096, sizeof(TFloat) * max_emb); + if(err != 0) { + fprintf(stderr, "posix_memalign(%d) failed: %d\n", sizeof(TFloat) * max_emb, err); + exit(EXIT_FAILURE); + } +#pragma acc enter data create(lengths[:max_emb]) + + unsigned int k = 0; // index in tree + const unsigned int max_k = (tree.nparens / 2) - 1; + + const unsigned int num_prop_chunks = propstack_multi.get_num_stacks(); + while (k &propstack = propstack_multi.get_prop_stack(ck); + su::PropStack &countstack = countstack_multi.get_prop_stack(ck); + const unsigned int tstart = propstack_multi.get_start(ck); + const unsigned int tend = propstack_multi.get_end(ck); + unsigned int my_filled_emb = 0; + unsigned int my_k=k_start; + + while ((my_filled_embbypass_tips && tree.isleaf(node)) + continue; + + if (ck==0) { // they all do the same thing, so enough for the first to update the global state + lengths[filled_emb] = tree.lengths[node]; + filled_emb++; + } + taskObj.embed_range(node_proportions, node_counts, tstart, tend, my_filled_emb); + my_filled_emb++; + } + if (ck==0) { // they all do the same thing, so enough for the first to update the global state + k=my_k; + } + } + +#pragma acc wait +#pragma acc update device(lengths[:filled_emb]) + taskObj.sync_embedded(filled_emb); + taskObj._run(filled_emb,lengths); + filled_emb = 0; + + su::try_report(task_p, k, max_k); + } + +#pragma acc wait + if(want_total) { + const uint64_t start_idx = task_p->start; + const uint64_t stop_idx = task_p->stop; + + TFloat * const dm_stripes_buf = taskObj.dm_stripes.buf; + const TFloat * const dm_stripes_total_buf = taskObj.dm_stripes_total.buf; + +#pragma acc parallel loop collapse(2) present(dm_stripes_buf,dm_stripes_total_buf) + for(uint64_t i = start_idx; i < stop_idx; i++) + for(uint64_t j = 0; j < n_samples; j++) { + uint64_t idx = (i-start_idx)*n_samples_r+j; + dm_stripes_buf[idx]=dm_stripes_buf[idx]/dm_stripes_total_buf[idx]; + // taskObj.dm_stripes[i][j] = taskObj.dm_stripes[i][j] / taskObj.dm_stripes_total[i][j]; + } + + } + + +#pragma acc exit data delete(lengths[:max_emb]) +#pragma acc exit data delete(sample_total_counts[:n_samples_r]) + free(lengths); + free(sample_total_counts); +} + +void SUCMP_NM::unifrac_vaw(const su::biom_interface &table, + const su::BPTree &tree, + su::Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const su::task_parameters* task_p) { + switch(unifrac_method) { + case su::unweighted: + unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_normalized: + unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_unnormalized: + unifrac_vawTT,double>( table, tree, false, dm_stripes,dm_stripes_total,task_p); + break; + case su::generalized: + unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::unweighted_fp32: + unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_normalized_fp32: + unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_unnormalized_fp32: + unifrac_vawTT,float >( table, tree, false, dm_stripes,dm_stripes_total,task_p); + break; + case su::generalized_fp32: + unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + default: + fprintf(stderr, "Unknown unifrac task\n"); + exit(1); + break; + } +} + diff --git a/R/unifrac_cpp/unifrac_cmp.hpp b/R/unifrac_cpp/unifrac_cmp.hpp new file mode 100644 index 000000000..9d5a57877 --- /dev/null +++ b/R/unifrac_cpp/unifrac_cmp.hpp @@ -0,0 +1,38 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#ifdef SUCMP_NM + +/* Note: Allow multiple definitions of this header, using different SUCMP_NM */ + +#include "task_parameters.hpp" +#include "tree.hpp" +#include "biom_interface.hpp" + +#include "unifrac_internal.hpp" + +namespace SUCMP_NM { + + void unifrac(const su::biom_interface &table, + const su::BPTree &tree, + su::Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const su::task_parameters* task_p); + + void unifrac_vaw(const su::biom_interface &table, + const su::BPTree &tree, + su::Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const su::task_parameters* task_p); + +} + +#endif /* SUCMP_NM */ diff --git a/R/unifrac_cpp/unifrac_internal.cpp b/R/unifrac_cpp/unifrac_internal.cpp new file mode 100644 index 000000000..1fff6e148 --- /dev/null +++ b/R/unifrac_cpp/unifrac_internal.cpp @@ -0,0 +1,290 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include "tree.hpp" +#include "biom_interface.hpp" +#include "affinity.hpp" +#include +#include +#include +#include +#include +#include +#include + +#include "unifrac_internal.hpp" + +static pthread_mutex_t printf_mutex; +static bool* report_status; + +static int sync_printf(const char *format, ...) { + // https://stackoverflow.com/a/23587285/19741 + va_list args; + va_start(args, format); + + pthread_mutex_lock(&printf_mutex); + vprintf(format, args); + pthread_mutex_unlock(&printf_mutex); + + va_end(args); +} + +static void sig_handler(int signo) { + // http://www.thegeekstuff.com/2012/03/catch-signals-sample-c-code + if (signo == SIGUSR1) { + if(report_status == NULL) + fprintf(stderr, "Cannot report status.\n"); + else { + for(int i = 0; i < CPU_SETSIZE; i++) { + report_status[i] = true; + } + } + } +} + +using namespace su; + +void su::try_report(const su::task_parameters* task_p, unsigned int k, unsigned int max_k) { + if(__builtin_expect(report_status[task_p->tid], false)) { + sync_printf("tid:%u\tstart:%u\tstop:%u\tk:%u\ttotal:%u\n", task_p->tid, task_p->start, task_p->stop, k, max_k); + report_status[task_p->tid] = false; + } +} + +void su::register_report_status() { + // register a signal handler so we can ask the master thread for its + // progress + if (signal(SIGUSR1, sig_handler) == SIG_ERR) + fprintf(stderr, "Can't catch SIGUSR1\n"); + + report_status = (bool*)calloc(sizeof(bool), CPU_SETSIZE); + pthread_mutex_init(&printf_mutex, NULL); +} + +void su::remove_report_status() { + if(report_status != NULL) { + pthread_mutex_destroy(&printf_mutex); + free(report_status); + report_status = NULL; + } +} + +template +PropStack::PropStack(uint32_t vecsize) +: prop_stack() +, prop_map() +, defaultsize(vecsize) +{ + prop_map.reserve(1000); +} + +template +PropStack::~PropStack() { + // drain stack + for(unsigned int i = 0; i < prop_stack.size(); i++) { + TFloat *vec = prop_stack.top(); + prop_stack.pop(); + free(vec); + } + + // drain the map + for(auto it = prop_map.begin(); it != prop_map.end(); it++) { + TFloat *vec = it->second; + free(vec); + } + prop_map.clear(); +} + +template +TFloat* PropStack::get(uint32_t i) { + return prop_map[i]; +} + +template +void PropStack::push(uint32_t node) { + TFloat* vec = prop_map[node]; + prop_map.erase(node); + prop_stack.push(vec); +} + +template +TFloat* PropStack::pop(uint32_t node) { + /* + * if we don't have any available vectors, create one + * add it to our record of known vectors so we can track our mallocs + */ + TFloat *vec; + int err = 0; + if(prop_stack.empty()) { + err = posix_memalign((void **)&vec, 32, sizeof(TFloat) * defaultsize); + if(vec == NULL || err != 0) { + fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", + sizeof(TFloat) * defaultsize, err, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + } + else { + vec = prop_stack.top(); + prop_stack.pop(); + } + + prop_map[node] = vec; + return vec; +} + +// make sure they get instantiated +template class su::PropStack; +template class su::PropStack; + + +void su::initialize_stripes(std::vector &dm_stripes, + std::vector &dm_stripes_total, + bool want_total, + const su::task_parameters* task_p) { + int err = 0; + for(unsigned int i = task_p->start; i < task_p->stop; i++){ + err = posix_memalign((void **)&dm_stripes[i], 4096, sizeof(double) * task_p->n_samples); + if(dm_stripes[i] == NULL || err != 0) { + fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", + sizeof(double) * task_p->n_samples, err, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + for(unsigned int j = 0; j < task_p->n_samples; j++) + dm_stripes[i][j] = 0.; + + if(want_total) { + err = posix_memalign((void **)&dm_stripes_total[i], 4096, sizeof(double) * task_p->n_samples); + if(dm_stripes_total[i] == NULL || err != 0) { + fprintf(stderr, "Failed to allocate %zd bytes err %d; [%s]:%d\n", + sizeof(double) * task_p->n_samples, err, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + for(unsigned int j = 0; j < task_p->n_samples; j++) + dm_stripes_total[i][j] = 0.; + } + } +} + +template +void su::set_proportions(TFloat* __restrict__ props, + const BPTree &tree, + uint32_t node, + const biom_interface &table, + PropStack &ps, + bool normalize) { + if(tree.isleaf(node)) { + table.get_obs_data(tree.names[node], props); + if (normalize) { +#pragma omp parallel for schedule(static) + for(unsigned int i = 0; i < table.n_samples; i++) { + props[i] /= table.sample_counts[i]; + } + } + + } else { + unsigned int current = tree.leftchild(node); + unsigned int right = tree.rightchild(node); + +#pragma omp parallel for schedule(static) + for(unsigned int i = 0; i < table.n_samples; i++) + props[i] = 0; + + while(current <= right && current != 0) { + TFloat * __restrict__ vec = ps.get(current); // pull from prop map + ps.push(current); // remove from prop map, place back on stack + +#pragma omp parallel for schedule(static) + for(unsigned int i = 0; i < table.n_samples; i++) + props[i] = props[i] + vec[i]; + + current = tree.rightsibling(current); + } + } +} + +// make sure they get instantiated +template void su::set_proportions(float* __restrict__ props, + const BPTree &tree, + uint32_t node, + const biom_interface &table, + PropStack &ps, + bool normalize); +template void su::set_proportions(double* __restrict__ props, + const BPTree &tree, + uint32_t node, + const biom_interface &table, + PropStack &ps, + bool normalize); + +template +void su::set_proportions_range(TFloat* __restrict__ props, + const BPTree &tree, + uint32_t node, + const biom_interface &table, + unsigned int start, unsigned int end, + PropStack &ps, + bool normalize) { + const unsigned int els = end-start; + if(tree.isleaf(node)) { + table.get_obs_data_range(tree.names[node], start, end, normalize, props); + } else { + const unsigned int right = tree.rightchild(node); + unsigned int current = tree.leftchild(node); + + for(unsigned int i = 0; i < els; i++) + props[i] = 0; + + while(current <= right && current != 0) { + const TFloat * __restrict__ vec = ps.get(current); // pull from prop map + ps.push(current); // remove from prop map, place back on stack + + for(unsigned int i = 0; i < els; i++) + props[i] += vec[i]; + + current = tree.rightsibling(current); + } + } +} + +// make sure they get instantiated +template void su::set_proportions_range(float* __restrict__ props, + const BPTree &tree, + uint32_t node, + const biom_interface &table, + unsigned int start, unsigned int end, + PropStack &ps, + bool normalize); +template void su::set_proportions_range(double* __restrict__ props, + const BPTree &tree, + uint32_t node, + const biom_interface &table, + unsigned int start, unsigned int end, + PropStack &ps, + bool normalize); + +std::vector su::make_strides(unsigned int n_samples) { + uint32_t n_rotations = (n_samples + 1) / 2; + std::vector dm_stripes(n_rotations); + + int err = 0; + for(unsigned int i = 0; i < n_rotations; i++) { + double* tmp; + err = posix_memalign((void **)&tmp, 32, sizeof(double) * n_samples); + if(tmp == NULL || err != 0) { + fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", + sizeof(double) * n_samples, err, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + for(unsigned int j = 0; j < n_samples; j++) + tmp[j] = 0.0; + dm_stripes[i] = tmp; + } + return dm_stripes; +} + diff --git a/R/unifrac_cpp/unifrac_internal.hpp b/R/unifrac_cpp/unifrac_internal.hpp new file mode 100644 index 000000000..b53b0b5ae --- /dev/null +++ b/R/unifrac_cpp/unifrac_internal.hpp @@ -0,0 +1,96 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#ifndef __UNIFRAC_INTERNAL +#define __UNIFRAC_INTERNAL 1 + +#include +#include +#include +#include "biom_interface.hpp" +#include "task_parameters.hpp" +#include "unifrac.hpp" + +namespace su { + // helper reporting functions + void register_report_status(); + void remove_report_status(); + void try_report(const su::task_parameters* task_p, unsigned int k, unsigned int max_k); + + template + class PropStack { + private: + std::stack prop_stack; + std::unordered_map prop_map; + uint32_t defaultsize; + public: + PropStack(uint32_t vecsize); + virtual ~PropStack(); + TFloat* pop(uint32_t i); + void push(uint32_t i); + TFloat* get(uint32_t i); + }; + + // Helper class with default constructor + // The default is small enough to fit in L1 cache + template + class PropStackFixed : public PropStack { + public: + static const uint32_t DEF_VEC_SIZE = 1024*sizeof(double)/sizeof(TFloat); + + PropStackFixed() : PropStack(DEF_VEC_SIZE) {} + }; + + // Helper class that splits a large vec_size into several smaller chunks of def_size + template + class PropStackMulti { + protected: + const uint32_t vecsize; + std::vector > multi; + + public: + PropStackMulti(uint32_t _vecsize) + : vecsize(_vecsize) + , multi((vecsize + (PropStackFixed::DEF_VEC_SIZE-1))/PropStackFixed::DEF_VEC_SIZE) // round up + {} + ~PropStackMulti() {} + + uint32_t get_num_stacks() const {return (vecsize + (PropStackFixed::DEF_VEC_SIZE-1))/PropStackFixed::DEF_VEC_SIZE;} + + uint32_t get_start(uint32_t idx) const {return idx*PropStackFixed::DEF_VEC_SIZE;} + uint32_t get_end(uint32_t idx) const {return std::min((idx+1)*PropStackFixed::DEF_VEC_SIZE, vecsize);} + + PropStackFixed &get_prop_stack(uint32_t idx) {return multi[idx];} + }; + + template + void set_proportions(TFloat* __restrict__ props, + const BPTree &tree, uint32_t node, + const biom_interface &table, + PropStack &ps, + bool normalize = true); + + template + void set_proportions_range(TFloat* __restrict__ props, + const BPTree &tree, uint32_t node, + const biom_interface &table,unsigned int start, unsigned int end, + PropStack &ps, + bool normalize = true); + + + void initialize_stripes(std::vector &dm_stripes, + std::vector &dm_stripes_total, + bool want_total, + const su::task_parameters* task_p); + + std::vector make_strides(unsigned int n_samples); + +} + +#endif diff --git a/R/unifrac_cpp/unifrac_task.cpp b/R/unifrac_cpp/unifrac_task.cpp new file mode 100644 index 000000000..0a40a7310 --- /dev/null +++ b/R/unifrac_cpp/unifrac_task.cpp @@ -0,0 +1,785 @@ +#include +#include "unifrac_task.hpp" +#include + + + + +template +void SUCMP_NM::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { + const uint64_t start_idx = this->task_p->start; + const uint64_t stop_idx = this->task_p->stop; + const uint64_t n_samples = this->task_p->n_samples; + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + + // openacc only works well with local variables + const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; + TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; + + bool * const __restrict__ zcheck = this->zcheck; + TFloat * const __restrict__ sums = this->sums; + + const uint64_t step_size = SUCMP_NM::UnifracUnnormalizedWeightedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + + // check for zero values and pre-compute single column sums +#ifdef _OPENACC +#pragma acc parallel loop present(embedded_proportions,lengths,zcheck,sums) +#else +#pragma omp parallel for default(shared) +#endif + for(uint64_t k=0; k::acc_vector_size; +#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,lengths,zcheck,sums) async +#else + // use dynamic scheduling due to non-homogeneity in the loop +#pragma omp parallel for default(shared) schedule(dynamic,1) +#endif + for(uint64_t sk = 0; sk < sample_steps ; sk++) { + for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t ik = 0; ik < step_size ; ik++) { + const uint64_t k = sk*step_size + ik; + + if (k>=n_samples) continue; // past the limit + + const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + const bool allzero_k = zcheck[k]; + const bool allzero_l1 = zcheck[l1]; + + if (allzero_k && allzero_l1) { + // nothing to do, would have to add 0 + } else { + TFloat my_stripe; + + if (allzero_k || allzero_l1) { + // one side has all zeros + // we can use the distributed property, and use the pre-computed values + + const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 + k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 + + // keep reads in the same place to maximize GPU warp performance + my_stripe = sums[ridx]; + + } else { + // both sides non zero, use the explicit but slow approach + my_stripe = 0.0; + +#pragma acc loop seq + for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); +#endif +} + +template +void SUCMP_NM::UnifracVawUnnormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { + const uint64_t start_idx = this->task_p->start; + const uint64_t stop_idx = this->task_p->stop; + const uint64_t n_samples = this->task_p->n_samples; + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + + // openacc only works well with local variables + const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; + const TFloat * const __restrict__ embedded_counts = this->embedded_counts; + const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; + TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; + + const uint64_t step_size = SUCMP_NM::UnifracVawUnnormalizedWeightedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + + // point of thread +#ifdef _OPENACC + const unsigned int acc_vector_size = SUCMP_NM::UnifracVawUnnormalizedWeightedTask::acc_vector_size; +#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,lengths) async +#else +#pragma omp parallel for default(shared) schedule(dynamic,1) +#endif + for(uint64_t sk = 0; sk < sample_steps ; sk++) { + for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t ik = 0; ik < step_size ; ik++) { + const uint64_t k = sk*step_size + ik; + const uint64_t idx = (stripe-start_idx) * n_samples_r; + TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; + //TFloat *dm_stripe = dm_stripes[stripe]; + + if (k>=n_samples) continue; // past the limit + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; + + TFloat my_stripe = dm_stripe[k]; + +#pragma acc loop seq + for (uint64_t emb=0; emb 0) { + TFloat u1 = embedded_proportions[offset + k]; + TFloat v1 = embedded_proportions[offset + l1]; + TFloat diff1 = fabs(u1 - v1); + TFloat length = lengths[emb]; + + my_stripe += (diff1 * length) / vaw; + } + } + + dm_stripe[k] = my_stripe; + } + + } + } + +#ifdef _OPENACC + // next iteration will use the alternative space + std::swap(this->embedded_proportions,this->embedded_proportions_alt); +#endif +} + +template +void SUCMP_NM::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { + const uint64_t start_idx = this->task_p->start; + const uint64_t stop_idx = this->task_p->stop; + const uint64_t n_samples = this->task_p->n_samples; + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + + // openacc only works well with local variables + const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; + TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; + TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; + + bool * const __restrict__ zcheck = this->zcheck; + TFloat * const __restrict__ sums = this->sums; + + const uint64_t step_size = SUCMP_NM::UnifracNormalizedWeightedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + + // check for zero values and pre-compute single column sums +#ifdef _OPENACC +#pragma acc parallel loop present(embedded_proportions,lengths,zcheck,sums) +#else +#pragma omp parallel for default(shared) +#endif + for(uint64_t k=0; k::acc_vector_size; +#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,lengths,zcheck,sums) async +#else + // use dynamic scheduling due to non-homogeneity in the loop +#pragma omp parallel for schedule(dynamic,1) default(shared) +#endif + for(uint64_t sk = 0; sk < sample_steps ; sk++) { + for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t ik = 0; ik < step_size ; ik++) { + const uint64_t k = sk*step_size + ik; + + if (k>=n_samples) continue; // past the limit + + const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + const bool allzero_k = zcheck[k]; + const bool allzero_l1 = zcheck[l1]; + + if (allzero_k && allzero_l1) { + // nothing to do, would have to add 0 + } else { + const uint64_t idx = (stripe-start_idx) * n_samples_r; + TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; + TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; + //TFloat *dm_stripe = dm_stripes[stripe]; + //TFloat *dm_stripe_total = dm_stripes_total[stripe]; + + // the totals can always use the distributed property + dm_stripe_total[k] += sums[k] + sums[l1]; + + TFloat my_stripe; + + if (allzero_k || allzero_l1) { + // one side has all zeros + // we can use the distributed property, and use the pre-computed values + + const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 + k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 + + // keep reads in the same place to maximize GPU warp performance + my_stripe = sums[ridx]; + + } else { + // both sides non zero, use the explicit but slow approach + + my_stripe = 0.0; + +#pragma acc loop seq + for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); +#endif +} + +template +void SUCMP_NM::UnifracVawNormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { + const uint64_t start_idx = this->task_p->start; + const uint64_t stop_idx = this->task_p->stop; + const uint64_t n_samples = this->task_p->n_samples; + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + + // openacc only works well with local variables + const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; + const TFloat * const __restrict__ embedded_counts = this->embedded_counts; + const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; + TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; + TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; + + const uint64_t step_size = SUCMP_NM::UnifracVawNormalizedWeightedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + + // point of thread +#ifdef _OPENACC + const unsigned int acc_vector_size = SUCMP_NM::UnifracVawNormalizedWeightedTask::acc_vector_size; +#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async +#else +#pragma omp parallel for schedule(dynamic,1) default(shared) +#endif + for(uint64_t sk = 0; sk < sample_steps ; sk++) { + for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t ik = 0; ik < step_size ; ik++) { + const uint64_t k = sk*step_size + ik; + const uint64_t idx = (stripe-start_idx) * n_samples_r; + TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; + TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; + //TFloat *dm_stripe = dm_stripes[stripe]; + //TFloat *dm_stripe_total = dm_stripes_total[stripe]; + + if (k>=n_samples) continue; // past the limit + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; + + TFloat my_stripe = dm_stripe[k]; + TFloat my_stripe_total = dm_stripe_total[k]; + +#pragma acc loop seq + for (uint64_t emb=0; emb 0) { + TFloat u1 = embedded_proportions[offset + k]; + TFloat v1 = embedded_proportions[offset + l1]; + TFloat diff1 = fabs(u1 - v1); + TFloat length = lengths[emb]; + + my_stripe += (diff1 * length) / vaw; + my_stripe_total += ((u1 + v1) * length) / vaw; + } + } + + dm_stripe[k] = my_stripe; + dm_stripe_total[k] = my_stripe_total; + + } + + } + } + +#ifdef _OPENACC + // next iteration will use the alternative space + std::swap(this->embedded_proportions,this->embedded_proportions_alt); +#endif +} + +template +void SUCMP_NM::UnifracGeneralizedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { + const uint64_t start_idx = this->task_p->start; + const uint64_t stop_idx = this->task_p->stop; + const uint64_t n_samples = this->task_p->n_samples; + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + + // openacc only works well with local variables + const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; + TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; + TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; + + const TFloat g_unifrac_alpha = this->task_p->g_unifrac_alpha; + + const uint64_t step_size = SUCMP_NM::UnifracGeneralizedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + + // point of thread +#ifdef _OPENACC + const unsigned int acc_vector_size = SUCMP_NM::UnifracGeneralizedTask::acc_vector_size; +#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,lengths) async +#else +#pragma omp parallel for schedule(dynamic,1) default(shared) +#endif + for(uint64_t sk = 0; sk < sample_steps ; sk++) { + for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t ik = 0; ik < step_size ; ik++) { + const uint64_t k = sk*step_size + ik; + const uint64_t idx = (stripe-start_idx) * n_samples_r; + TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; + TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; + //TFloat *dm_stripe = dm_stripes[stripe]; + //TFloat *dm_stripe_total = dm_stripes_total[stripe]; + + if (k>=n_samples) continue; // past the limit + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + TFloat my_stripe = dm_stripe[k]; + TFloat my_stripe_total = dm_stripe_total[k]; + +#pragma acc loop seq + for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); +#endif +} + +template +void SUCMP_NM::UnifracVawGeneralizedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { + const uint64_t start_idx = this->task_p->start; + const uint64_t stop_idx = this->task_p->stop; + const uint64_t n_samples = this->task_p->n_samples; + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + + const TFloat g_unifrac_alpha = this->task_p->g_unifrac_alpha; + + // openacc only works well with local variables + const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; + const TFloat * const __restrict__ embedded_counts = this->embedded_counts; + const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; + TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; + TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; + + const uint64_t step_size = SUCMP_NM::UnifracVawGeneralizedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + // quick hack, to be finished + + // point of thread +#ifdef _OPENACC + const unsigned int acc_vector_size = SUCMP_NM::UnifracVawGeneralizedTask::acc_vector_size; +#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async +#else +#pragma omp parallel for schedule(dynamic,1) default(shared) +#endif + for(uint64_t sk = 0; sk < sample_steps ; sk++) { + for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t ik = 0; ik < step_size ; ik++) { + const uint64_t k = sk*step_size + ik; + const uint64_t idx = (stripe-start_idx) * n_samples_r; + TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; + TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; + //TFloat *dm_stripe = dm_stripes[stripe]; + //TFloat *dm_stripe_total = dm_stripes_total[stripe]; + + if (k>=n_samples) continue; // past the limit + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; + + TFloat my_stripe = dm_stripe[k]; + TFloat my_stripe_total = dm_stripe_total[k]; + +#pragma acc loop seq + for (uint64_t emb=0; emb 0) { + TFloat u1 = embedded_proportions[offset + k]; + TFloat v1 = embedded_proportions[offset + l1]; + TFloat length = lengths[emb]; + + TFloat sum1 = (u1 + v1) / vaw; + TFloat sub1 = fabs(u1 - v1) / vaw; + TFloat sum_pow1 = pow(sum1, g_unifrac_alpha) * length; + + my_stripe += sum_pow1 * (sub1 / sum1); + my_stripe_total += sum_pow1; + } + } + + dm_stripe[k] = my_stripe; + dm_stripe_total[k] = my_stripe_total; + + } + } + } + +#ifdef _OPENACC + // next iteration will use the alternative space + std::swap(this->embedded_proportions,this->embedded_proportions_alt); +#endif +} + +template +void SUCMP_NM::UnifracUnweightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { + const uint64_t start_idx = this->task_p->start; + const uint64_t stop_idx = this->task_p->stop; + const uint64_t n_samples = this->task_p->n_samples; + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + + // openacc only works well with local variables + const uint64_t * const __restrict__ embedded_proportions = this->embedded_proportions; + TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; + TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; + + TFloat * const __restrict__ sums = this->sums; + + const uint64_t step_size = SUCMP_NM::UnifracUnweightedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + + const uint64_t filled_embs_els = filled_embs/64; + const uint64_t filled_embs_rem = filled_embs%64; + + const uint64_t filled_embs_els_round = (filled_embs+63)/64; + + + // pre-compute sums of length elements, since they are likely to be accessed many times + // We will use a 8-bit map, to keep it small enough to keep in L1 cache +#ifdef _OPENACC +#pragma acc parallel loop collapse(2) gang present(lengths,sums) async +#else +#pragma omp parallel for default(shared) +#endif + for (uint64_t emb_el=0; emb_el> 0) & 1) * pl[0]) + (((b8_i >> 1) & 1) * pl[1]) + + (((b8_i >> 2) & 1) * pl[2]) + (((b8_i >> 3) & 1) * pl[3]) + + (((b8_i >> 4) & 1) * pl[4]) + (((b8_i >> 5) & 1) * pl[5]) + + (((b8_i >> 6) & 1) * pl[6]) + (((b8_i >> 7) & 1) * pl[7]); + } + } + } + if (filled_embs_rem>0) { // add also the overflow elements + const uint64_t emb_el=filled_embs_els; +#ifdef _OPENACC +#pragma acc parallel loop gang present(lengths,sums) async +#else + // no advantage of OMP, too small +#endif + for (uint64_t sub8=0; sub8<8; sub8++) { + // we are summing we have enough buffer in sums + const uint64_t emb8 = emb_el*8+sub8; + TFloat * __restrict__ psum = &(sums[emb8<<8]); + +#pragma acc loop vector + // compute all the combinations for this block, set to 0 any past the limit + // as above + for (uint64_t b8_i=0; b8_i<0x100; b8_i++) { + TFloat val= 0; + for (uint64_t li=(emb8*8); li> (li-(emb8*8))) & 1) * lengths[li]; + } + psum[b8_i] = val; + } + } + } + + // point of thread +#ifdef _OPENACC +#pragma acc wait + const unsigned int acc_vector_size = SUCMP_NM::UnifracUnweightedTask::acc_vector_size; +#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,sums) async +#else + // use dynamic scheduling due to non-homogeneity in the loop +#pragma omp parallel for schedule(dynamic,1) default(shared) +#endif + for(uint64_t sk = 0; sk < sample_steps ; sk++) { + for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t ik = 0; ik < step_size ; ik++) { + const uint64_t k = sk*step_size + ik; + const uint64_t idx = (stripe-start_idx) * n_samples_r; + TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; + TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; + //TFloat *dm_stripe = dm_stripes[stripe]; + //TFloat *dm_stripe_total = dm_stripes_total[stripe]; + + if (k>=n_samples) continue; // past the limit + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + bool did_update = false; + TFloat my_stripe = 0.0; + TFloat my_stripe_total = 0.0; + +#pragma acc loop seq + for (uint64_t emb_el=0; emb_el> 8) & 0xff)] + + psum[0x200+((x1 >> 16) & 0xff)] + + psum[0x300+((x1 >> 24) & 0xff)] + + psum[0x400+((x1 >> 32) & 0xff)] + + psum[0x500+((x1 >> 40) & 0xff)] + + psum[0x600+((x1 >> 48) & 0xff)] + + psum[0x700+((x1 >> 56) )]; + my_stripe_total += psum[ (o1 & 0xff)] + + psum[0x100+((o1 >> 8) & 0xff)] + + psum[0x200+((o1 >> 16) & 0xff)] + + psum[0x300+((o1 >> 24) & 0xff)] + + psum[0x400+((o1 >> 32) & 0xff)] + + psum[0x500+((o1 >> 40) & 0xff)] + + psum[0x600+((o1 >> 48) & 0xff)] + + psum[0x700+((o1 >> 56) )]; + } + } + + if (did_update) { + dm_stripe[k] += my_stripe; + dm_stripe_total[k] += my_stripe_total; + } + + } + + } + } + +#ifdef _OPENACC + // next iteration will use the alternative space + std::swap(this->embedded_proportions,this->embedded_proportions_alt); +#endif +} + +template +void SUCMP_NM::UnifracVawUnweightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { + const uint64_t start_idx = this->task_p->start; + const uint64_t stop_idx = this->task_p->stop; + const uint64_t n_samples = this->task_p->n_samples; + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + + // openacc only works well with local variables + const uint32_t * const __restrict__ embedded_proportions = this->embedded_proportions; + const TFloat * const __restrict__ embedded_counts = this->embedded_counts; + const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; + TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; + TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; + + const uint64_t step_size = SUCMP_NM::UnifracVawUnweightedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + + const uint64_t filled_embs_els = (filled_embs+31)/32; // round up + + // point of thread +#ifdef _OPENACC + const unsigned int acc_vector_size = SUCMP_NM::UnifracVawUnweightedTask::acc_vector_size; +#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async +#else +#pragma omp parallel for schedule(dynamic,1) default(shared) +#endif + for(uint64_t sk = 0; sk < sample_steps ; sk++) { + for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t ik = 0; ik < step_size ; ik++) { + const uint64_t k = sk*step_size + ik; + const uint64_t idx = (stripe-start_idx) * n_samples_r; + TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; + TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; + //TFloat *dm_stripe = dm_stripes[stripe]; + //TFloat *dm_stripe_total = dm_stripes_total[stripe]; + + if (k>=n_samples) continue; // past the limit + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + TFloat my_stripe = dm_stripe[k]; + TFloat my_stripe_total = dm_stripe_total[k]; + + const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; + +#pragma acc loop seq + for (uint64_t emb_el=0; emb_el 0) { + TFloat length = lengths[emb]; + TFloat lv1 = length / vaw; + + my_stripe += ((x1 >> ei) & 1)*lv1; + my_stripe_total += ((o1 >> ei) & 1)*lv1; + } + } + } + } + + dm_stripe[k] = my_stripe; + dm_stripe_total[k] = my_stripe_total; + + } + + } + } + +#ifdef _OPENACC + // next iteration will use the alternative space + std::swap(this->embedded_proportions,this->embedded_proportions_alt); +#endif +} + diff --git a/R/unifrac_cpp/unifrac_task.hpp b/R/unifrac_cpp/unifrac_task.hpp new file mode 100644 index 000000000..7424c8e0e --- /dev/null +++ b/R/unifrac_cpp/unifrac_task.hpp @@ -0,0 +1,577 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include "task_parameters.hpp" +#include +#include +#include +#include +#include +#include + + +#ifndef __UNIFRAC_TASKS +#define __UNIFRAC_TASKS 1 + +#ifdef _OPENACC + +#define SUCMP_NM su_acc + + #ifndef SMALLGPU + // defaultt on larger alignment, which improves performance on GPUs like V100 +#define UNIFRAC_BLOCK 64 + #else + // smaller GPUs prefer smaller allignment +#define UNIFRAC_BLOCK 32 + #endif + +#else + +#define SUCMP_NM su_cpu + + +// CPUs don't need such a big alignment +#define UNIFRAC_BLOCK 16 +#endif + +namespace SUCMP_NM { + + // Note: This adds a copy, which is suboptimal + // But was the easiest way to get a contiguous buffer + // And it does allow for fp32 compute, when desired + template + class UnifracTaskVector { + private: + std::vector &dm_stripes; + const su::task_parameters* const task_p; + + public: + const unsigned int start_idx; + const unsigned int n_samples; + const uint64_t n_samples_r; + TFloat* const buf; + + UnifracTaskVector(std::vector &_dm_stripes, const su::task_parameters* _task_p) + : dm_stripes(_dm_stripes), task_p(_task_p) + , start_idx(task_p->start), n_samples(task_p->n_samples) + , n_samples_r(((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK) // round up + , buf((dm_stripes[start_idx]==NULL) ? NULL : new TFloat[n_samples_r*(task_p->stop-start_idx)]) // dm_stripes could be null, in which case keep it null + { + TFloat* const ibuf = buf; + if (ibuf != NULL) { +#ifdef _OPENACC + const uint64_t bufels = n_samples_r * (task_p->stop-start_idx); +#endif + for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { + double * dm_stripe = dm_stripes[stripe]; + TFloat * buf_stripe = this->operator[](stripe); + for(unsigned int j=0; jstop-start_idx); +#pragma acc exit data copyout(ibuf[:bufels]) +#endif + for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { + double * dm_stripe = dm_stripes[stripe]; + TFloat * buf_stripe = this->operator[](stripe); + for(unsigned int j=0; j + class UnifracTaskBase { + public: + UnifracTaskVector dm_stripes; + UnifracTaskVector dm_stripes_total; + + const su::task_parameters* task_p; + + const unsigned int max_embs; + TEmb * embedded_proportions; +#ifdef _OPENACC + protected: + // alternate buffer only needed in async environments, like openacc + TEmb * embedded_proportions_alt; // used as temp + public: +#endif + + UnifracTaskBase(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) + : dm_stripes(_dm_stripes,_task_p), dm_stripes_total(_dm_stripes_total,_task_p), task_p(_task_p) + , max_embs(_max_embs) + , embedded_proportions(initialize_embedded(dm_stripes.n_samples_r,_max_embs)) +#ifdef _OPENACC + , embedded_proportions_alt(initialize_embedded(dm_stripes.n_samples_r,_max_embs)) +#endif + {} + + /* remove + // Note: not const, since they share a mutable state + UnifracTaskBase(UnifracTaskBase &baseObj) + : dm_stripes(baseObj.dm_stripes), dm_stripes_total(baseObj.dm_stripes_total), task_p(baseObj.task_p) {} + */ + + virtual ~UnifracTaskBase() + { +#ifdef _OPENACC + const uint64_t n_samples_r = dm_stripes.n_samples_r; + const uint64_t bsize = n_samples_r * get_emb_els(max_embs); +#pragma acc exit data delete(embedded_proportions_alt[:bsize]) +#pragma acc exit data delete(embedded_proportions[:bsize]) + free(embedded_proportions_alt); +#endif + free(embedded_proportions); + } + + void sync_embedded_proportions(unsigned int filled_embs) + { +#ifdef _OPENACC + const uint64_t n_samples_r = dm_stripes.n_samples_r; + const uint64_t bsize = n_samples_r * get_emb_els(filled_embs); +#pragma acc update device(embedded_proportions[:bsize]) +#endif + } + + static unsigned int get_emb_els(unsigned int max_embs); + + static TEmb *initialize_embedded(const uint64_t n_samples_r, unsigned int max_embs) { + uint64_t bsize = n_samples_r * get_emb_els(max_embs); + + TEmb* buf = NULL; + int err = posix_memalign((void **)&buf, 4096, sizeof(TEmb) * bsize); + if(buf == NULL || err != 0) { + fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", + sizeof(TEmb) * bsize, err, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } +#pragma acc enter data create(buf[:bsize]) + return buf; + } + + void embed_proportions_range(const TFloat* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb); + void embed_proportions(const TFloat* __restrict__ in, unsigned int emb) {embed_proportions_range(in,0,dm_stripes.n_samples,emb);} + + + + // + // ===== Internal, do not use directly ======= + // + + + // Just copy from one buffer to another + // May convert between fp formats in the process (if TOut!=double) + template void embed_proportions_range_straight(TOut* __restrict__ out, const TFloat* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) const + { + const unsigned int n_samples = dm_stripes.n_samples; + const uint64_t n_samples_r = dm_stripes.n_samples_r; + const uint64_t offset = emb * n_samples_r; + + for(unsigned int i = start; i < end; i++) { + out[offset + i] = in[i-start]; + } + + if (end==n_samples) { + // avoid NaNs + for(unsigned int i = n_samples; i < n_samples_r; i++) { + out[offset + i] = 0.0; + } + } + } + + // packed bool + // Compute (in[:]>0) on each element, and store only the boolean bit. + // The output values are stored in a multi-byte format, one bit per emb index, + // so it will likely take multiple passes to store all the values + // + // Note: assumes we are processing emb in increasing order, starting from 0 + template void embed_proportions_range_bool(TOut* __restrict__ out, const TFloat* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) const + { + const unsigned int n_packed = sizeof(TOut)*8;// e.g. 32 for unit32_t + const unsigned int n_samples = dm_stripes.n_samples; + const uint64_t n_samples_r = dm_stripes.n_samples_r; + // The output values are stored in a multi-byte format, one bit per emb index + // Compute the element to store the bit into, as well as whichbit in that element + unsigned int emb_block = emb/n_packed; // beginning of the element block + unsigned int emb_bit = emb%n_packed; // bit inside the elements + const uint64_t offset = emb_block * n_samples_r; + + if (emb_bit==0) { + // assign for emb_bit==0, so it clears the other bits + // assumes we processing emb in increasing order, starting from 0 + for(unsigned int i = start; i < end; i++) { + out[offset + i] = (in[i-start] > 0); + } + + if (end==n_samples) { + // avoid NaNs + for(unsigned int i = n_samples; i < n_samples_r; i++) { + out[offset + i] = 0; + } + } + } else { + // just update my bit + for(unsigned int i = start; i < end; i++) { + out[offset + i] |= (TOut(in[i-start] > 0) << emb_bit); + } + + // the rest of the els are already OK + } + } + }; + + // straight embeded_proportions + template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_straight(embedded_proportions,in,start,end,emb);} + template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_straight(embedded_proportions,in,start,end,emb);} + template<> inline void UnifracTaskBase::embed_proportions_range(const float* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_straight(embedded_proportions,in,start,end,emb);} + template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return max_embs;} + template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return max_embs;} + template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return max_embs;} + + //packed bool embeded_proportions + template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} + template<> inline void UnifracTaskBase::embed_proportions_range(const float* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} + template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+31)/32;} + template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+31)/32;} + + template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} + template<> inline void UnifracTaskBase::embed_proportions_range(const float* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} + template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+63)/64;} + template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+63)/64;} + + /* void unifrac tasks + * + * all methods utilize the same function signature. that signature is as follows: + * + * dm_stripes vector the stripes of the distance matrix being accumulated + * into for unique branch length + * dm_stripes vector the stripes of the distance matrix being accumulated + * into for total branch length (e.g., to normalize unweighted unifrac) + * embedded_proportions the proportions vector for a sample, or rather + * the counts vector normalized to 1. this vector is embedded as it is + * duplicated: if A, B and C are proportions for features A, B, and C, the + * vector will look like [A B C A B C]. + * length the branch length of the current node to its parent. + * task_p task specific parameters. + */ + + template + class UnifracTask : public UnifracTaskBase { + protected: + // Use one cache line on CPU + // On GPU, shaing a cache line is actually a good thing + static const unsigned int step_size = 16*4/sizeof(TFloat); + +#ifdef _OPENACC + // Use as big vector size as we can, to maximize cache line reuse + static const unsigned int acc_vector_size = 2048; +#endif + + public: + + UnifracTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) + : UnifracTaskBase(_dm_stripes, _dm_stripes_total, _max_embs, _task_p) {} + + /* delete + UnifracTask(UnifracTaskBase &baseObj, const TEmb * _embedded_proportions, unsigned int _max_embs) + : UnifracTaskBase(baseObj) + , embedded_proportions(_embedded_proportions), max_embs(_max_embs) {} + */ + + + virtual ~UnifracTask() {} + + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) = 0; + + protected: + static const unsigned int RECOMMENDED_MAX_EMBS_STRAIGHT = 128-16; // a little less to leave a bit of space of maxed-out L1 + // packed uses 32x less memory,so this should be 32x larger than straight... but there are additional structures, so use half of that + static const unsigned int RECOMMENDED_MAX_EMBS_BOOL = 64*32; + + }; + + + template + class UnifracUnnormalizedWeightedTask : public UnifracTask { + public: + static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; + + UnifracUnnormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) + : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) + { + const unsigned int n_samples = this->task_p->n_samples; + + zcheck = NULL; + sums = NULL; + posix_memalign((void **)&zcheck, 4096, sizeof(bool) * n_samples); + posix_memalign((void **)&sums , 4096, sizeof(TFloat) * n_samples); +#pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) + } + + virtual ~UnifracUnnormalizedWeightedTask() + { +#ifdef _OPENACC + const unsigned int n_samples = this->task_p->n_samples; +#pragma acc exit data delete(sums[:n_samples],zcheck[:n_samples]) +#endif + free(sums); + free(zcheck); + } + + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} + + void _run(unsigned int filled_embs, const TFloat * __restrict__ length); + protected: + // temp buffers + bool *zcheck; + TFloat *sums; + }; + template + class UnifracNormalizedWeightedTask : public UnifracTask { + public: + static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; + + UnifracNormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) + : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) + { + const unsigned int n_samples = this->task_p->n_samples; + + zcheck = NULL; + sums = NULL; + posix_memalign((void **)&zcheck, 4096, sizeof(bool) * n_samples); + posix_memalign((void **)&sums , 4096, sizeof(TFloat) * n_samples); +#pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) + } + + virtual ~UnifracNormalizedWeightedTask() + { +#ifdef _OPENACC + const unsigned int n_samples = this->task_p->n_samples; +#pragma acc exit data delete(sums[:n_samples],zcheck[:n_samples]) +#endif + free(sums); + free(zcheck); + } + + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} + + void _run(unsigned int filled_embs, const TFloat * __restrict__ length); + protected: + // temp buffers + bool *zcheck; + TFloat *sums; + }; + template + class UnifracUnweightedTask : public UnifracTask { + public: + static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_BOOL; + + // Note: _max_emb MUST be multiple of 64 + UnifracUnweightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) + : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) + { + const unsigned int bsize = _max_embs*(0x400/32); + sums = NULL; + posix_memalign((void **)&sums, 4096, sizeof(TFloat) * bsize); +#pragma acc enter data create(sums[:bsize]) + } + + virtual ~UnifracUnweightedTask() + { +#ifdef _OPENACC + const unsigned int bsize = this->max_embs*(0x400/32); +#pragma acc exit data delete(sums[:bsize]) +#endif + free(sums); + } + + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} + + void _run(unsigned int filled_embs, const TFloat * __restrict__ length); + private: + TFloat *sums; // temp buffer + }; + template + class UnifracGeneralizedTask : public UnifracTask { + public: + static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; + + UnifracGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) + : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) {} + + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} + + void _run(unsigned int filled_embs, const TFloat * __restrict__ length); + }; + + /* void unifrac_vaw tasks + * + * all methods utilize the same function signature. that signature is as follows: + * + * dm_stripes vector the stripes of the distance matrix being accumulated + * into for unique branch length + * dm_stripes vector the stripes of the distance matrix being accumulated + * into for total branch length (e.g., to normalize unweighted unifrac) + * embedded_proportions the proportions vector for a sample, or rather + * the counts vector normalized to 1. this vector is embedded as it is + * duplicated: if A, B and C are proportions for features A, B, and C, the + * vector will look like [A B C A B C]. + * embedded_counts the counts vector embedded in the same way and order as + * embedded_proportions. the values of this array are unnormalized feature + * counts for the subtree. + * sample_total_counts the total unnormalized feature counts for all samples + * embedded in the same way and order as embedded_proportions. + * length the branch length of the current node to its parent. + * task_p task specific parameters. + */ + template + class UnifracVawTask : public UnifracTaskBase { + protected: +#ifdef _OPENACC + // The parallel nature of GPUs needs a largish step + #ifndef SMALLGPU + // default to larger step, which makes a big difference for bigger GPUs like V100 + static const unsigned int step_size = 32; + // keep the vector size just big enough to keep the used emb array inside the 32k buffer + static const unsigned int acc_vector_size = 32*32*8/sizeof(TFloat); + #else + // smaller GPUs prefer a slightly smaller step + static const unsigned int step_size = 16; + // keep the vector size just big enough to keep the used emb array inside the 32k buffer + static const unsigned int acc_vector_size = 32*32*8/sizeof(TFloat); + #endif +#else + // The serial nature of CPU cores prefers a small step + static const unsigned int step_size = 4; +#endif + + public: + TFloat * const embedded_counts; + const TFloat * const sample_total_counts; + + static const unsigned int RECOMMENDED_MAX_EMBS = 128; + + UnifracVawTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, + const TFloat * _sample_total_counts, + unsigned int _max_embs, const su::task_parameters* _task_p) + : UnifracTaskBase(_dm_stripes, _dm_stripes_total, _max_embs, _task_p) + , embedded_counts(UnifracTaskBase::initialize_embedded(this->dm_stripes.n_samples_r,_max_embs)), sample_total_counts(_sample_total_counts) {} + + + /* delete + UnifracVawTask(UnifracTaskBase &baseObj, + const TEmb * _embedded_proportions, const TFloat * _sample_total_counts, unsigned int _max_embs) + : UnifracTaskBase(baseObj) + , embedded_proportions(_embedded_proportions), embedded_counts(initialize_embedded()), sample_total_counts(_sample_total_counts), max_embs(_max_embs) {} + */ + + + virtual ~UnifracVawTask() {} + + void sync_embedded_counts(unsigned int filled_embs) + { +#ifdef _OPENACC + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + const uint64_t bsize = n_samples_r * filled_embs; +#pragma acc update device(embedded_counts[:bsize]) +#endif + } + + void sync_embedded(unsigned int filled_embs) { this->sync_embedded_proportions(filled_embs); this->sync_embedded_counts(filled_embs);} + + void embed_range(const TFloat* __restrict__ in_proportions, const TFloat* __restrict__ in_counts, unsigned int start, unsigned int end, unsigned int emb) { + this->embed_proportions_range(in_proportions,start,end,emb); + this->embed_proportions_range_straight(this->embedded_counts,in_counts,start,end,emb); + } + void embed(const TFloat* __restrict__ in_proportions, const double* __restrict__ in_counts, unsigned int emb) { embed_range(in_proportions,in_counts,0,this->dm_stripes.n_samples,emb);} + + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) = 0; + }; + + template + class UnifracVawUnnormalizedWeightedTask : public UnifracVawTask { + public: + UnifracVawUnnormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, + const TFloat * _sample_total_counts, + unsigned int _max_embs, const su::task_parameters* _task_p) + : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} + + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} + + void _run(unsigned int filled_embs, const TFloat * __restrict__ length); + }; + template + class UnifracVawNormalizedWeightedTask : public UnifracVawTask { + public: + UnifracVawNormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, + const TFloat * _sample_total_counts, + unsigned int _max_embs, const su::task_parameters* _task_p) + : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} + + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} + + void _run(unsigned int filled_embs, const TFloat * __restrict__ length); + }; + template + class UnifracVawUnweightedTask : public UnifracVawTask { + public: + UnifracVawUnweightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, + const TFloat * _sample_total_counts, + unsigned int _max_embs, const su::task_parameters* _task_p) + : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} + + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} + + void _run(unsigned int filled_embs, const TFloat * __restrict__ length); + }; + template + class UnifracVawGeneralizedTask : public UnifracVawTask { + public: + UnifracVawGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, + const TFloat * _sample_total_counts, + unsigned int _max_embs, const su::task_parameters* _task_p) + : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} + + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} + + void _run(unsigned int filled_embs, const TFloat * __restrict__ length); + }; + +} + +#endif From 6b2d218e4dee8010453ac4727c6e087d2bc53622 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Thu, 21 Nov 2024 11:23:18 +0200 Subject: [PATCH 02/48] Copy faith-related code into separate files --- R/unifrac_cpp/R_interface/rapi_test.R | 34 +- R/unifrac_cpp/api.cpp | 54 ++- R/unifrac_cpp/api_s.cpp | 84 +++++ R/unifrac_cpp/api_s.hpp | 42 +++ R/unifrac_cpp/biom_interface_s.hpp | 72 ++++ R/unifrac_cpp/biom_s.cpp | 324 ++++++++++++++++ R/unifrac_cpp/biom_s.hpp | 114 ++++++ R/unifrac_cpp/su_R_s.cpp | 125 +++++++ R/unifrac_cpp/test_api.cpp | 1 + R/unifrac_cpp/tree_s.cpp | 511 ++++++++++++++++++++++++++ R/unifrac_cpp/tree_s.hpp | 150 ++++++++ R/unifrac_cpp/unifrac_internal_s.cpp | 150 ++++++++ R/unifrac_cpp/unifrac_internal_s.hpp | 44 +++ R/unifrac_cpp/unifrac_s.cpp | 71 ++++ R/unifrac_cpp/unifrac_s.hpp | 30 ++ 15 files changed, 1773 insertions(+), 33 deletions(-) create mode 100644 R/unifrac_cpp/api_s.cpp create mode 100644 R/unifrac_cpp/api_s.hpp create mode 100644 R/unifrac_cpp/biom_interface_s.hpp create mode 100644 R/unifrac_cpp/biom_s.cpp create mode 100644 R/unifrac_cpp/biom_s.hpp create mode 100644 R/unifrac_cpp/su_R_s.cpp create mode 100644 R/unifrac_cpp/tree_s.cpp create mode 100644 R/unifrac_cpp/tree_s.hpp create mode 100644 R/unifrac_cpp/unifrac_internal_s.cpp create mode 100644 R/unifrac_cpp/unifrac_internal_s.hpp create mode 100644 R/unifrac_cpp/unifrac_s.cpp create mode 100644 R/unifrac_cpp/unifrac_s.hpp diff --git a/R/unifrac_cpp/R_interface/rapi_test.R b/R/unifrac_cpp/R_interface/rapi_test.R index 97dde62b7..4d1b0662f 100644 --- a/R/unifrac_cpp/R_interface/rapi_test.R +++ b/R/unifrac_cpp/R_interface/rapi_test.R @@ -1,4 +1,8 @@ library(Rcpp) +library(mia) +library(biomformat) +library(ape) +library(rhdf5) equals <- function(x, y, msg){ if (x!=y) @@ -9,35 +13,35 @@ equals <- function(x, y, msg){ aboutEquals <- function(x, y, msg){ if((x-y)>0.005) stop(msg) - - } + source = "R/unifrac_cpp/su_R.cpp" sourceCpp(source) table = "test.biom" tree = "test.tre" nthreads = 1 -print('Testing UniFrac..') -unif = unifrac(table, tree, nthreads) +b <- read_hdf5_biom("R/unifrac_cpp/R_interface/test.biom") +outfile <- tempfile() +write_biom(b, outfile) +bb <- read_biom(outfile) -exp = c(0.2000000, 0.5714286, 0.6000000, 0.5000000, 0.2000000, - 0.4285714, 0.6666667, 0.6000000, 0.3333333, 0.7142857, - 0.8571429, 0.4285714, 0.3333333, 0.4000000, 0.6000000) +fname <- "R/unifrac_cpp/R_interface/test.tre" +newick <- readChar(fname, file.info(fname)$size) +z <- treetest(newick) -equals(unif["n_samples"][[1]], 6, "n_samples != 6") -equals(unif["cf_size"][[1]], 15, "cf_size != 15") -equals(unif["is_upper_triangle"][[1]], TRUE, "is_upper_triagnle != TRUE") +tree <- ape::read.tree("R/unifrac_cpp/R_interface/test.tre") +treese <- makeTreeSEFromBiom(bb, treefilename=tree) +treese2 <- changeTree(treese, rowTree = tree) -for ( i in 1:15){ - aboutEquals(unif["c_form"][[1]][i], exp[i], "Output not as expected") -} -print('Success.') +test <- tempfile() +rowTree(treese2) print('Testing Faith PD..') -faith = faith_pd(table, tree) +#faith = faith_pd(table, tree) +faith <- faith_pd_new(treese2, newick) exp = c(4, 5, 6, 3, 2, 5) diff --git a/R/unifrac_cpp/api.cpp b/R/unifrac_cpp/api.cpp index d6b1995dc..fec69a268 100644 --- a/R/unifrac_cpp/api.cpp +++ b/R/unifrac_cpp/api.cpp @@ -11,8 +11,16 @@ #include #include -#include -#include + +/* Platform-specific memory management headers - for windows, we need to add #if defined(_WIN32) and rewrite the mmap portions (possibly with memoryapi.h?)*/ +#ifdef __linux__ +#include +#elif _WIN32 +#include +#endif + +/* Fast compression algorithm */ +#include #define MMAP_FD_MASK 0x0fff #define MMAP_FLAG 0x1000 @@ -167,20 +175,30 @@ void initialize_mat_full_no_biom_T(TMat* &result, const char* const * sample_ids } else { std::string mmap_template(mmap_dir); mmap_template+="/su_mmap_XXXXXX"; - // note: mkostemp will update mmap_template in place - int fd=mkostemp((char *) mmap_template.c_str(), O_NOATIME ); - if (fd<0) { - result->matrix = NULL; - // leave error handling to the caller - } else { - // remove the file name, so it will be destroyed on close - unlink(mmap_template.c_str()); - // make it big enough - ftruncate(fd,msize); - // now can be used, just like a malloc-ed buffer - result->matrix = (TReal*)mmap(NULL, msize,PROT_READ|PROT_WRITE, MAP_SHARED|MAP_NORESERVE, fd, 0); - result->flags=(uint32_t(fd) & MMAP_FD_MASK) | MMAP_FLAG; - } + + #ifdef __linux__ // Linux-specific memory handling + + // note: mkostemp will update mmap_template in place + int fd=mkostemp((char *) mmap_template.c_str(), O_NOATIME ); // replace for windows + + if (fd<0) { + result->matrix = NULL; + // leave error handling to the caller + } else { + // remove the file name, so it will be destroyed on close + unlink(mmap_template.c_str()); + // make it big enough + ftruncate(fd,msize); + // now can be used, just like a malloc-ed buffer + result->matrix = (TReal*)mmap(NULL, msize,PROT_READ|PROT_WRITE, MAP_SHARED|MAP_NORESERVE, fd, 0); // replace for windows + result->flags=(uint32_t(fd) & MMAP_FD_MASK) | MMAP_FLAG; + } + + #elif _WIN32 // Windows-specific memory handling + + + + #endif } for(unsigned int i = 0; i < n_samples; i++) { @@ -244,7 +262,7 @@ inline void destroy_mat_full_T(TMat** result) { free((*result)->matrix); } else { uint64_t n_samples = (*result)->n_samples; - munmap((*result)->matrix, sizeof(TReal)*n_samples*n_samples); + munmap((*result)->matrix, sizeof(TReal)*n_samples*n_samples); // replace for windows int fd = (*result)->flags & MMAP_FD_MASK; close(fd); @@ -902,7 +920,7 @@ IOStatus write_vec(const char* output_filename, r_vec* result) { } IOStatus write_partial(const char* output_filename, const partial_mat_t* result) { - int fd = open(output_filename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR ); + int fd = open(output_filename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR ); // replace for windows if (fd==-1) return write_error; int cnt = -1; diff --git a/R/unifrac_cpp/api_s.cpp b/R/unifrac_cpp/api_s.cpp new file mode 100644 index 000000000..f09781c1a --- /dev/null +++ b/R/unifrac_cpp/api_s.cpp @@ -0,0 +1,84 @@ +#include "api_s.hpp" +#include "biom_s.hpp" +#include "tree_s.hpp" +#include "unifrac_s.hpp" +#include +#include +#include +#include +#include + +#include +#include + +#include + +using namespace su; +using namespace std; + +// https://stackoverflow.com/a/19841704/19741 +bool is_file_exists(const char *fileName) { + std::ifstream infile(fileName); + return infile.good(); +} + +void initialize_results_vec(r_vec* &result, biom& table){ + // Stores results for Faith PD + result = (r_vec*)malloc(sizeof(results_vec)); + result->n_samples = table.n_samples; + result->values = (double*)malloc(sizeof(double) * result->n_samples); + result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); + + for(unsigned int i = 0; i < result->n_samples; i++) { + size_t len = table.sample_ids[i].length(); + result->sample_ids[i] = (char*)malloc(sizeof(char) * len + 1); + table.sample_ids[i].copy(result->sample_ids[i], len); + result->sample_ids[i][len] = '\0'; + result->values[i] = 0; + } + +} + +void destroy_results_vec(r_vec** result) { + // for Faith PD + for(unsigned int i = 0; i < (*result)->n_samples; i++) { + free((*result)->sample_ids[i]); + }; + free((*result)->sample_ids); + free((*result)->values); + free(*result); +} + + +/* +#define PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) std::ifstream ifs(tree_filename); \ + std::string content = std::string(std::istreambuf_iterator(ifs), \ + std::istreambuf_iterator()); \ + su::BPTree tree = su::BPTree(content); \ + su::biom table = su::biom(biom_filename); \ + if(table.n_samples <= 0 | table.n_obs <= 0) { \ + return table_empty; \ + } \ + std::string bad_id = su::test_table_ids_are_subset_of_tree(table, tree); \ + if(bad_id != "") { \ + return table_and_tree_do_not_overlap; \ + } \ + std::unordered_set to_keep(table.obs_ids.begin(), \ + table.obs_ids.end()); \ + su::BPTree tree_sheared = tree.shear(to_keep).collapse(); \ + */ + +compute_status faith_pd_one_off(const Rcpp::S4 & treeSE, r_vec** result, std::string newick){ + + // Check that tree and table are non-empty and match before calling the c++ code + // shear the tree (to contain only the obs in the table?) - Also should be done before the call? + + su::BPTree tree = su::BPTree(newick); + + initialize_results_vec(*result, table); + + // compute faithpd + su::faith_pd(table, tree_sheared, std::ref((*result)->values)); + + return okay; +} \ No newline at end of file diff --git a/R/unifrac_cpp/api_s.hpp b/R/unifrac_cpp/api_s.hpp new file mode 100644 index 000000000..34c81afe1 --- /dev/null +++ b/R/unifrac_cpp/api_s.hpp @@ -0,0 +1,42 @@ +#include "task_parameters.hpp" + +#ifdef __cplusplus +#include +#include +#define EXTERN extern "C" + +#else +#include +#define EXTERN +#endif + +typedef enum compute_status {okay=0, tree_missing, table_missing, table_empty, unknown_method, table_and_tree_do_not_overlap, output_error} ComputeStatus; + +/* a result vector + * + * n_samples the number of samples. + * values the score values of length n_samples. + * sample_ids the sample IDs of length n_samples. + */ +typedef struct results_vec{ + unsigned int n_samples; + double* values; + char** sample_ids; +} r_vec; + +void destroy_results_vec(r_vec** result); + +/* compute Faith PD + * biom_filename the filename to the biom table. + * tree_filename the filename to the correspodning tree. + * result the resulting vector of computed Faith PD values + * + * faith_pd_one_off returns the following error codes: + * + * okay : no problems encountered + * table_missing : the filename for the table does not exist + * tree_missing : the filename for the tree does not exist + * table_empty : the table does not have any entries + */ +EXTERN ComputeStatus faith_pd_one_off(const Rcpp::S4 & treeSE, + r_vec** result, std::string newick); diff --git a/R/unifrac_cpp/biom_interface_s.hpp b/R/unifrac_cpp/biom_interface_s.hpp new file mode 100644 index 000000000..bbfc80e4d --- /dev/null +++ b/R/unifrac_cpp/biom_interface_s.hpp @@ -0,0 +1,72 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2021-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + + +#ifndef _UNIFRAC_BIOM_INTERFACE_H +#define _UNIFRAC_BIOM_INTERFACE_H + +#include +#include + +namespace su { + class biom_interface { + public: + // cache the IDs contained within the table + std::vector sample_ids; + std::vector obs_ids; + + // cache both index pointers into both CSC and CSR representations + std::vector sample_indptr; + std::vector obs_indptr; + + uint32_t n_samples; // the number of samples + uint32_t n_obs; // the number of observations + uint32_t nnz; // the total number of nonzero entries + double *sample_counts; + + /* default constructor + * + * Automatically create the needed objects. + * All other initialization happens in children constructors. + */ + biom_interface() {} + + /* default destructor + * + * Automatically destroy the objects. + * All other cleanup must have been performed by the children constructors. + */ + virtual ~biom_interface() {} + + /* get a dense vector of observation data + * + * @param id The observation ID to fetch + * @param out An allocated array of at least size n_samples. + * Values of an index position [0, n_samples) which do not + * have data will be zero'd. + */ + virtual void get_obs_data(const std::string &id, double* out) const = 0; + virtual void get_obs_data(const std::string &id, float* out) const = 0; + + /* get a dense vector of a range of observation data + * + * @param id The observation ID to fetc + * @param start Initial index + * @param end First index past the end + * @param normalize If set, divide by sample_counts + * @param out An allocated array of at least size (end-start). First element will corrrectpoint to index start. + * Values of an index position [0, (end-start)) which do not + * have data will be zero'd. + */ + virtual void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, double* out) const = 0; + virtual void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, float* out) const = 0; + }; +} + +#endif /* _UNIFRAC_BIOOM_INTERFACE_H */ diff --git a/R/unifrac_cpp/biom_s.cpp b/R/unifrac_cpp/biom_s.cpp new file mode 100644 index 000000000..3e4017992 --- /dev/null +++ b/R/unifrac_cpp/biom_s.cpp @@ -0,0 +1,324 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include +#include +#include +#include "biom_s.hpp" + +using namespace H5; +using namespace su; + +/* datasets defined by the BIOM 2.x spec */ +const std::string OBS_INDPTR = std::string("/observation/matrix/indptr"); +const std::string OBS_INDICES = std::string("/observation/matrix/indices"); +const std::string OBS_DATA = std::string("/observation/matrix/data"); +const std::string OBS_IDS = std::string("/observation/ids"); + +const std::string SAMPLE_INDPTR = std::string("/sample/matrix/indptr"); +const std::string SAMPLE_INDICES = std::string("/sample/matrix/indices"); +const std::string SAMPLE_DATA = std::string("/sample/matrix/data"); +const std::string SAMPLE_IDS = std::string("/sample/ids"); + +biom::biom(std::string filename) { + file = H5File(filename.c_str(), H5F_ACC_RDONLY); + + /* establish the datasets */ + obs_indices = file.openDataSet(OBS_INDICES.c_str()); + obs_data = file.openDataSet(OBS_DATA.c_str()); + sample_indices = file.openDataSet(SAMPLE_INDICES.c_str()); + sample_data = file.openDataSet(SAMPLE_DATA.c_str()); + + /* cache IDs and indptr */ + sample_ids = std::vector(); + obs_ids = std::vector(); + sample_indptr = std::vector(); + obs_indptr = std::vector(); + + load_ids(OBS_IDS.c_str(), obs_ids); + load_ids(SAMPLE_IDS.c_str(), sample_ids); + load_indptr(OBS_INDPTR.c_str(), obs_indptr); + load_indptr(SAMPLE_INDPTR.c_str(), sample_indptr); + + /* cache shape and nnz info */ + n_samples = sample_ids.size(); + n_obs = obs_ids.size(); + set_nnz(); + + /* define a mapping between an ID and its corresponding offset */ + obs_id_index = std::unordered_map(); + sample_id_index = std::unordered_map(); + + create_id_index(obs_ids, obs_id_index); + create_id_index(sample_ids, sample_id_index); + + /* load obs sparse data */ + obs_indices_resident = (uint32_t**)malloc(sizeof(uint32_t**) * n_obs); + if(obs_indices_resident == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(uint32_t**) * n_obs, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + obs_data_resident = (double**)malloc(sizeof(double**) * n_obs); + if(obs_data_resident == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(double**) * n_obs, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + obs_counts_resident = (unsigned int*)malloc(sizeof(unsigned int) * n_obs); + if(obs_counts_resident == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(unsigned int) * n_obs, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + + uint32_t *current_indices = NULL; + double *current_data = NULL; + for(unsigned int i = 0; i < obs_ids.size(); i++) { + std::string id_ = obs_ids[i]; + unsigned int n = get_obs_data_direct(id_, current_indices, current_data); + obs_counts_resident[i] = n; + obs_indices_resident[i] = current_indices; + obs_data_resident[i] = current_data; + } + sample_counts = get_sample_counts(); +} + +biom::~biom() { + for(unsigned int i = 0; i < n_obs; i++) { + free(obs_indices_resident[i]); + free(obs_data_resident[i]); + } + free(obs_indices_resident); + free(obs_data_resident); + free(obs_counts_resident); +} + +void biom::set_nnz() { + // should these be cached? + DataType dtype = obs_data.getDataType(); + DataSpace dataspace = obs_data.getSpace(); + + hsize_t dims[1]; + dataspace.getSimpleExtentDims(dims, NULL); + nnz = dims[0]; +} + +void biom::load_ids(const char *path, std::vector &ids) { + DataSet ds_ids = file.openDataSet(path); + DataType dtype = ds_ids.getDataType(); + DataSpace dataspace = ds_ids.getSpace(); + + hsize_t dims[1]; + dataspace.getSimpleExtentDims(dims, NULL); + + /* the IDs are a dataset of variable length strings */ + char **dataout = (char**)malloc(sizeof(char*) * dims[0]); + if(dataout == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(char*) * dims[0], __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + ds_ids.read((void*)dataout, dtype); + + ids.reserve(dims[0]); + for(unsigned int i = 0; i < dims[0]; i++) { + ids.push_back(dataout[i]); + } + + for(unsigned int i = 0; i < dims[0]; i++) + free(dataout[i]); + free(dataout); +} + +void biom::load_indptr(const char *path, std::vector &indptr) { + DataSet ds = file.openDataSet(path); + DataType dtype = ds.getDataType(); + DataSpace dataspace = ds.getSpace(); + + hsize_t dims[1]; + dataspace.getSimpleExtentDims(dims, NULL); + + uint32_t *dataout = (uint32_t*)malloc(sizeof(uint32_t) * dims[0]); + if(dataout == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(uint32_t) * dims[0], __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + ds.read((void*)dataout, dtype); + + indptr.reserve(dims[0]); + for(unsigned int i = 0; i < dims[0]; i++) + indptr.push_back(dataout[i]); + free(dataout); +} + +void biom::create_id_index(std::vector &ids, + std::unordered_map &map) { + uint32_t count = 0; + map.reserve(ids.size()); + for(auto i = ids.begin(); i != ids.end(); i++, count++) { + map[*i] = count; + } +} + +unsigned int biom::get_obs_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out) { + uint32_t idx = obs_id_index.at(id); + uint32_t start = obs_indptr[idx]; + uint32_t end = obs_indptr[idx + 1]; + + hsize_t count[1] = {end - start}; + hsize_t offset[1] = {start}; + + DataType indices_dtype = obs_indices.getDataType(); + DataType data_dtype = obs_data.getDataType(); + + DataSpace indices_dataspace = obs_indices.getSpace(); + DataSpace data_dataspace = obs_data.getSpace(); + + DataSpace indices_memspace(1, count, NULL); + DataSpace data_memspace(1, count, NULL); + + indices_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); + data_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); + + current_indices_out = (uint32_t*)malloc(sizeof(uint32_t) * count[0]); + if(current_indices_out == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(uint32_t) * count[0], __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + current_data_out = (double*)malloc(sizeof(double) * count[0]); + if(current_data_out == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(double) * count[0], __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + + obs_indices.read((void*)current_indices_out, indices_dtype, indices_memspace, indices_dataspace); + obs_data.read((void*)current_data_out, data_dtype, data_memspace, data_dataspace); + + return count[0]; +} + +template +void biom::get_obs_data_TT(const std::string &id, TFloat* out) const { + uint32_t idx = obs_id_index.at(id); + unsigned int count = obs_counts_resident[idx]; + const uint32_t * const indices = obs_indices_resident[idx]; + const double * const data = obs_data_resident[idx]; + + // reset our output buffer + for(unsigned int i = 0; i < n_samples; i++) + out[i] = 0.0; + + for(unsigned int i = 0; i < count; i++) { + out[indices[i]] = data[i]; + } +} + +void biom::get_obs_data(const std::string &id, double* out) const { + biom::get_obs_data_TT(id,out); +} + +void biom::get_obs_data(const std::string &id, float* out) const { + biom::get_obs_data_TT(id,out); +} + + +// note: out is supposed to be fully filled, i.e. out[start:end] +template +void biom::get_obs_data_range_TT(const std::string &id, unsigned int start, unsigned int end, bool normalize, TFloat* out) const { + uint32_t idx = obs_id_index.at(id); + unsigned int count = obs_counts_resident[idx]; + const uint32_t * const indices = obs_indices_resident[idx]; + const double * const data = obs_data_resident[idx]; + + // reset our output buffer + for(unsigned int i = start; i < end; i++) + out[i-start] = 0.0; + + if (normalize) { + for(unsigned int i = 0; i < count; i++) { + const int32_t j = indices[i]; + if ((j>=start)&&(j=start)&&(j +#include +#include +#include + +#include "biom_interface_s.hpp" + +namespace su { + class biom : public biom_interface { + public: + /* default constructor + * + * @param filename The path to the BIOM table to read + */ + biom(std::string filename); + + /* default destructor + * + * Temporary arrays are freed + */ + virtual ~biom(); + + /* get a dense vector of observation data + * + * @param id The observation ID to fetch + * @param out An allocated array of at least size n_samples. + * Values of an index position [0, n_samples) which do not + * have data will be zero'd. + */ + void get_obs_data(const std::string &id, double* out) const; + void get_obs_data(const std::string &id, float* out) const; + + /* get a dense vector of a range of observation data + * + * @param id The observation ID to fetc + * @param start Initial index + * @param end First index past the end + * @param normalize If set, divide by sample_counts + * @param out An allocated array of at least size (end-start). First element will corrrectpoint to index start. + * Values of an index position [0, (end-start)) which do not + * have data will be zero'd. + */ + void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, double* out) const; + void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, float* out) const; + + private: + /* retain DataSet handles within the HDF5 file */ + H5::DataSet obs_indices; + H5::DataSet sample_indices; + H5::DataSet obs_data; + H5::DataSet sample_data; + H5::H5File file; + uint32_t **obs_indices_resident; + double **obs_data_resident; + unsigned int *obs_counts_resident; + + unsigned int get_obs_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out); + unsigned int get_sample_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out); + double* get_sample_counts(); + + /* At construction, lookups mapping IDs -> index position within an + * axis are defined + */ + std::unordered_map obs_id_index; + std::unordered_map sample_id_index; + + /* load ids from an axis + * + * @param path The dataset path to the ID dataset to load + * @param ids The variable representing the IDs to load into + */ + void load_ids(const char *path, std::vector &ids); + + /* load the index pointer for an axis + * + * @param path The dataset path to the index pointer to load + * @param indptr The vector to load the data into + */ + void load_indptr(const char *path, std::vector &indptr); + + /* count the number of nonzero values and set nnz */ + void set_nnz(); + + /* create an index mapping an ID to its corresponding index + * position. + * + * @param ids A vector of IDs to index + * @param map A hash table to populate + */ + void create_id_index(std::vector &ids, + std::unordered_map &map); + + + // templatized version + template void get_obs_data_TT(const std::string &id, TFloat* out) const; + template void get_obs_data_range_TT(const std::string &id, unsigned int start, unsigned int end, bool normalize, TFloat* out) const; + }; +} + +#endif /* _UNIFRAC_BIOM_H */ + diff --git a/R/unifrac_cpp/su_R_s.cpp b/R/unifrac_cpp/su_R_s.cpp new file mode 100644 index 000000000..7ff5458fd --- /dev/null +++ b/R/unifrac_cpp/su_R_s.cpp @@ -0,0 +1,125 @@ +#include +#include +#include +#include "api_s.hpp" +#include "tree_s.hpp" + +using namespace std; +using namespace Rcpp; + +/* +// [[Rcpp::export]] +Rcpp::List faith_pd(const char* table, const char* tree){ + r_vec* result = NULL; + ComputeStatus status; + status = faith_pd_one_off(table, tree, &result); + vector values; + for(int i = 0; i < result->n_samples; i++){ + values.push_back(result->values[i]); + } + + return Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, + Rcpp::Named("faith_pd") = values); + +} +*/ + +// [[Rcpp::export]] +Rcpp::List faith_pd_new(const Rcpp::S4 & treeSE, Rcpp::String tree){ + r_vec* result = NULL; + ComputeStatus status; + std::string newick(tree.get_cstring()); + status = faith_pd_one_off(treeSE, &result, tree); + vector values; + for(int i = 0; i < result->n_samples; i++){ + values.push_back(result->values[i]); + } + + Rcpp::List rlist = Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, + Rcpp::Named("faith_pd") = values); + + destroy_results_vec(result); + + return rlist; + + /* + Rcpp::List rowTree = treeSE.slot("rowTree"); + const List & phylo = rowTree["phylo"]; + const Rcpp::NumericMatrix & edge = phylo["edge"]; + + return rowTree; + */ +} + + +std::vector newick(Rcpp::String ins) { + std::string newick(ins.get_cstring()); + std::vector result = std::vector(); + char last_structure; + bool potential_single_descendent = false; + int count = 0; + bool in_quote = false; + for(auto c = newick.begin(); c != newick.end(); c++) { + if(*c == '\'') + in_quote = !in_quote; + + if(in_quote) + continue; + + switch(*c) { + case '(': + // opening of a node + count++; + result.push_back(true); + last_structure = *c; + potential_single_descendent = true; + break; + case ')': + // closing of a node + if(potential_single_descendent || (last_structure == ',')) { + // we have a single descendent or a last child (i.e. ",)" scenario) + count += 3; + result.push_back(true); + result.push_back(false); + result.push_back(false); + potential_single_descendent = false; + } else { + // it is possible still to have a single descendent in the case of + // multiple single descendents (e.g., (...()...) ) + count += 1; + result.push_back(false); + } + last_structure = *c; + break; + case ',': + if(last_structure != ')') { + // we have a new tip + count += 2; + result.push_back(true); + result.push_back(false); + } + potential_single_descendent = false; + last_structure = *c; + break; + default: + break; + } + } + return result; +} + +// [[Rcpp::export]] +Rcpp::LogicalVector treetest(std::string n){ + + Rcpp::LogicalVector result = Rcpp::LogicalVector(); + std::vector raw = newick(n); + if(raw.size() > 0) { + std::cout << raw.size(); + } + result = Rcpp::LogicalVector::import(raw.begin(), raw.end()); + return result; + +} + + + diff --git a/R/unifrac_cpp/test_api.cpp b/R/unifrac_cpp/test_api.cpp index ac87716cc..11304ae05 100644 --- a/R/unifrac_cpp/test_api.cpp +++ b/R/unifrac_cpp/test_api.cpp @@ -13,6 +13,7 @@ * test harness adapted from * https://github.com/noporpoise/BitArray/blob/master/dev/bit_array_test.c */ + const char *suite_name; char suite_pass; int suites_run = 0, suites_failed = 0, suites_empty = 0; diff --git a/R/unifrac_cpp/tree_s.cpp b/R/unifrac_cpp/tree_s.cpp new file mode 100644 index 000000000..eb7c7ad7f --- /dev/null +++ b/R/unifrac_cpp/tree_s.cpp @@ -0,0 +1,511 @@ +#include "tree_s.hpp" +#include +#include + +#include + +using namespace su; + +BPTree::BPTree(std::string newick) { + openclose = std::vector(); + lengths = std::vector(); + names = std::vector(); + excess = std::vector(); + + select_0_index = std::vector(); + select_1_index = std::vector(); + structure = std::vector(); + structure.reserve(500000); // a fair sized tree... avoid reallocs, and its not _that_ much waste if this is wrong + + // three pass for parse. not ideal, but easier to map from IOW code + newick_to_bp(newick); + + // resize is correct here as we are not performing a push_back + openclose.resize(nparens); + lengths.resize(nparens); + names.resize(nparens); + select_0_index.resize(nparens / 2); + select_1_index.resize(nparens / 2); + excess.resize(nparens); + + structure_to_openclose(); + newick_to_metadata(newick); + index_and_cache(); +} + +BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names) { + structure = input_structure; + lengths = input_lengths; + names = input_names; + + nparens = structure.size(); + + openclose = std::vector(); + select_0_index = std::vector(); + select_1_index = std::vector(); + openclose.resize(nparens); + select_0_index.resize(nparens / 2); + select_1_index.resize(nparens / 2); + excess.resize(nparens); + + structure_to_openclose(); + index_and_cache(); +} + +BPTree::BPTree(const Rcpp::S4 & treeSE) { + const Rcpp::S4 & rowTree = treeSE.slot("RowTree"); + structure = std::vector(); + structure.reserve(500000); // a fair sized tree... avoid reallocs, and its not _that_ much waste if this is wrong +} + + +BPTree BPTree::mask(std::vector topology_mask, std::vector in_lengths) { + + std::vector new_structure = std::vector(); + std::vector new_lengths = std::vector(); + std::vector new_names = std::vector(); + + uint32_t count = 0; + for(auto i = topology_mask.begin(); i != topology_mask.end(); i++) { + if(*i) + count++; + } + + new_structure.resize(count); + new_lengths.resize(count); + new_names.resize(count); + + auto mask_it = topology_mask.begin(); + auto base_it = this->structure.begin(); + uint32_t new_idx = 0; + uint32_t old_idx = 0; + for(; mask_it != topology_mask.end(); mask_it++, base_it++, old_idx++) { + if(*mask_it) { + new_structure[new_idx] = this->structure[old_idx]; + new_lengths[new_idx] = in_lengths[old_idx]; + new_names[new_idx] = this->names[old_idx]; + new_idx++; + } + } + + return BPTree(new_structure, new_lengths, new_names); +} + +std::unordered_set BPTree::get_tip_names() { + std::unordered_set observed; + + for(unsigned int i = 0; i < this->nparens; i++) { + if(this->isleaf(i)) { + observed.insert(this->names[i]); + } + } + + return observed; +} + +BPTree BPTree::shear(std::unordered_set to_keep) { + std::vector shearmask = std::vector(this->nparens); + int32_t p; + + for(unsigned int i = 0; i < this->nparens; i++) { + if(this->isleaf(i) && to_keep.count(this->names[i]) > 0) { + shearmask[i] = true; + shearmask[i+1] = true; + + p = this->parent(i); + while(p != -1 && !shearmask[p]) { + shearmask[p] = true; + shearmask[this->close(p)] = true; + p = this->parent(p); + } + } + } + return this->mask(shearmask, this->lengths); +} + +BPTree BPTree::collapse() { + std::vector collapsemask = std::vector(this->nparens); + std::vector new_lengths = std::vector(this->lengths); + + uint32_t current, first, last; + + for(uint32_t i = 0; i < this->nparens / 2; i++) { + current = this->preorderselect(i); + + if(this->isleaf(current) or (current == 0)) { // 0 == root + collapsemask[current] = true; + collapsemask[this->close(current)] = true; + } else { + first = this->leftchild(current); + last = this->rightchild(current); + + if(first == last) { + new_lengths[first] = new_lengths[first] + new_lengths[current]; + } else { + collapsemask[current] = true; + collapsemask[this->close(current)] = true; + } + } + } + + return this->mask(collapsemask, new_lengths); +} + /* + mask = bit_array_create(self.B.size) + bit_array_set_bit(mask, self.root()) + bit_array_set_bit(mask, self.close(self.root())) + + new_lengths = self._lengths.copy() + new_lengths_ptr = new_lengths.data + + with nogil: + for i in range(n): + current = self.preorderselect(i) + + if self.isleaf(current): + bit_array_set_bit(mask, current) + bit_array_set_bit(mask, self.close(current)) + else: + first = self.fchild(current) + last = self.lchild(current) + + if first == last: + new_lengths_ptr[first] = new_lengths_ptr[first] + \ + new_lengths_ptr[current] + else: + bit_array_set_bit(mask, current) + bit_array_set_bit(mask, self.close(current)) + + new_bp = self._mask_from_self(mask, new_lengths) + bit_array_free(mask) + return new_bp +*/ + + +BPTree::~BPTree() { +} + +void BPTree::index_and_cache() { + // should probably do the open/close in here too + unsigned int idx = 0; + auto i = structure.begin(); + auto k0 = select_0_index.begin(); + auto k1 = select_1_index.begin(); + auto e_it = excess.begin(); + unsigned int e = 0; + + for(; i != structure.end(); i++, idx++ ) { + if(*i) { + *(k1++) = idx; + *(e_it++) = ++e; + } + else { + *(k0++) = idx; + *(e_it++) = --e; + } + } +} + +uint32_t BPTree::postorderselect(uint32_t k) const { + return open(select_0_index[k]); +} + +uint32_t BPTree::preorderselect(uint32_t k) const { + return select_1_index[k]; +} + +inline uint32_t BPTree::open(uint32_t i) const { + return structure[i] ? i : openclose[i]; +} + +inline uint32_t BPTree::close(uint32_t i) const { + return structure[i] ? openclose[i] : i; +} + +bool BPTree::isleaf(unsigned int idx) const { + return (structure[idx] && !structure[idx + 1]); +} + +uint32_t BPTree::leftchild(uint32_t i) const { + // aka fchild + if(isleaf(i)) + return 0; // this is awkward, using 0 which is root, but a root cannot be a child. edge case + else + return i + 1; +} + +uint32_t BPTree::rightchild(uint32_t i) const { + // aka lchild + if(isleaf(i)) + return 0; // this is awkward, using 0 which is root, but a root cannot be a child. edge case + else + return open(close(i) - 1); +} + +uint32_t BPTree::rightsibling(uint32_t i) const { + // aka nsibling + uint32_t position = close(i) + 1; + if(position >= nparens) + return 0; // will return 0 if no sibling as root cannot have a sibling + else if(structure[position]) + return position; + else + return 0; +} + +int32_t BPTree::parent(uint32_t i) const { + return enclose(i); +} + +int32_t BPTree::enclose(uint32_t i) const { + if(structure[i]) + return bwd(i, -2) + 1; + else + return bwd(i - 1, -2) + 1; +} + +int32_t BPTree::bwd(uint32_t i, int d) const { + uint32_t target_excess = excess[i] + d; + for(int current_idx = i - 1; current_idx >= 0; current_idx--) { + if(excess[current_idx] == target_excess) + return current_idx; + } + return -1; +} + +void BPTree::rowTree_to_bp(const Rcpp::List & rowTree) { + Rcpp::List phylo = rowTree["phylo"]; + Rcpp::NumericMatrix edge = phylo["edge"]; + Rcpp::StringVector tips = phylo["tip.label"]; + uint32_t ntips = tips.size(); + std::stack nodes = std::stack(); // phylo tips are always numbered from 1 to number of tips; + std::vector levels = std::vector(ntips, 0); // Use this to keep track of how many descendants the current branch's node have + + int previousNode = 0; + int currentNode = 0; + int nextNode = 0; + int count = 0; + + bool potential_single_descendent = false; + + for (int i = 0; i < edge.nrow(); i++){ + currentNode = edge(i, 1); + if(nodes.size() == 0 || nodes.top() != currentNode ) { // Either were at the root, or entering a new node + if(nodes.top() < currentNode) { // We've exhausted the node and moved backwards in the tree + + } + nodes.push(currentNode); + count++; + structure.push_back(true); + potential_single_descendent = true; + break; + } + nextNode = edge(i, 2); + + if(nextNode <= ntips) { // Node is a tip + count += 2; + structure.push_back(true); + structure.push_back(false); + } + + previousNode = currentNode; + } + +} + +void BPTree::newick_to_bp(std::string newick) { + char last_structure; + bool potential_single_descendent = false; + int count = 0; + bool in_quote = false; + for(auto c = newick.begin(); c != newick.end(); c++) { + if(*c == '\'') + in_quote = !in_quote; + + if(in_quote) + continue; + + switch(*c) { + case '(': + // opening of a node + count++; + structure.push_back(true); + last_structure = *c; + potential_single_descendent = true; + break; + case ')': + // closing of a node + if(potential_single_descendent || (last_structure == ',')) { + // we have a single descendent or a last child (i.e. ",)" scenario) + count += 3; + structure.push_back(true); + structure.push_back(false); + structure.push_back(false); + potential_single_descendent = false; + } else { + // it is possible still to have a single descendent in the case of + // multiple single descendents (e.g., (...()...) ) + count += 1; + structure.push_back(false); + } + last_structure = *c; + break; + case ',': + if(last_structure != ')') { + // we have a new tip + count += 2; + structure.push_back(true); + structure.push_back(false); + } + potential_single_descendent = false; + last_structure = *c; + break; + default: + break; + } + } + nparens = structure.size(); +} + + +void BPTree::structure_to_openclose() { + std::stack oc; + unsigned int open_idx; + unsigned int i = 0; + + for(auto it = structure.begin(); it != structure.end(); it++, i++) { + if(*it) { + oc.push(i); + } else { + open_idx = oc.top(); + oc.pop(); + openclose[i] = open_idx; + openclose[open_idx] = i; + } + } +} +// trim from end +// from http://stackoverflow.com/a/217605 +static inline std::string &rtrim(std::string &s) { + s.erase(std::find_if(s.rbegin(), s.rend(), + std::not1(std::ptr_fun(std::isspace))).base(), s.end()); + return s; +} + + +//// WEIRDNESS. THIS SOLVES IT WITH THE RTRIM. ISOLATE, MOVE TO CONSTRUCTOR. +void BPTree::newick_to_metadata(std::string newick) { + newick = rtrim(newick); + + std::string::iterator start = newick.begin(); + std::string::iterator end = newick.end(); + std::string token; + char last_structure = '\0'; + + unsigned int structure_idx = 0; + unsigned int lag = 0; + unsigned int open_idx; + + while(start != end) { + token = tokenize(start, end); + // this sucks. + if(token.length() == 1 && is_structure_character(token[0])) { + switch(token[0]) { + case '(': + structure_idx++; + break; + case ')': + case ',': + structure_idx++; + if(last_structure == ')') + lag++; + break; + } + } else { + // puts us on the corresponding closing parenthesis + structure_idx += lag; + lag = 0; + + open_idx = open(structure_idx); + set_node_metadata(open_idx, token); + // std::cout << structure_idx << " <-> " << open_idx << " " << token << std::endl; + // make sure to advance an extra position if we are a leaf as the + // as a leaf is by definition a 10, and doing a single advancement + // would put the structure to token mapping out of sync + if(isleaf(open_idx)) + structure_idx += 2; + else + structure_idx += 1; + + } + last_structure = token[0]; + } +} + +void BPTree::set_node_metadata(unsigned int open_idx, std::string &token) { + double length = 0.0; + std::string name = std::string(); + unsigned int colon_idx = token.find_last_of(':'); + + if(colon_idx == 0) + length = std::stof(token.substr(1)); + else if(colon_idx < token.length()) { + name = token.substr(0, colon_idx); + length = std::stof(token.substr(colon_idx + 1)); + } else + name = token; + + names[open_idx] = name; + lengths[open_idx] = length; +} + +inline bool BPTree::is_structure_character(char c) const { + return (c == '(' || c == ')' || c == ',' || c == ';'); +} + +std::string BPTree::tokenize(std::string::iterator &start, const std::string::iterator &end) { + bool inquote = false; + bool isquote = false; + char c; + std::string token; + + do { + c = *start; + start++; + + if(c == '\n') { + continue; + } + + isquote = c == '\''; + + if(inquote && isquote) { + inquote = false; + continue; + } else if(!inquote && isquote) { + inquote = true; + continue; + } + + if(is_structure_character(c) && !inquote) { + if(token.length() == 0) + token.push_back(c); + break; + } + + token.push_back(c); + + + } while(start != end); + + return token; +} + +std::vector BPTree::get_structure() { + return structure; +} + +std::vector BPTree::get_openclose() { + return openclose; +} + diff --git a/R/unifrac_cpp/tree_s.hpp b/R/unifrac_cpp/tree_s.hpp new file mode 100644 index 000000000..690262dc4 --- /dev/null +++ b/R/unifrac_cpp/tree_s.hpp @@ -0,0 +1,150 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#ifndef __UNIFRAC_TREE_H +#define __UNIFRAC_TREE_H 1 + +#include +#include +#include +#include +#include + +#include + +namespace su { + class BPTree { + public: + /* tracked attributes */ + std::vector lengths; + std::vector names; + + /* total number of parentheses */ + uint32_t nparens; + + /* default constructor + * + * @param newick A newick string + */ + BPTree(std::string newick); + + /* constructor from a defined topology + * + * @param input_structure A boolean vector defining the topology + * @param input_lengths A vector of double of the branch lengths + * @param input_names A vector of str of the vertex names + */ + BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names); + + /* constructor from a TreeSummarizedExperiment + * + * @param treeSE A boolean vector defining the topology + * @param input_lengths A vector of double of the branch lengths + * @param input_names A vector of str of the vertex names + */ + BPTree(const Rcpp::S4 & treeSE); + + ~BPTree(); + + /* postorder tree traversal + * + * Get the index position of the ith node in a postorder tree + * traversal. + * + * @param i The ith node in a postorder traversal + */ + uint32_t postorderselect(uint32_t i)const ; + + /* preorder tree traversal + * + * Get the index position of the ith node in a preorder tree + * traversal. + * + * @param i The ith node in a preorder traversal + */ + uint32_t preorderselect(uint32_t i) const; + + /* Test if the node at an index position is a leaf + * + * @param i The node to evaluate + */ + bool isleaf(uint32_t i) const; + + /* Get the left child of a node + * + * @param i The node to obtain the left child from + */ + uint32_t leftchild(uint32_t i) const ; + + /* Get the right child of a node + * + * @param i The node to obtain the right child from + */ + uint32_t rightchild(uint32_t i) const; + + /* Get the right sibling of a node + * + * @param i The node to obtain the right sibling from + */ + uint32_t rightsibling(uint32_t i) const; + + /* Get the parent of a node + * + * @param i The node to obtain the parent of + */ + int32_t parent(uint32_t i) const; + + /* get the names at the tips of the tree */ + std::unordered_set get_tip_names(); + + /* public getters */ + std::vector get_structure(); + std::vector get_openclose(); + + /* serialize the structure as a sequence of 1s and 0s */ + void print() { + for(auto c = structure.begin(); c != structure.end(); c++) { + if(*c) + std::cout << "1"; + else + std::cout << "0"; + } + std::cout << std::endl; + } + BPTree mask(std::vector topology_mask, std::vector in_lengths); // mask self + + BPTree shear(std::unordered_set to_keep); + + BPTree collapse(); + + private: + std::vector structure; // the topology + std::vector openclose; // cache'd mapping between parentheses + std::vector select_0_index; // cache of select 0 + std::vector select_1_index; // cache of select 1 + std::vector excess; + + void index_and_cache(); // construct the select caches + void rowTree_to_bp(const Rcpp::List & rowTree); // convert rowTree to parentheses? + void newick_to_bp(std::string newick); // convert a newick string to parentheses + void newick_to_metadata(std::string newick); // convert newick to attributes + void structure_to_openclose(); // set the cache mapping between parentheses pairs + void set_node_metadata(unsigned int open_idx, std::string &token); // set attributes for a node + bool is_structure_character(char c) const; // test if a character is a newick structure + inline uint32_t open(uint32_t i) const; // obtain the index of the opening for a given parenthesis + inline uint32_t close(uint32_t i) const; // obtain the index of the closing for a given parenthesis + std::string tokenize(std::string::iterator &start, const std::string::iterator &end); // newick -> tokens + + int32_t bwd(uint32_t i, int32_t d) const; + int32_t enclose(uint32_t i) const; + }; +} + +#endif /* UNIFRAC_TREE_H */ + diff --git a/R/unifrac_cpp/unifrac_internal_s.cpp b/R/unifrac_cpp/unifrac_internal_s.cpp new file mode 100644 index 000000000..970a95a35 --- /dev/null +++ b/R/unifrac_cpp/unifrac_internal_s.cpp @@ -0,0 +1,150 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include "tree_s.hpp" +#include "biom_interface_s.hpp" +#include +#include +#include +#include +#include +#include +#include + +#include "unifrac_internal_s.hpp" + +using namespace su; + +template +PropStack::PropStack(uint32_t vecsize) +: prop_stack() +, prop_map() +, defaultsize(vecsize) +{ + prop_map.reserve(1000); +} + +template +PropStack::~PropStack() { + // drain stack + for(unsigned int i = 0; i < prop_stack.size(); i++) { + TFloat *vec = prop_stack.top(); + prop_stack.pop(); + free(vec); + } + + // drain the map + for(auto it = prop_map.begin(); it != prop_map.end(); it++) { + TFloat *vec = it->second; + free(vec); + } + prop_map.clear(); +} + +template +TFloat* PropStack::get(uint32_t i) { + return prop_map[i]; +} + +template +void PropStack::push(uint32_t node) { + TFloat* vec = prop_map[node]; + prop_map.erase(node); + prop_stack.push(vec); +} + +template +TFloat* PropStack::pop(uint32_t node) { + /* + * if we don't have any available vectors, create one + * add it to our record of known vectors so we can track our mallocs + */ + void *vec; + int err = 0; + if(prop_stack.empty()) { + // Linux-specific code + /*err = posix_memalign((void **)&vec, 32, sizeof(TFloat) * defaultsize); + if(vec == NULL || err != 0) { + fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", + sizeof(TFloat) * defaultsize, err, __FILE__, __LINE__); + exit(EXIT_FAILURE); + }*/ + // Windows-specific code + vec = _aligned_malloc(sizeof(TFloat) * defaultsize, 32); + if(vec == NULL || !vec) { + _get_errno(&err); + fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", + sizeof(TFloat) * defaultsize, err, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + } + else { + vec = prop_stack.top(); + prop_stack.pop(); + } + + prop_map[node] = (TFloat*) vec; + return (TFloat*) vec; +} + +// make sure they get instantiated +template class su::PropStack; +template class su::PropStack; + + +template +void su::set_proportions(TFloat* __restrict__ props, + const BPTree &tree, + uint32_t node, + const biom_interface &table, + PropStack &ps, + bool normalize) { + if(tree.isleaf(node)) { + table.get_obs_data(tree.names[node], props); // Here we basically just need the row for the specified node + if (normalize) { +#pragma omp parallel for schedule(static) + for(unsigned int i = 0; i < table.n_samples; i++) { + props[i] /= table.sample_counts[i]; + } + } + + } else { + unsigned int current = tree.leftchild(node); + unsigned int right = tree.rightchild(node); + +#pragma omp parallel for schedule(static) + for(unsigned int i = 0; i < table.n_samples; i++) + props[i] = 0; + + while(current <= right && current != 0) { + TFloat * __restrict__ vec = ps.get(current); // pull from prop map + ps.push(current); // remove from prop map, place back on stack + +#pragma omp parallel for schedule(static) + for(unsigned int i = 0; i < table.n_samples; i++) + props[i] = props[i] + vec[i]; + + current = tree.rightsibling(current); + } + } +} + +// make sure they get instantiated +template void su::set_proportions(float* __restrict__ props, + const BPTree &tree, + uint32_t node, + const biom_interface &table, + PropStack &ps, + bool normalize); +template void su::set_proportions(double* __restrict__ props, + const BPTree &tree, + uint32_t node, + const biom_interface &table, + PropStack &ps, + bool normalize); diff --git a/R/unifrac_cpp/unifrac_internal_s.hpp b/R/unifrac_cpp/unifrac_internal_s.hpp new file mode 100644 index 000000000..116cf8435 --- /dev/null +++ b/R/unifrac_cpp/unifrac_internal_s.hpp @@ -0,0 +1,44 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#ifndef __UNIFRAC_INTERNAL +#define __UNIFRAC_INTERNAL 1 + +#include +#include +#include +#include "biom_interface_s.hpp" +#include "unifrac_s.hpp" + +namespace su { + + template + class PropStack { + private: + std::stack prop_stack; + std::unordered_map prop_map; + uint32_t defaultsize; + public: + PropStack(uint32_t vecsize); + virtual ~PropStack(); + TFloat* pop(uint32_t i); + void push(uint32_t i); + TFloat* get(uint32_t i); + }; + + template + void set_proportions(TFloat* __restrict__ props, + const BPTree &tree, uint32_t node, + const biom_interface &table, + PropStack &ps, + bool normalize = true); + +} + +#endif diff --git a/R/unifrac_cpp/unifrac_s.cpp b/R/unifrac_cpp/unifrac_s.cpp new file mode 100644 index 000000000..a7db41928 --- /dev/null +++ b/R/unifrac_cpp/unifrac_s.cpp @@ -0,0 +1,71 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include "tree_s.hpp" +#include "biom_interface_s.hpp" +#include "unifrac_s.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unifrac_internal_s.hpp" + +using namespace su; + +std::string su::test_table_ids_are_subset_of_tree(su::biom_interface &table, su::BPTree &tree) { + std::unordered_set tip_names = tree.get_tip_names(); + std::unordered_set::const_iterator hit; + std::string a_missing_name = ""; + + for(auto i : table.obs_ids) { + hit = tip_names.find(i); + if(hit == tip_names.end()) { + a_missing_name = i; + break; + } + } + + return a_missing_name; +} + + + +// Computes Faith's PD for the samples in `table` over the phylogenetic +// tree given by `tree`. +// Assure that tree does not contain ids that are not in table +void su::faith_pd(biom_interface &table, + BPTree &tree, + double* result) { + PropStack propstack(table.n_samples); + + uint32_t node; + double *node_proportions; + double length; + + // for node in postorderselect + for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { + node = tree.postorderselect(k); + // get branch length + length = tree.lengths[node]; + + // get node proportions and set intermediate scores + node_proportions = propstack.pop(node); + set_proportions(node_proportions, tree, node, table, propstack); + + for (unsigned int sample = 0; sample < table.n_samples; sample++){ + // calculate contribution of node to score + result[sample] += (node_proportions[sample] > 0) * length; + } + } +} diff --git a/R/unifrac_cpp/unifrac_s.hpp b/R/unifrac_cpp/unifrac_s.hpp new file mode 100644 index 000000000..6302c5df8 --- /dev/null +++ b/R/unifrac_cpp/unifrac_s.hpp @@ -0,0 +1,30 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include +#include +#include +#include +#include + +#ifndef __UNIFRAC + +#include "task_parameters.hpp" +#include "biom_interface.hpp" + + namespace su { + + void faith_pd(biom_interface &table, BPTree &tree, double* result); + + std::string test_table_ids_are_subset_of_tree(biom_interface &table, BPTree &tree); + + } + +#define __UNIFRAC 1 +#endif From ec38762fd12e4c3880521a931eaeb8c3bb7fe0e5 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 6 Jan 2025 08:18:54 +0200 Subject: [PATCH 03/48] Fix code crashing due to out-of-bounds vector access --- R/unifrac_cpp/R_interface/rapi_test.R | 34 ++- R/unifrac_cpp/api_s.cpp | 70 +----- R/unifrac_cpp/api_s.hpp | 19 +- R/unifrac_cpp/biom_interface_s.hpp | 38 +-- R/unifrac_cpp/biom_s.cpp | 309 +++--------------------- R/unifrac_cpp/biom_s.hpp | 49 +--- R/unifrac_cpp/su_R_s.cpp | 105 ++------ R/unifrac_cpp/tree_s.cpp | 335 ++++++++++---------------- R/unifrac_cpp/tree_s.hpp | 19 +- R/unifrac_cpp/unifrac_internal_s.cpp | 118 +++------ R/unifrac_cpp/unifrac_internal_s.hpp | 15 +- R/unifrac_cpp/unifrac_s.cpp | 45 ++-- R/unifrac_cpp/unifrac_s.hpp | 11 +- 13 files changed, 329 insertions(+), 838 deletions(-) diff --git a/R/unifrac_cpp/R_interface/rapi_test.R b/R/unifrac_cpp/R_interface/rapi_test.R index 4d1b0662f..ed72b1882 100644 --- a/R/unifrac_cpp/R_interface/rapi_test.R +++ b/R/unifrac_cpp/R_interface/rapi_test.R @@ -15,7 +15,7 @@ aboutEquals <- function(x, y, msg){ stop(msg) } -source = "R/unifrac_cpp/su_R.cpp" +source = "R/unifrac_cpp/su_R_s.cpp" sourceCpp(source) table = "test.biom" tree = "test.tre" @@ -26,22 +26,36 @@ outfile <- tempfile() write_biom(b, outfile) bb <- read_biom(outfile) + fname <- "R/unifrac_cpp/R_interface/test.tre" +tree <- ape::rtree(500) +ape::write.tree(tree, file = fname, append = FALSE) + newick <- readChar(fname, file.info(fname)$size) +tree <- ape::read.tree(fname) + +identical(treetest(newick), treetest2(tree)) z <- treetest(newick) -tree <- ape::read.tree("R/unifrac_cpp/R_interface/test.tre") -treese <- makeTreeSEFromBiom(bb, treefilename=tree) +tree2 <- ape::reorder.phylo(tree, "postorder") +tree3 <- ape::read.tree("R/unifrac_cpp/R_interface/test2.tre") +treese <- makeTreeSEFromBiom(bb, treefilename=tree) treese2 <- changeTree(treese, rowTree = tree) -test <- tempfile() -rowTree(treese2) -print('Testing Faith PD..') +data(GlobalPatterns, package = "mia") +data(esophagus, package = "mia") +tse <- esophagus + #faith = faith_pd(table, tree) -faith <- faith_pd_new(treese2, newick) +faith_pd_new(tse) + + + +assays(tse)[[1]] +colSums(assays(tse)[[1]]) exp = c(4, 5, 6, 3, 2, 5) @@ -54,3 +68,9 @@ print('Success.') print('All tests pass') + +#Checks +#This functions assumes that the tree representation is in cladewise order - Ensure this with ape's reorder.phylo() function +#Ensure that the tree is non-empty, etc +#Ensure that the tree doesn't get modified at any point +#Which assays are normally used for the calculations? \ No newline at end of file diff --git a/R/unifrac_cpp/api_s.cpp b/R/unifrac_cpp/api_s.cpp index f09781c1a..4768a6913 100644 --- a/R/unifrac_cpp/api_s.cpp +++ b/R/unifrac_cpp/api_s.cpp @@ -16,69 +16,21 @@ using namespace su; using namespace std; -// https://stackoverflow.com/a/19841704/19741 -bool is_file_exists(const char *fileName) { - std::ifstream infile(fileName); - return infile.good(); -} - -void initialize_results_vec(r_vec* &result, biom& table){ - // Stores results for Faith PD - result = (r_vec*)malloc(sizeof(results_vec)); - result->n_samples = table.n_samples; - result->values = (double*)malloc(sizeof(double) * result->n_samples); - result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); - - for(unsigned int i = 0; i < result->n_samples; i++) { - size_t len = table.sample_ids[i].length(); - result->sample_ids[i] = (char*)malloc(sizeof(char) * len + 1); - table.sample_ids[i].copy(result->sample_ids[i], len); - result->sample_ids[i][len] = '\0'; - result->values[i] = 0; - } - -} - -void destroy_results_vec(r_vec** result) { - // for Faith PD - for(unsigned int i = 0; i < (*result)->n_samples; i++) { - free((*result)->sample_ids[i]); - }; - free((*result)->sample_ids); - free((*result)->values); - free(*result); -} - - -/* -#define PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) std::ifstream ifs(tree_filename); \ - std::string content = std::string(std::istreambuf_iterator(ifs), \ - std::istreambuf_iterator()); \ - su::BPTree tree = su::BPTree(content); \ - su::biom table = su::biom(biom_filename); \ - if(table.n_samples <= 0 | table.n_obs <= 0) { \ - return table_empty; \ - } \ - std::string bad_id = su::test_table_ids_are_subset_of_tree(table, tree); \ - if(bad_id != "") { \ - return table_and_tree_do_not_overlap; \ - } \ - std::unordered_set to_keep(table.obs_ids.begin(), \ - table.obs_ids.end()); \ - su::BPTree tree_sheared = tree.shear(to_keep).collapse(); \ - */ - -compute_status faith_pd_one_off(const Rcpp::S4 & treeSE, r_vec** result, std::string newick){ +std::vector faith_pd_one_off(const Rcpp::S4 & treeSE){ // Check that tree and table are non-empty and match before calling the c++ code // shear the tree (to contain only the obs in the table?) - Also should be done before the call? - su::BPTree tree = su::BPTree(newick); - - initialize_results_vec(*result, table); + std::cout << "Start\n"; + su::BPTree tree = su::BPTree(treeSE); + std::cout << "Tree ok\n"; + su::tse table = su::tse(treeSE); + std::cout << "Table ok\n"; - // compute faithpd - su::faith_pd(table, tree_sheared, std::ref((*result)->values)); + std::vector results = su::faith_pd(table, tree); + std::cout << "Results ok\n"; - return okay; + // compute faithpd + return results; + //return std::vector(); } \ No newline at end of file diff --git a/R/unifrac_cpp/api_s.hpp b/R/unifrac_cpp/api_s.hpp index 34c81afe1..5d0afc48e 100644 --- a/R/unifrac_cpp/api_s.hpp +++ b/R/unifrac_cpp/api_s.hpp @@ -10,22 +10,6 @@ #define EXTERN #endif -typedef enum compute_status {okay=0, tree_missing, table_missing, table_empty, unknown_method, table_and_tree_do_not_overlap, output_error} ComputeStatus; - -/* a result vector - * - * n_samples the number of samples. - * values the score values of length n_samples. - * sample_ids the sample IDs of length n_samples. - */ -typedef struct results_vec{ - unsigned int n_samples; - double* values; - char** sample_ids; -} r_vec; - -void destroy_results_vec(r_vec** result); - /* compute Faith PD * biom_filename the filename to the biom table. * tree_filename the filename to the correspodning tree. @@ -38,5 +22,4 @@ void destroy_results_vec(r_vec** result); * tree_missing : the filename for the tree does not exist * table_empty : the table does not have any entries */ -EXTERN ComputeStatus faith_pd_one_off(const Rcpp::S4 & treeSE, - r_vec** result, std::string newick); +std::vector faith_pd_one_off(const Rcpp::S4 & treeSE); \ No newline at end of file diff --git a/R/unifrac_cpp/biom_interface_s.hpp b/R/unifrac_cpp/biom_interface_s.hpp index bbfc80e4d..73e09e0bd 100644 --- a/R/unifrac_cpp/biom_interface_s.hpp +++ b/R/unifrac_cpp/biom_interface_s.hpp @@ -14,35 +14,37 @@ #include #include +#include + +//Faith calculations mainly need n_samples, get_obs_data and sample_counts +//sample_counts - OK +//n_samples - OK +//get_obs_data + namespace su { - class biom_interface { + class tse_interface { public: // cache the IDs contained within the table std::vector sample_ids; std::vector obs_ids; - // cache both index pointers into both CSC and CSR representations - std::vector sample_indptr; - std::vector obs_indptr; - uint32_t n_samples; // the number of samples uint32_t n_obs; // the number of observations - uint32_t nnz; // the total number of nonzero entries - double *sample_counts; + std::vector sample_counts; // Counts summed per sample /* default constructor * * Automatically create the needed objects. * All other initialization happens in children constructors. */ - biom_interface() {} + tse_interface() {} /* default destructor * * Automatically destroy the objects. * All other cleanup must have been performed by the children constructors. */ - virtual ~biom_interface() {} + virtual ~tse_interface() {} /* get a dense vector of observation data * @@ -51,22 +53,8 @@ namespace su { * Values of an index position [0, n_samples) which do not * have data will be zero'd. */ - virtual void get_obs_data(const std::string &id, double* out) const = 0; - virtual void get_obs_data(const std::string &id, float* out) const = 0; - - /* get a dense vector of a range of observation data - * - * @param id The observation ID to fetc - * @param start Initial index - * @param end First index past the end - * @param normalize If set, divide by sample_counts - * @param out An allocated array of at least size (end-start). First element will corrrectpoint to index start. - * Values of an index position [0, (end-start)) which do not - * have data will be zero'd. - */ - virtual void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, double* out) const = 0; - virtual void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, float* out) const = 0; - }; + virtual std::vector get_obs_data(const std::string &id) const = 0; + }; } #endif /* _UNIFRAC_BIOOM_INTERFACE_H */ diff --git a/R/unifrac_cpp/biom_s.cpp b/R/unifrac_cpp/biom_s.cpp index 3e4017992..156383dcd 100644 --- a/R/unifrac_cpp/biom_s.cpp +++ b/R/unifrac_cpp/biom_s.cpp @@ -10,46 +10,28 @@ #include #include #include +#include #include "biom_s.hpp" -using namespace H5; -using namespace su; - -/* datasets defined by the BIOM 2.x spec */ -const std::string OBS_INDPTR = std::string("/observation/matrix/indptr"); -const std::string OBS_INDICES = std::string("/observation/matrix/indices"); -const std::string OBS_DATA = std::string("/observation/matrix/data"); -const std::string OBS_IDS = std::string("/observation/ids"); - -const std::string SAMPLE_INDPTR = std::string("/sample/matrix/indptr"); -const std::string SAMPLE_INDICES = std::string("/sample/matrix/indices"); -const std::string SAMPLE_DATA = std::string("/sample/matrix/data"); -const std::string SAMPLE_IDS = std::string("/sample/ids"); +#include -biom::biom(std::string filename) { - file = H5File(filename.c_str(), H5F_ACC_RDONLY); +using namespace su; - /* establish the datasets */ - obs_indices = file.openDataSet(OBS_INDICES.c_str()); - obs_data = file.openDataSet(OBS_DATA.c_str()); - sample_indices = file.openDataSet(SAMPLE_INDICES.c_str()); - sample_data = file.openDataSet(SAMPLE_DATA.c_str()); - - /* cache IDs and indptr */ - sample_ids = std::vector(); +tse::tse(const Rcpp::S4 & treeSE) { + sample_ids = std::vector(); obs_ids = std::vector(); - sample_indptr = std::vector(); - obs_indptr = std::vector(); - - load_ids(OBS_IDS.c_str(), obs_ids); - load_ids(SAMPLE_IDS.c_str(), sample_ids); - load_indptr(OBS_INDPTR.c_str(), obs_indptr); - load_indptr(SAMPLE_INDPTR.c_str(), sample_indptr); + + Rcpp::S4 colData = treeSE.slot("colData"); + Rcpp::StringVector rownames = colData.slot("rownames"); + sample_ids = Rcpp::as>(rownames); + + Rcpp::List rowTree = treeSE.slot("rowTree"); + Rcpp::List phylo = rowTree["phylo"]; + Rcpp::StringVector tip_label = phylo["tip.label"]; + obs_ids = Rcpp::as>(tip_label); - /* cache shape and nnz info */ n_samples = sample_ids.size(); n_obs = obs_ids.size(); - set_nnz(); /* define a mapping between an ID and its corresponding offset */ obs_id_index = std::unordered_map(); @@ -57,109 +39,16 @@ biom::biom(std::string filename) { create_id_index(obs_ids, obs_id_index); create_id_index(sample_ids, sample_id_index); - - /* load obs sparse data */ - obs_indices_resident = (uint32_t**)malloc(sizeof(uint32_t**) * n_obs); - if(obs_indices_resident == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(uint32_t**) * n_obs, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - obs_data_resident = (double**)malloc(sizeof(double**) * n_obs); - if(obs_data_resident == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double**) * n_obs, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - obs_counts_resident = (unsigned int*)malloc(sizeof(unsigned int) * n_obs); - if(obs_counts_resident == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(unsigned int) * n_obs, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - - uint32_t *current_indices = NULL; - double *current_data = NULL; - for(unsigned int i = 0; i < obs_ids.size(); i++) { - std::string id_ = obs_ids[i]; - unsigned int n = get_obs_data_direct(id_, current_indices, current_data); - obs_counts_resident[i] = n; - obs_indices_resident[i] = current_indices; - obs_data_resident[i] = current_data; - } + sample_counts = get_sample_counts(); -} - -biom::~biom() { - for(unsigned int i = 0; i < n_obs; i++) { - free(obs_indices_resident[i]); - free(obs_data_resident[i]); - } - free(obs_indices_resident); - free(obs_data_resident); - free(obs_counts_resident); -} - -void biom::set_nnz() { - // should these be cached? - DataType dtype = obs_data.getDataType(); - DataSpace dataspace = obs_data.getSpace(); - - hsize_t dims[1]; - dataspace.getSimpleExtentDims(dims, NULL); - nnz = dims[0]; -} - -void biom::load_ids(const char *path, std::vector &ids) { - DataSet ds_ids = file.openDataSet(path); - DataType dtype = ds_ids.getDataType(); - DataSpace dataspace = ds_ids.getSpace(); - - hsize_t dims[1]; - dataspace.getSimpleExtentDims(dims, NULL); - - /* the IDs are a dataset of variable length strings */ - char **dataout = (char**)malloc(sizeof(char*) * dims[0]); - if(dataout == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(char*) * dims[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - ds_ids.read((void*)dataout, dtype); - - ids.reserve(dims[0]); - for(unsigned int i = 0; i < dims[0]; i++) { - ids.push_back(dataout[i]); - } - for(unsigned int i = 0; i < dims[0]; i++) - free(dataout[i]); - free(dataout); } -void biom::load_indptr(const char *path, std::vector &indptr) { - DataSet ds = file.openDataSet(path); - DataType dtype = ds.getDataType(); - DataSpace dataspace = ds.getSpace(); - - hsize_t dims[1]; - dataspace.getSimpleExtentDims(dims, NULL); +tse::~tse() { - uint32_t *dataout = (uint32_t*)malloc(sizeof(uint32_t) * dims[0]); - if(dataout == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(uint32_t) * dims[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - ds.read((void*)dataout, dtype); - - indptr.reserve(dims[0]); - for(unsigned int i = 0; i < dims[0]; i++) - indptr.push_back(dataout[i]); - free(dataout); } -void biom::create_id_index(std::vector &ids, +void tse::create_id_index(std::vector &ids, std::unordered_map &map) { uint32_t count = 0; map.reserve(ids.size()); @@ -168,157 +57,39 @@ void biom::create_id_index(std::vector &ids, } } -unsigned int biom::get_obs_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out) { - uint32_t idx = obs_id_index.at(id); - uint32_t start = obs_indptr[idx]; - uint32_t end = obs_indptr[idx + 1]; - - hsize_t count[1] = {end - start}; - hsize_t offset[1] = {start}; - - DataType indices_dtype = obs_indices.getDataType(); - DataType data_dtype = obs_data.getDataType(); - - DataSpace indices_dataspace = obs_indices.getSpace(); - DataSpace data_dataspace = obs_data.getSpace(); - - DataSpace indices_memspace(1, count, NULL); - DataSpace data_memspace(1, count, NULL); - - indices_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); - data_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); - - current_indices_out = (uint32_t*)malloc(sizeof(uint32_t) * count[0]); - if(current_indices_out == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(uint32_t) * count[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - current_data_out = (double*)malloc(sizeof(double) * count[0]); - if(current_data_out == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double) * count[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - - obs_indices.read((void*)current_indices_out, indices_dtype, indices_memspace, indices_dataspace); - obs_data.read((void*)current_data_out, data_dtype, data_memspace, data_dataspace); - - return count[0]; -} - +//Basically just gets the row for the specified id template -void biom::get_obs_data_TT(const std::string &id, TFloat* out) const { +std::vector tse::get_obs_data_TT(const std::string &id, TFloat t) const { + std::vector out = std::vector(); uint32_t idx = obs_id_index.at(id); - unsigned int count = obs_counts_resident[idx]; - const uint32_t * const indices = obs_indices_resident[idx]; - const double * const data = obs_data_resident[idx]; - - // reset our output buffer - for(unsigned int i = 0; i < n_samples; i++) - out[i] = 0.0; - - for(unsigned int i = 0; i < count; i++) { - out[indices[i]] = data[i]; + for(unsigned int i = 0; i < n_samples; i++) { + out.push_back(assay(idx, i)); } + return out; } -void biom::get_obs_data(const std::string &id, double* out) const { - biom::get_obs_data_TT(id,out); -} - -void biom::get_obs_data(const std::string &id, float* out) const { - biom::get_obs_data_TT(id,out); +std::vector tse::get_obs_data(const std::string &id) const { + double t = 0.0; + return(tse::get_obs_data_TT(id, t)); } -// note: out is supposed to be fully filled, i.e. out[start:end] -template -void biom::get_obs_data_range_TT(const std::string &id, unsigned int start, unsigned int end, bool normalize, TFloat* out) const { - uint32_t idx = obs_id_index.at(id); - unsigned int count = obs_counts_resident[idx]; - const uint32_t * const indices = obs_indices_resident[idx]; - const double * const data = obs_data_resident[idx]; +//Returns a pointer-based array - can perhaps be changed to simply referring to the R object's internal storage? +//What exactly does this array contain? It contains n_samples elements which are doubles. +//I'm fairly sure that it just sums the counts over samples. Basically just get a column sum. +//std::vector uses move semantics so it shouldn't affect memory usage too much +//the R representation is inherently 'dense' so we can just iterate over the columns +//Might be useful to store? - // reset our output buffer - for(unsigned int i = start; i < end; i++) - out[i-start] = 0.0; - - if (normalize) { - for(unsigned int i = 0; i < count; i++) { - const int32_t j = indices[i]; - if ((j>=start)&&(j=start)&&(j tse::get_sample_counts() { + std::vector sample_counts = std::vector(); - DataSpace indices_memspace(1, count, NULL); - DataSpace data_memspace(1, count, NULL); - - indices_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); - data_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); - - current_indices_out = (uint32_t*)malloc(sizeof(uint32_t) * count[0]); - if(current_indices_out == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(uint32_t) * count[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - current_data_out = (double*)malloc(sizeof(double) * count[0]); - if(current_data_out == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double) * count[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - - sample_indices.read((void*)current_indices_out, indices_dtype, indices_memspace, indices_dataspace); - sample_data.read((void*)current_data_out, data_dtype, data_memspace, data_dataspace); - - return count[0]; -} - -double* biom::get_sample_counts() { - double *sample_counts = (double*)calloc(sizeof(double), n_samples); - for(unsigned int i = 0; i < n_obs; i++) { - unsigned int count = obs_counts_resident[i]; - uint32_t *indices = obs_indices_resident[i]; - double *data = obs_data_resident[i]; - for(unsigned int j = 0; j < count; j++) { - uint32_t index = indices[j]; - double datum = data[j]; - sample_counts[index] += datum; + for(unsigned int i = 0; i < n_samples; i++) { + unsigned int sum = 0; + for(unsigned int j = 0; j < n_obs; j++){ + sum += assay(j, i); } + sample_counts.push_back(sum); } - return sample_counts; + return(sample_counts); } diff --git a/R/unifrac_cpp/biom_s.hpp b/R/unifrac_cpp/biom_s.hpp index 011c9c8b5..78f5d8d49 100644 --- a/R/unifrac_cpp/biom_s.hpp +++ b/R/unifrac_cpp/biom_s.hpp @@ -18,20 +18,22 @@ #include "biom_interface_s.hpp" +#include + namespace su { - class biom : public biom_interface { + class tse : public tse_interface { public: /* default constructor * - * @param filename The path to the BIOM table to read + * @param treeSE An R TreeSummarizedExperiment object */ - biom(std::string filename); + tse(const Rcpp::S4 & treeSE); /* default destructor * * Temporary arrays are freed */ - virtual ~biom(); + virtual ~tse(); /* get a dense vector of observation data * @@ -40,36 +42,12 @@ namespace su { * Values of an index position [0, n_samples) which do not * have data will be zero'd. */ - void get_obs_data(const std::string &id, double* out) const; - void get_obs_data(const std::string &id, float* out) const; - - /* get a dense vector of a range of observation data - * - * @param id The observation ID to fetc - * @param start Initial index - * @param end First index past the end - * @param normalize If set, divide by sample_counts - * @param out An allocated array of at least size (end-start). First element will corrrectpoint to index start. - * Values of an index position [0, (end-start)) which do not - * have data will be zero'd. - */ - void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, double* out) const; - void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, float* out) const; + std::vector get_obs_data(const std::string &id) const; private: - /* retain DataSet handles within the HDF5 file */ - H5::DataSet obs_indices; - H5::DataSet sample_indices; - H5::DataSet obs_data; - H5::DataSet sample_data; - H5::H5File file; - uint32_t **obs_indices_resident; - double **obs_data_resident; - unsigned int *obs_counts_resident; - - unsigned int get_obs_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out); - unsigned int get_sample_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out); - double* get_sample_counts(); + Rcpp::NumericMatrix assay; // Access to the raw sample counts in R's memory + + std::vector get_sample_counts(); /* At construction, lookups mapping IDs -> index position within an * axis are defined @@ -103,11 +81,10 @@ namespace su { void create_id_index(std::vector &ids, std::unordered_map &map); - // templatized version - template void get_obs_data_TT(const std::string &id, TFloat* out) const; - template void get_obs_data_range_TT(const std::string &id, unsigned int start, unsigned int end, bool normalize, TFloat* out) const; - }; + template std::vector get_obs_data_TT(const std::string &id, TFloat t) const; + + }; } #endif /* _UNIFRAC_BIOM_H */ diff --git a/R/unifrac_cpp/su_R_s.cpp b/R/unifrac_cpp/su_R_s.cpp index 7ff5458fd..cccae83a9 100644 --- a/R/unifrac_cpp/su_R_s.cpp +++ b/R/unifrac_cpp/su_R_s.cpp @@ -4,9 +4,12 @@ #include "api_s.hpp" #include "tree_s.hpp" +#include + using namespace std; using namespace Rcpp; + /* // [[Rcpp::export]] Rcpp::List faith_pd(const char* table, const char* tree){ @@ -25,11 +28,23 @@ Rcpp::List faith_pd(const char* table, const char* tree){ */ // [[Rcpp::export]] -Rcpp::List faith_pd_new(const Rcpp::S4 & treeSE, Rcpp::String tree){ +void faith_pd_new(const Rcpp::S4 & treeSE){ + + std::vector results = faith_pd_one_off(treeSE); + + std::cout << results.size() << "\n"; + + if(results.size() >= 20){ + for(unsigned int i = 0; i < 20; i++){ + std::cout << results[i] << "\n"; + } + } + //get_sample_counts(treeSE); + /* r_vec* result = NULL; ComputeStatus status; - std::string newick(tree.get_cstring()); - status = faith_pd_one_off(treeSE, &result, tree); + status = faith_pd_one_off(treeSE, &result); + vector values; for(int i = 0; i < result->n_samples; i++){ values.push_back(result->values[i]); @@ -38,88 +53,16 @@ Rcpp::List faith_pd_new(const Rcpp::S4 & treeSE, Rcpp::String tree){ Rcpp::List rlist = Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, Rcpp::Named("faith_pd") = values); - destroy_results_vec(result); - - return rlist; - - /* - Rcpp::List rowTree = treeSE.slot("rowTree"); - const List & phylo = rowTree["phylo"]; - const Rcpp::NumericMatrix & edge = phylo["edge"]; + destroy_results_vec(&result); - return rowTree; + return rlist; */ -} - - -std::vector newick(Rcpp::String ins) { - std::string newick(ins.get_cstring()); - std::vector result = std::vector(); - char last_structure; - bool potential_single_descendent = false; - int count = 0; - bool in_quote = false; - for(auto c = newick.begin(); c != newick.end(); c++) { - if(*c == '\'') - in_quote = !in_quote; - - if(in_quote) - continue; - - switch(*c) { - case '(': - // opening of a node - count++; - result.push_back(true); - last_structure = *c; - potential_single_descendent = true; - break; - case ')': - // closing of a node - if(potential_single_descendent || (last_structure == ',')) { - // we have a single descendent or a last child (i.e. ",)" scenario) - count += 3; - result.push_back(true); - result.push_back(false); - result.push_back(false); - potential_single_descendent = false; - } else { - // it is possible still to have a single descendent in the case of - // multiple single descendents (e.g., (...()...) ) - count += 1; - result.push_back(false); - } - last_structure = *c; - break; - case ',': - if(last_structure != ')') { - // we have a new tip - count += 2; - result.push_back(true); - result.push_back(false); - } - potential_single_descendent = false; - last_structure = *c; - break; - default: - break; - } - } - return result; -} - -// [[Rcpp::export]] -Rcpp::LogicalVector treetest(std::string n){ - Rcpp::LogicalVector result = Rcpp::LogicalVector(); - std::vector raw = newick(n); - if(raw.size() > 0) { - std::cout << raw.size(); - } - result = Rcpp::LogicalVector::import(raw.begin(), raw.end()); - return result; + //Rcpp::List rowTree = treeSE.slot("rowTree"); + //const List & phylo = rowTree["phylo"]; + //const Rcpp::NumericMatrix & edge = phylo["edge"]; + //return rowTree; } - diff --git a/R/unifrac_cpp/tree_s.cpp b/R/unifrac_cpp/tree_s.cpp index eb7c7ad7f..ba996d8da 100644 --- a/R/unifrac_cpp/tree_s.cpp +++ b/R/unifrac_cpp/tree_s.cpp @@ -6,33 +6,6 @@ using namespace su; -BPTree::BPTree(std::string newick) { - openclose = std::vector(); - lengths = std::vector(); - names = std::vector(); - excess = std::vector(); - - select_0_index = std::vector(); - select_1_index = std::vector(); - structure = std::vector(); - structure.reserve(500000); // a fair sized tree... avoid reallocs, and its not _that_ much waste if this is wrong - - // three pass for parse. not ideal, but easier to map from IOW code - newick_to_bp(newick); - - // resize is correct here as we are not performing a push_back - openclose.resize(nparens); - lengths.resize(nparens); - names.resize(nparens); - select_0_index.resize(nparens / 2); - select_1_index.resize(nparens / 2); - excess.resize(nparens); - - structure_to_openclose(); - newick_to_metadata(newick); - index_and_cache(); -} - BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names) { structure = input_structure; lengths = input_lengths; @@ -53,9 +26,44 @@ BPTree::BPTree(std::vector input_structure, std::vector input_leng } BPTree::BPTree(const Rcpp::S4 & treeSE) { - const Rcpp::S4 & rowTree = treeSE.slot("RowTree"); + + //Initialize vectors + openclose = std::vector(); + lengths = std::vector(); + names = std::vector(); + excess = std::vector(); + + select_0_index = std::vector(); + select_1_index = std::vector(); + + //Load the tree structure structure = std::vector(); structure.reserve(500000); // a fair sized tree... avoid reallocs, and its not _that_ much waste if this is wrong + const Rcpp::List & rowTree = treeSE.slot("rowTree"); + rowTree_to_bp(rowTree); //Also sets the size of nparens + + std::cout << "BP ok\n"; + + //Resize vectors + // resize is correct here as we are not performing a push_back + openclose.resize(nparens); + lengths.resize(nparens); + names.resize(nparens); + excess.resize(nparens); + + select_0_index.resize(nparens / 2); + select_1_index.resize(nparens / 2); + + //Builds a vector that lets us find the corresponding indices for each true/false pair + structure_to_openclose(); + std::cout << "structure ok\n"; + //Get metadata + rowTree_to_metadata(rowTree); + std::cout << "metadata ok\n"; + + //Finalize + index_and_cache(); // This causes a crash for some reason + std::cout << "cache ok\n"; } @@ -193,7 +201,7 @@ void BPTree::index_and_cache() { auto k1 = select_1_index.begin(); auto e_it = excess.begin(); unsigned int e = 0; - + for(; i != structure.end(); i++, idx++ ) { if(*i) { *(k1++) = idx; @@ -273,101 +281,59 @@ int32_t BPTree::bwd(uint32_t i, int d) const { return -1; } +// The algorithms that this class uses need the tree to be stored in a binary format +// In terms of the Newick format, an opening bracket corresponds to a TRUE, a closing bracket to a FALSE, and a tip to a TRUE FALSE +// This functions assumes that the tree representation is in cladewise order - Ensure this with ape's reorder.phylo() function void BPTree::rowTree_to_bp(const Rcpp::List & rowTree) { Rcpp::List phylo = rowTree["phylo"]; Rcpp::NumericMatrix edge = phylo["edge"]; Rcpp::StringVector tips = phylo["tip.label"]; - uint32_t ntips = tips.size(); - std::stack nodes = std::stack(); // phylo tips are always numbered from 1 to number of tips; - std::vector levels = std::vector(ntips, 0); // Use this to keep track of how many descendants the current branch's node have - - int previousNode = 0; + + uint32_t ntips = tips.size(); // phylo tips are always numbered from 1 to number of tips; + + std::stack nodes; // Keeps track of the branch's internal nodes + int currentNode = 0; int nextNode = 0; - int count = 0; - bool potential_single_descendent = false; - for (int i = 0; i < edge.nrow(); i++){ - currentNode = edge(i, 1); - if(nodes.size() == 0 || nodes.top() != currentNode ) { // Either were at the root, or entering a new node - if(nodes.top() < currentNode) { // We've exhausted the node and moved backwards in the tree - - } + // Goal: Insert true when a branch starts, a false when it closes, and a true-false for each tip. + + for (unsigned int i = 0; i < edge.nrow(); i++){ + currentNode = edge(i, 0); + nextNode = edge(i, 1); + + if(nodes.size() > 0 && currentNode < nodes.top()) { + // We've exhausted the branch and moved backwards in the tree + do { + nodes.pop(); + structure.push_back(false); + } while(currentNode != nodes.top()); + } + + if(nodes.size() == 0 || currentNode > nodes.top() ) { + // We are either at the root, or entering a new node nodes.push(currentNode); - count++; structure.push_back(true); - potential_single_descendent = true; - break; - } - nextNode = edge(i, 2); + } - if(nextNode <= ntips) { // Node is a tip - count += 2; + if(nextNode <= ntips) { + // We've found a tip structure.push_back(true); structure.push_back(false); } - previousNode = currentNode; - } - -} - -void BPTree::newick_to_bp(std::string newick) { - char last_structure; - bool potential_single_descendent = false; - int count = 0; - bool in_quote = false; - for(auto c = newick.begin(); c != newick.end(); c++) { - if(*c == '\'') - in_quote = !in_quote; - - if(in_quote) - continue; - - switch(*c) { - case '(': - // opening of a node - count++; - structure.push_back(true); - last_structure = *c; - potential_single_descendent = true; - break; - case ')': - // closing of a node - if(potential_single_descendent || (last_structure == ',')) { - // we have a single descendent or a last child (i.e. ",)" scenario) - count += 3; - structure.push_back(true); - structure.push_back(false); - structure.push_back(false); - potential_single_descendent = false; - } else { - // it is possible still to have a single descendent in the case of - // multiple single descendents (e.g., (...()...) ) - count += 1; - structure.push_back(false); - } - last_structure = *c; - break; - case ',': - if(last_structure != ')') { - // we have a new tip - count += 2; - structure.push_back(true); - structure.push_back(false); - } - potential_single_descendent = false; - last_structure = *c; - break; - default: - break; + if(i == edge.nrow() - 1) { + // We've reached the end of the tree + do { + nodes.pop(); + structure.push_back(false); + } while(nodes.size() > 0); } } nparens = structure.size(); } - void BPTree::structure_to_openclose() { std::stack oc; unsigned int open_idx; @@ -384,123 +350,76 @@ void BPTree::structure_to_openclose() { } } } -// trim from end -// from http://stackoverflow.com/a/217605 -static inline std::string &rtrim(std::string &s) { - s.erase(std::find_if(s.rbegin(), s.rend(), - std::not1(std::ptr_fun(std::isspace))).base(), s.end()); - return s; -} - -//// WEIRDNESS. THIS SOLVES IT WITH THE RTRIM. ISOLATE, MOVE TO CONSTRUCTOR. -void BPTree::newick_to_metadata(std::string newick) { - newick = rtrim(newick); +//Add metadata (lengths and names) to the tree representation +//I think we can just iterate through the structure, and whenever we hit a true decide if it's a leaf or not, and then add the corresponding label/length +//edge.length has (nodes + tips) elements - leaves at the start, nodes at the end +//tip.label has (tips) elements +//root.edge and node.labels are optional, giving the length of the root and the internal node (including root) labels, respectively +void BPTree::rowTree_to_metadata(const Rcpp::List & rowTree) { + Rcpp::List phylo = rowTree["phylo"]; + Rcpp::NumericVector edgelength = phylo["edge.length"]; + Rcpp::NumericMatrix edges = phylo["edge"]; + Rcpp::StringVector tips = phylo["tip.label"]; - std::string::iterator start = newick.begin(); - std::string::iterator end = newick.end(); - std::string token; - char last_structure = '\0'; - - unsigned int structure_idx = 0; - unsigned int lag = 0; - unsigned int open_idx; + const uint32_t n_edges = edgelength.size(); + uint32_t ntips = tips.size(); - while(start != end) { - token = tokenize(start, end); - // this sucks. - if(token.length() == 1 && is_structure_character(token[0])) { - switch(token[0]) { - case '(': - structure_idx++; - break; - case ')': - case ',': - structure_idx++; - if(last_structure == ')') - lag++; - break; - } - } else { - // puts us on the corresponding closing parenthesis - structure_idx += lag; - lag = 0; + //Used to find the correct lengths for the nodes - Includes the root + std::vector edge_v(n_edges + 1, 0.0); + + for(unsigned int i = 0; i < n_edges; i++){ + edge_v.at(edges(i,1) - 1) = edgelength[i]; + } + + if(phylo.containsElementNamed("root.edge")) { + edge_v.at(ntips) = phylo["root.edge"]; + } + + bool hasNodeLabels = false; + Rcpp::StringVector nodes; + + if(phylo.containsElementNamed("node.labels")) { + hasNodeLabels = true; + nodes = phylo["node.labels"]; + } + + unsigned int tip_idx = 0; // tip indices run from 0 to ntips-1 + unsigned int node_idx = 0; // node indices run from ntips to ntips + nnodes - 1 + unsigned int edge_idx = 0; // Used to store the index of the edge for picking lengths; + + for(unsigned int i = 0; i < structure.size(); i++) { + if(structure[i]){ + std::string label = std::string(); + double length = 0.0; - open_idx = open(structure_idx); - set_node_metadata(open_idx, token); - // std::cout << structure_idx << " <-> " << open_idx << " " << token << std::endl; - // make sure to advance an extra position if we are a leaf as the - // as a leaf is by definition a 10, and doing a single advancement - // would put the structure to token mapping out of sync - if(isleaf(open_idx)) - structure_idx += 2; - else - structure_idx += 1; + if(isleaf(i)){ + //Tips can be expected to have both a length and a label + label = Rcpp::as(tips[tip_idx]); + length = edge_v[tip_idx]; + tip_idx++; + } + else{ + //Nodes always have lengths (except the root, which may have it optionally, but defaults to 0.0) + //Nodes may also optionally have labels (which includes the root label) + length = edge_v[ntips + node_idx]; + if(hasNodeLabels){ + label = Rcpp::as(nodes[node_idx]); + } + node_idx++; + } + set_node_metadata(i,label, length); } - last_structure = token[0]; } } -void BPTree::set_node_metadata(unsigned int open_idx, std::string &token) { - double length = 0.0; - std::string name = std::string(); - unsigned int colon_idx = token.find_last_of(':'); - - if(colon_idx == 0) - length = std::stof(token.substr(1)); - else if(colon_idx < token.length()) { - name = token.substr(0, colon_idx); - length = std::stof(token.substr(colon_idx + 1)); - } else - name = token; - +//This takes a label and a length and assigns them to the correct places +void BPTree::set_node_metadata(unsigned int open_idx, std::string name, double length) { names[open_idx] = name; lengths[open_idx] = length; } -inline bool BPTree::is_structure_character(char c) const { - return (c == '(' || c == ')' || c == ',' || c == ';'); -} - -std::string BPTree::tokenize(std::string::iterator &start, const std::string::iterator &end) { - bool inquote = false; - bool isquote = false; - char c; - std::string token; - - do { - c = *start; - start++; - - if(c == '\n') { - continue; - } - - isquote = c == '\''; - - if(inquote && isquote) { - inquote = false; - continue; - } else if(!inquote && isquote) { - inquote = true; - continue; - } - - if(is_structure_character(c) && !inquote) { - if(token.length() == 0) - token.push_back(c); - break; - } - - token.push_back(c); - - - } while(start != end); - - return token; -} - std::vector BPTree::get_structure() { return structure; } diff --git a/R/unifrac_cpp/tree_s.hpp b/R/unifrac_cpp/tree_s.hpp index 690262dc4..f691f1d47 100644 --- a/R/unifrac_cpp/tree_s.hpp +++ b/R/unifrac_cpp/tree_s.hpp @@ -28,12 +28,6 @@ namespace su { /* total number of parentheses */ uint32_t nparens; - /* default constructor - * - * @param newick A newick string - */ - BPTree(std::string newick); - /* constructor from a defined topology * * @param input_structure A boolean vector defining the topology @@ -44,9 +38,7 @@ namespace su { /* constructor from a TreeSummarizedExperiment * - * @param treeSE A boolean vector defining the topology - * @param input_lengths A vector of double of the branch lengths - * @param input_names A vector of str of the vertex names + * @param treeSE An R treeSE object */ BPTree(const Rcpp::S4 & treeSE); @@ -117,6 +109,7 @@ namespace su { } std::cout << std::endl; } + BPTree mask(std::vector topology_mask, std::vector in_lengths); // mask self BPTree shear(std::unordered_set to_keep); @@ -131,15 +124,13 @@ namespace su { std::vector excess; void index_and_cache(); // construct the select caches - void rowTree_to_bp(const Rcpp::List & rowTree); // convert rowTree to parentheses? - void newick_to_bp(std::string newick); // convert a newick string to parentheses + void rowTree_to_bp(const Rcpp::List & rowTree); // convert ape tree structure to boolean structure + void rowTree_to_metadata(const Rcpp::List & rowTree); // assign attributes void newick_to_metadata(std::string newick); // convert newick to attributes void structure_to_openclose(); // set the cache mapping between parentheses pairs - void set_node_metadata(unsigned int open_idx, std::string &token); // set attributes for a node - bool is_structure_character(char c) const; // test if a character is a newick structure + void set_node_metadata(unsigned int open_idx, std::string label, double length); // set attributes for a node inline uint32_t open(uint32_t i) const; // obtain the index of the opening for a given parenthesis inline uint32_t close(uint32_t i) const; // obtain the index of the closing for a given parenthesis - std::string tokenize(std::string::iterator &start, const std::string::iterator &end); // newick -> tokens int32_t bwd(uint32_t i, int32_t d) const; int32_t enclose(uint32_t i) const; diff --git a/R/unifrac_cpp/unifrac_internal_s.cpp b/R/unifrac_cpp/unifrac_internal_s.cpp index 970a95a35..c87082e1b 100644 --- a/R/unifrac_cpp/unifrac_internal_s.cpp +++ b/R/unifrac_cpp/unifrac_internal_s.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include "unifrac_internal_s.hpp" @@ -32,65 +33,26 @@ PropStack::PropStack(uint32_t vecsize) template PropStack::~PropStack() { - // drain stack - for(unsigned int i = 0; i < prop_stack.size(); i++) { - TFloat *vec = prop_stack.top(); - prop_stack.pop(); - free(vec); - } - - // drain the map - for(auto it = prop_map.begin(); it != prop_map.end(); it++) { - TFloat *vec = it->second; - free(vec); - } - prop_map.clear(); } template -TFloat* PropStack::get(uint32_t i) { - return prop_map[i]; +std::vector PropStack::get(uint32_t i) { + if(prop_map.count(i) > 0){ + return prop_map.at(i); + } + else { + return(std::vector()); + } } template -void PropStack::push(uint32_t node) { - TFloat* vec = prop_map[node]; - prop_map.erase(node); - prop_stack.push(vec); +void PropStack::clear(uint32_t i) { + prop_map[i] = std::vector(); } template -TFloat* PropStack::pop(uint32_t node) { - /* - * if we don't have any available vectors, create one - * add it to our record of known vectors so we can track our mallocs - */ - void *vec; - int err = 0; - if(prop_stack.empty()) { - // Linux-specific code - /*err = posix_memalign((void **)&vec, 32, sizeof(TFloat) * defaultsize); - if(vec == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(TFloat) * defaultsize, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - }*/ - // Windows-specific code - vec = _aligned_malloc(sizeof(TFloat) * defaultsize, 32); - if(vec == NULL || !vec) { - _get_errno(&err); - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(TFloat) * defaultsize, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - } - else { - vec = prop_stack.top(); - prop_stack.pop(); - } - - prop_map[node] = (TFloat*) vec; - return (TFloat*) vec; +void PropStack::update(uint32_t node, std::vector vec) { + prop_map[node] = vec; } // make sure they get instantiated @@ -99,52 +61,50 @@ template class su::PropStack; template -void su::set_proportions(TFloat* __restrict__ props, - const BPTree &tree, +std::vector su::set_proportions(const BPTree &tree, uint32_t node, - const biom_interface &table, + const tse_interface &table, PropStack &ps, bool normalize) { + + std::vector props = std::vector(); if(tree.isleaf(node)) { - table.get_obs_data(tree.names[node], props); // Here we basically just need the row for the specified node - if (normalize) { -#pragma omp parallel for schedule(static) - for(unsigned int i = 0; i < table.n_samples; i++) { - props[i] /= table.sample_counts[i]; - } - } - + std::string leaf = tree.names[node]; + props = table.get_obs_data(leaf); // Here we basically just need the row for the specified node + if (normalize) { +//#pragma omp parallel for schedule(static) + for(unsigned int i = 0; i < table.n_samples; i++) { + props[i] /= table.sample_counts[i]; + } + } } else { unsigned int current = tree.leftchild(node); unsigned int right = tree.rightchild(node); -#pragma omp parallel for schedule(static) - for(unsigned int i = 0; i < table.n_samples; i++) - props[i] = 0; - +//#pragma omp parallel for schedule(static) + + for(unsigned int i = 0; i < table.n_samples; i++){ + props.push_back(0); + } + while(current <= right && current != 0) { - TFloat * __restrict__ vec = ps.get(current); // pull from prop map - ps.push(current); // remove from prop map, place back on stack - -#pragma omp parallel for schedule(static) + std::vector vec = ps.get(current); // pull from prop map + ps.clear(current); // remove from prop map, place back on stack +//#pragma omp parallel for schedule(static) for(unsigned int i = 0; i < table.n_samples; i++) props[i] = props[i] + vec[i]; - + current = tree.rightsibling(current); } + } + ps.update(node, props); + return(props); } // make sure they get instantiated -template void su::set_proportions(float* __restrict__ props, - const BPTree &tree, - uint32_t node, - const biom_interface &table, - PropStack &ps, - bool normalize); -template void su::set_proportions(double* __restrict__ props, - const BPTree &tree, +template std::vector su::set_proportions(const BPTree &tree, uint32_t node, - const biom_interface &table, + const tse_interface &table, PropStack &ps, bool normalize); diff --git a/R/unifrac_cpp/unifrac_internal_s.hpp b/R/unifrac_cpp/unifrac_internal_s.hpp index 116cf8435..49b0690bc 100644 --- a/R/unifrac_cpp/unifrac_internal_s.hpp +++ b/R/unifrac_cpp/unifrac_internal_s.hpp @@ -21,21 +21,20 @@ namespace su { template class PropStack { private: - std::stack prop_stack; - std::unordered_map prop_map; + std::stack> prop_stack; + std::unordered_map> prop_map; uint32_t defaultsize; public: PropStack(uint32_t vecsize); virtual ~PropStack(); - TFloat* pop(uint32_t i); - void push(uint32_t i); - TFloat* get(uint32_t i); + void clear(uint32_t i); + void update(uint32_t i, std::vector vec); + std::vector get(uint32_t i); }; template - void set_proportions(TFloat* __restrict__ props, - const BPTree &tree, uint32_t node, - const biom_interface &table, + std::vector set_proportions(const BPTree &tree, uint32_t node, + const tse_interface &table, PropStack &ps, bool normalize = true); diff --git a/R/unifrac_cpp/unifrac_s.cpp b/R/unifrac_cpp/unifrac_s.cpp index a7db41928..2d0f298c3 100644 --- a/R/unifrac_cpp/unifrac_s.cpp +++ b/R/unifrac_cpp/unifrac_s.cpp @@ -21,51 +21,40 @@ #include "unifrac_internal_s.hpp" -using namespace su; - -std::string su::test_table_ids_are_subset_of_tree(su::biom_interface &table, su::BPTree &tree) { - std::unordered_set tip_names = tree.get_tip_names(); - std::unordered_set::const_iterator hit; - std::string a_missing_name = ""; - - for(auto i : table.obs_ids) { - hit = tip_names.find(i); - if(hit == tip_names.end()) { - a_missing_name = i; - break; - } - } - - return a_missing_name; -} - +#include +using namespace su; // Computes Faith's PD for the samples in `table` over the phylogenetic // tree given by `tree`. // Assure that tree does not contain ids that are not in table -void su::faith_pd(biom_interface &table, - BPTree &tree, - double* result) { - PropStack propstack(table.n_samples); +std::vector su::faith_pd(tse_interface &table, + BPTree &tree) { + PropStack propstack(table.n_samples); // construction seems to go okay + uint32_t node; - double *node_proportions; + std::vector node_proportions; double length; + + std::vector results = std::vector(); // for node in postorderselect for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { node = tree.postorderselect(k); + // get branch length length = tree.lengths[node]; - + // get node proportions and set intermediate scores - node_proportions = propstack.pop(node); - set_proportions(node_proportions, tree, node, table, propstack); - + + node_proportions = set_proportions(tree, node, table, propstack); + for (unsigned int sample = 0; sample < table.n_samples; sample++){ // calculate contribution of node to score - result[sample] += (node_proportions[sample] > 0) * length; + results.push_back((node_proportions[sample] > 0) * length); } } + return results; + //return(std::vector()); } diff --git a/R/unifrac_cpp/unifrac_s.hpp b/R/unifrac_cpp/unifrac_s.hpp index 6302c5df8..9d52fba3d 100644 --- a/R/unifrac_cpp/unifrac_s.hpp +++ b/R/unifrac_cpp/unifrac_s.hpp @@ -13,17 +13,16 @@ #include #include +#include + #ifndef __UNIFRAC #include "task_parameters.hpp" -#include "biom_interface.hpp" +#include "biom_interface_s.hpp" +#include "tree_s.hpp" namespace su { - - void faith_pd(biom_interface &table, BPTree &tree, double* result); - - std::string test_table_ids_are_subset_of_tree(biom_interface &table, BPTree &tree); - + std::vector faith_pd(tse_interface &table, su::BPTree &tree); } #define __UNIFRAC 1 From cefd2869e9f83010fa53f3df3f3e546d67f45f0b Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Wed, 15 Jan 2025 15:13:37 +0200 Subject: [PATCH 04/48] C++ code functional for some datasets The C++ code works for rooted trees, but not unrooted ones. Possibly a bug in my implementation, more testing needed. --- R/unifrac_cpp/R_interface/rapi_test.R | 91 +++++++-------- R/unifrac_cpp/R_interface/test.tre | 2 +- R/unifrac_cpp/api_s.cpp | 4 +- R/unifrac_cpp/api_s.hpp | 2 +- R/unifrac_cpp/biom_s.cpp | 5 + R/unifrac_cpp/su_R_s.cpp | 161 +++++++++++++++++++------- R/unifrac_cpp/tree_s.cpp | 16 ++- R/unifrac_cpp/tree_s.hpp | 5 +- R/unifrac_cpp/unifrac_internal_s.cpp | 3 + R/unifrac_cpp/unifrac_s.cpp | 5 +- 10 files changed, 189 insertions(+), 105 deletions(-) diff --git a/R/unifrac_cpp/R_interface/rapi_test.R b/R/unifrac_cpp/R_interface/rapi_test.R index ed72b1882..823f9e953 100644 --- a/R/unifrac_cpp/R_interface/rapi_test.R +++ b/R/unifrac_cpp/R_interface/rapi_test.R @@ -1,76 +1,65 @@ library(Rcpp) library(mia) -library(biomformat) +library(miaSim) library(ape) -library(rhdf5) - -equals <- function(x, y, msg){ - if (x!=y) - stop(msg) - - -} -aboutEquals <- function(x, y, msg){ - if((x-y)>0.005) - stop(msg) -} +library(picante) source = "R/unifrac_cpp/su_R_s.cpp" sourceCpp(source) -table = "test.biom" -tree = "test.tre" -nthreads = 1 -b <- read_hdf5_biom("R/unifrac_cpp/R_interface/test.biom") -outfile <- tempfile() -write_biom(b, outfile) -bb <- read_biom(outfile) +data(GlobalPatterns, package = "mia") +data(esophagus, package = "mia") +data(HintikkaXOData, package = "mia") +data(Tengeler2020, package = "mia") # This dataset produces divergent values for some reason - Presumably something to do with the tree being unrooted +tse <- Tengeler2020 +rowTree(tse) <- ape::reorder.phylo(rowTree(tse), "cladewise") +ts1 <- rowTree(tse) -fname <- "R/unifrac_cpp/R_interface/test.tre" -tree <- ape::rtree(500) -ape::write.tree(tree, file = fname, append = FALSE) +fname <- "R/unifrac_cpp/R_interface/tree.tre" +ape::write.tree(ts1, fname) newick <- readChar(fname, file.info(fname)$size) -tree <- ape::read.tree(fname) -identical(treetest(newick), treetest2(tree)) -z <- treetest(newick) +y <- rowTree_to_bp(ts1) +x <- newick_to_bp(newick) -tree2 <- ape::reorder.phylo(tree, "postorder") -tree3 <- ape::read.tree("R/unifrac_cpp/R_interface/test2.tre") +faith <- faith_pd(tse, is.rooted(rowTree(tse))) +x <- estimateDiversity(tse) +faith2 <- colData(x)$faith -treese <- makeTreeSEFromBiom(bb, treefilename=tree) -treese2 <- changeTree(treese, rowTree = tree) - - -data(GlobalPatterns, package = "mia") -data(esophagus, package = "mia") -tse <- esophagus +faith - faith2 +#Checks +#This functions assumes that the tree representation is in cladewise order - Ensure this with ape's reorder.phylo() function +#Ensure that the tree is non-empty, etc +#Ensure that the tree doesn't get modified at any point +#Which assays are normally used for the calculations? -#faith = faith_pd(table, tree) -faith_pd_new(tse) -assays(tse)[[1]] -colSums(assays(tse)[[1]]) -exp = c(4, 5, 6, 3, 2, 5) +tse2 <- estimateDiversity(tse_hubbell, index = "faith") +colData(tse2)$faith +faith_pd(tse_hubbell) -equals(faith["n_samples"][[1]], 6, "n_samples != 6") -for ( i in 1:6){ - aboutEquals(faith["faith_pd"][[1]][i], exp[i], "Output not as expected") -} -print('Success.') +t <- ape::rtree(12706, rooted = F, tip.label = rownames(tse2)) +tse2 <- tse[[1]] +rowTree(tse2) <- t +tse2 <- estimateDiversity(tse2) +faith2 <- colData(tse2)$faith +faith <- faith_pd(tse2, T) +faith - faith2 -print('All tests pass') +data(phylocom, package = "picante") +x <- phylocom$sample +y <- assay(tse) +t1 <- phylocom$phylo +t2 <- rowTree(tse) +picante::pd(x, t1) +picante::pd(t(y), t2, include.root=F) -#Checks -#This functions assumes that the tree representation is in cladewise order - Ensure this with ape's reorder.phylo() function -#Ensure that the tree is non-empty, etc -#Ensure that the tree doesn't get modified at any point -#Which assays are normally used for the calculations? \ No newline at end of file +rowTree(tse) <- ape::reorder.phylo(rowTree(tse), "cladewise") diff --git a/R/unifrac_cpp/R_interface/test.tre b/R/unifrac_cpp/R_interface/test.tre index 1ba14a5b6..401910e9c 100644 --- a/R/unifrac_cpp/R_interface/test.tre +++ b/R/unifrac_cpp/R_interface/test.tre @@ -1 +1 @@ -(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1); +((((((((((((((((((t866:0.8313991029,t1687:0.3733365205):0.6214818014,(t3269:0.6072595785,t1686:0.7529063553):0.8813435675):0.2431555497,(((t2720:0.1146772795,t3354:0.0820990426):0.6279464895,(t1508:0.6595000362,t3157:0.5381378131):0.6036559318):0.9752331681,(t2936:0.2551624496,t3000:0.05830493267):0.2189676561):0.6016189293):0.6093351725,(t1409:0.06985254632,(((t3454:0.8042654931,(t4167:0.2065694619,t4059:0.3304703967):0.09274322307):0.1545934496,(t1963:0.8274634348,t4111:0.9223411726):0.1472488912):0.950080327,(t3893:0.9364278358,(t1832:0.9313541986,t3662:0.3542085034):0.9250311363):0.8204278164):0.2623061107):0.8409996089):0.05529688462,(t333:0.4158636159,t1767:0.5100084862):0.190012262):0.6363199549,((t937:0.1614484172,t4631:0.7026415281):0.9756822931,t3887:0.001032893546):0.2980521361):0.6100336898,t907:0.3821411079):0.1475626735,(((t1562:0.6701928617,(t3716:0.2011611089,t1016:0.3637938015):0.4790978113):0.6990956133,(((((t1107:0.6637271889,t2840:0.7656728218):0.517867452,t3661:0.4304010128):0.06343328184,((t3192:0.9852911846,t4126:0.3187654694):0.5443374345,t2336:0.8467219928):0.5904103168):0.1341467793,(((t2414:0.203484318,(t412:0.8888965137,((t1212:0.2144512064,(t4862:0.4041996542,t4283:0.03360790829):0.5145890177):0.708200976,t1790:0.9912529374):0.3069289338):0.7088423646):0.2650691359,t2795:0.3816543671):0.1875626091,((t2757:0.2675529977,t2965:0.4615569487):0.1818874718,((t4621:0.7309411902,t2619:0.849376152):0.987313323,t4410:0.3897798315):0.7727495972):0.4278311483):0.8209839279):0.005038926611,(((t675:0.1684947007,t1676:0.5381485247):0.7439194887,t159:0.4282397088):0.06210065517,t1678:0.4105465328):0.5742849279):0.3442878388):0.7197340357,(((t3296:0.320776033,t1552:0.7741130814):0.8737377529,t1738:0.1963442313):0.9038650303,((t2940:0.4459364363,(((t402:0.5717571357,t591:0.9301284561):0.636806218,((t3490:0.5776866795,t1304:0.4995453237):0.6047905236,t3083:0.7596928983):0.3777288443):0.3528932505,(t2372:0.8104584259,t4045:0.357436178):0.5526586915):0.3186718693):0.3538908418,((((t1876:0.8001238538,t277:0.9216230945):0.9068796132,((t758:0.6737462839,t886:0.1913464598):0.5354478974,(t1289:0.5047061227,t3972:0.7230309916):0.9546566226):0.7594838168):0.1208872946,t1267:0.03093142342):0.7116731715,(((t2896:0.6969843917,(t1879:0.8709465866,t1347:0.8908235182):0.1870362924):0.2875384493,t4303:0.5906736872):0.3181333772,(t1430:0.09074050607,((t2559:0.7986080786,t2581:0.2758466483):0.9652930978,t1368:0.1860396883):0.5633136057):0.7996817871):0.2368269465):0.1223834648):0.9785036482):0.1878155204):0.5875749625):0.6865005223,((((t930:0.1181095932,(t107:0.8333898354,(t3892:0.9600333602,t4747:0.4963327439):0.6568270572):0.7843931997):0.07698448002,t4362:0.3409487531):0.4231897073,((((((t529:0.04959728243,(t640:0.2147065429,t3690:0.2545317535):0.1329500256):0.7216223071,((t1673:0.6350189031,t3293:0.6712232393):0.6078203127,t4114:0.6816496283):0.7059944186):0.81562798,((t1752:0.1870343112,t1934:0.9348407013):0.4338724373,(((t4911:0.3117885669,t1768:0.9695757565):0.2889108541,t550:0.2876443232):0.9507387944,t3871:0.6463846464):0.9826903841):0.4953189236):0.3031298388,(t3993:0.8383750333,t3406:0.1060100815):0.8212537798):0.5591524122,(t2948:0.6562415836,(t2734:0.8999172021,(t1930:0.9211484026,t216:0.3518829397):0.2120475902):0.9303359622):0.3419920085):0.3668603834,(t1333:0.9122814275,t4649:0.6358980944):0.6920417589):0.6566554087):0.643499925,((((t1412:0.4486003867,t755:0.7555005001):0.3969241746,t1656:0.7176111068):0.6536936334,t4807:0.3639599544):0.118114522,(((t2120:0.9193203296,t1639:0.7436536453):0.483570592,((t3770:0.7877304046,t4979:0.06589412759):0.08375591203,(t4736:0.3814637288,t2052:0.3728892156):0.2851516092):0.0205797716):0.02622013958,t1084:0.9509191096):0.405228748):0.561063454):0.2318455605):0.1753108837,(((((t2646:0.12860857,(t1622:0.3871864763,(t267:0.8130197593,t913:0.6790412655):0.6844102743):0.05539356847):0.7421562918,((t1883:0.7993078497,((t3238:0.08534995257,t3603:0.1386850916):0.6240059605,(t3570:0.3704532592,t3492:0.4279712306):0.2013301745):0.02492270293):0.4694976134,t2392:0.6688378071):0.2137457884):0.2944884764,((((t1013:0.3968843073,((t4864:0.9482195915,t2061:0.3587614582):0.123466908,(t3766:0.612816578,t2902:0.08410423715):0.7703144071):0.4174562509):0.7675790566,(t1163:0.7610452443,(t2815:0.1097113073,(t2884:0.1855741069,t910:0.9298445894):0.06679133931):0.1308139546):0.8947804321):0.547407127,((((t3512:0.3123587975,t538:0.1997674562):0.0274069747,t1816:0.7843595201):0.518524423,(t495:0.2170114561,t1073:0.3725154614):0.1091845315):0.7535025117,((t897:0.8108963056,(t1137:0.693028131,t3033:0.518422103):0.4107569826):0.6008314493,t373:0.1386215745):0.9438396601):0.2902637431):0.5673053069,((((t1193:0.7558353972,t4176:0.3539376841):0.9614218306,((t4720:0.1573807234,t2818:0.8950932315):0.2238170567,t4291:0.3897347897):0.8315451082):0.9570792422,((t1601:0.3854548566,t2367:0.501024422):0.5245058786,(t1293:0.5080356814,t1278:0.1580390946):0.2233934382):0.9534459224):0.08009390417,(((t3761:0.3336408893,t2426:0.3739440746):0.03781672195,(((t4264:0.6076085074,(t3159:0.2337467058,((t4580:0.05722609977,t4074:0.1294833701):0.01331580174,(t660:0.107948028,t4994:0.3645382558):0.7614466671):0.8255050587):0.2813613035):0.07591580786,((t444:0.4889106292,((t848:0.8255068781,t2916:0.8064365082):0.9674706035,t4376:0.1939471222):0.1076038913):0.4793788702,t3356:0.2319602058):0.9083877718):0.05033111176,(t2276:0.05197685934,t620:0.5648543704):0.5495417307):0.9923970785):0.5722151687,(t2495:0.5662884507,((t3402:0.447677698,t4573:0.5410352189):0.229000591,((((t1222:0.758369239,t3167:0.3261838197):0.9416625514,t1659:0.04069067468):0.354748891,(t1484:0.5904282497,t4031:0.2863436346):0.02740597283):0.06279171817,t3090:0.2235296504):0.6806243469):0.4830548868):0.2040736931):0.3763294795):0.2272899386):0.3996402575):0.3156907097,(((t1349:0.4669708272,((t3318:0.9977994349,(t4585:0.007838176563,(t677:0.3869352438,(t4316:0.08979598852,t2193:0.530110654):0.6284318732):0.4792206499):0.5654797545):0.2588597636,(t4909:0.3334113173,t2011:0.8704887505):0.3976832477):0.1914889184):0.4892883177,((((t2464:0.002572481288,t3482:0.2588424592):0.04849005351,((t4233:0.2146493508,t3351:0.442112681):0.9735390993,t1900:0.899863557):0.8962532862):0.9084300497,(t4976:0.7234419258,(t4137:0.08894893457,t284:0.3383265315):0.2182320838):0.5241291963):0.1267592236,(t3338:0.09958937974,(t2081:0.09586508386,t4513:0.1785695285):0.03012143169):0.1738754734):0.6673604408):0.08750341623,((t2742:0.1556912579,(t4020:0.2850743537,t4359:0.9686899327):0.08356065536):0.3017442834,(t2097:0.02868690901,t3850:0.5237277283):0.9745898312):0.650786048):0.7844161796):0.172100046,(((t3899:0.8545834734,(((t1969:0.2020547255,t4950:0.8966408703):0.4309893181,(t1584:0.4535946152,((t3526:0.9639999992,t449:0.658337642):0.08031757222,(t3100:0.6227267974,(((t4494:0.1360878113,t694:0.9735614178):0.2148821759,(t2201:0.3367159995,t2539:0.9978895185):0.2363615001):0.9136687769,((t3701:0.3443828623,t198:0.8449363422):0.5596893611,(t205:0.1922649972,t4414:0.6131308302):0.998525921):0.7875632562):0.9447442961):0.094211024):0.09039981849):0.0942905338):0.3552417455,((((t3674:0.3699161999,t4430:0.5583526609):0.3636858882,(t3680:0.3514062983,t3129:0.3560908763):0.77759952):0.8376044335,((t4809:0.7839681495,((t73:0.8458224731,t832:0.02920071315):0.5244866598,(t234:0.1105332496,t3436:0.6492459632):0.9089311964):0.143582796):0.8149665371,(t994:0.1810347207,t4717:0.3273448183):0.3870441106):0.6867058265):0.6309708101,(t238:0.2335639638,(((t2829:0.6940432303,((t2078:0.5121471782,t643:0.9560798111):0.2380270909,t4077:0.9250784921):0.09147981368):0.4799288791,t3989:0.5019952122):0.8400811804,((t588:0.2799502863,(t2762:0.8330721518,t3995:0.2293385617):0.8058388657):0.6619244178,((t2277:0.03030529013,(t1103:0.417617064,t1628:0.7531720959):0.2904342064):0.9922625371,t1640:0.4167473505):0.7044557438):0.7944476453):0.01753724366):0.7580058079):0.1679762842):0.4854042039):0.6624881688,((((t4488:0.04132898641,((t3362:0.5572877568,t2428:0.01979416958):0.1260109774,(t4244:0.2547936849,t895:0.727302857):0.9251294867):0.1607061664):0.6319261612,(t370:0.9452616326,t343:0.06567107118):0.7694829579):0.3651739305,(((t3928:0.5820202043,t684:0.8006292286):0.722828218,t1733:0.6531603646):0.1611002039,((t2364:0.6117900054,t3740:0.7533898307):0.4633000528,t1406:0.8537637237):0.2380282378):0.9738505706):0.1402269127,((t2286:0.8612355865,(t2915:0.4517645189,t3306:0.7157090202):0.09892280772):0.09952634852,t4156:0.5061485644):0.8300967051):0.4571315604):0.6464888712,((((t4620:0.3717226819,t2398:0.4639170691):0.9069901905,(t1695:0.4494498344,t208:0.9338418769):0.3048537313):0.5753543843,(((t1408:0.3348016278,t2050:0.6979630752):0.07184065925,t4349:0.4378902102):0.2745788111,(((t109:0.5284655394,t1647:0.6463780864):0.9625444652,t3034:0.8764540514):0.9901140176,(t1290:0.7389137307,(t3106:0.397667265,t394:0.6857739007):0.3048634599):0.1374744785):0.5048887343):0.2137722648):0.9844458397,((((t2055:0.6795427711,t1986:0.197292648):0.2835855929,(t1498:0.7202436905,(t2503:0.9803528474,t1747:0.0265964584):0.3603068111):0.1440980178):0.2559785696,(t37:0.8919375236,(t2715:0.0001679745037,(t4916:0.6175938013,(t1922:0.665739248,(t4121:0.02225081855,(t4116:0.6575521091,t1315:0.6776197732):0.2996522097):0.2809470573):0.2966181994):0.1982580761):0.2591595659):0.7671651486):0.5420052521,t1173:0.5751995419):0.5408020962):0.09271502146):0.694344946):0.8642061879):0.4587742444,(((((t3705:0.3077488469,(t2683:0.3824585932,(t1031:0.492785126,t3137:0.6769487069):0.3193018534):0.4485238849):0.5818818845,(t3988:0.3966336916,(t361:0.6868546931,(t1684:0.7659938822,t4279:0.08717331616):0.5659515623):0.195033377):0.6608112429):0.6918420675,t4089:0.1229582443):0.5935022221,((((((t1759:0.05596134486,(t1499:0.7477170415,t2058:0.06373910606):0.08335241559):0.5618415982,(((t3092:0.4642444279,t122:0.3389575589):0.1902088546,(t3165:0.9979357789,(t1777:0.8572698389,t3686:0.3864898013):0.4624512338):0.2065343531):0.1222046658,(((t1467:0.2027243823,t785:0.5290673098):0.5791875266,(t1192:0.8304317927,t4285:0.868538467):0.8903516214):0.2709914381,t3784:0.6495769273):0.6268899313):0.4878179457):0.8749882865,((t499:0.3672033937,t2018:0.5924020063):0.8508136154,t250:0.0178107298):0.280602918):0.1264096648,t3560:0.344473344):0.1474100943,((((((((t2756:0.4914784722,(t3875:0.6713702285,t4329:0.1033865751):0.04412020883):0.6977141672,(t3691:0.1388739347,t1593:0.05767298979):0.2015471144):0.6463584441,((t4977:0.4073140486,t3702:0.05452403403):0.6306369812,(t1887:0.1061515368,(t3759:0.7181896823,t4882:0.5734710088):0.7374663292):0.1986337157):0.6505447451):0.2866082962,t2526:0.2340005359):0.3219974504,(t13:0.2337360389,t1791:0.2469362153):0.6399987563):0.05970709957,((t1064:0.3444667293,t4800:0.9214540147):0.528797467,(t3472:0.8128262968,t4327:0.5139588483):0.1445924458):0.7723366928):0.03411449934,((t4710:0.7008015877,(t1960:0.082305711,(t340:0.1880501874,t3439:0.1643805823):0.3643843364):0.2740489526):0.8135308253,((((t3398:0.2251789223,((t1445:0.4711492194,t4705:0.6156857519):0.6195273409,t485:0.8202841245):0.7617390025):0.9760007358,t679:0.2885267076):0.1918063064,t4452:0.3304648919):0.5278047847,(t145:0.8493148971,t1903:0.6771793943):0.5363263367):0.1391534698):0.307604121):0.5899123165,((((t4995:0.6808952286,(t2171:0.4406946807,t1848:0.3042994367):0.1491642012):0.4726482916,((t2925:0.6971553112,t725:0.1424836561):0.9414467306,(t2:0.04259154852,t2673:0.804576179):0.1907947834):0.7031442164):0.02356514777,((((t300:0.8495152174,((t1911:0.3079968672,t221:0.7993741685):0.8231502238,t4735:0.03782425635):0.3011443706):0.7581174325,(((t1183:0.2796791503,t479:0.703756982):0.3783575273,((t4253:0.7978001726,(t2290:0.2940325185,t4441:0.5233497315):0.4365729503):0.9907537305,((t4300:0.9818675399,t2282:0.2242055268):0.2348890863,(t4778:0.02383773611,t4899:0.4475373442):0.6738085677):0.3906179508):0.8571994142):0.1506352832,(t4975:0.09785921429,((t768:0.7420771497,t1965:0.7000401868):0.4112272328,t2003:0.07539929613):0.7877485934):0.8536505855):0.6284328497):0.4422431237,t4388:0.9619839406):0.06630207878,t4939:0.8316900609):0.8284955923):0.6957386476,((t126:0.2745769685,t318:0.515695011):0.8739172786,(t1806:0.4223237687,(t4235:0.6324440965,(t3558:0.3856438636,t100:0.1793875324):0.1103117014):0.6897586144):0.9797722392):0.07411500579):0.9114833896):0.1744418826):0.12724513,(((((t3575:0.2915715522,t2959:0.6054766681):0.2958707309,(t341:0.1462142987,(t2872:0.06567132124,(t4787:0.6459316753,t3186:0.3659973063):0.1336052534):0.4502089962):0.6560140238):0.7582268999,(t4984:0.5793036509,(((t3254:0.9246715868,t1862:0.4961994721):0.03015597421,(t4075:0.2329804187,t4443:0.01785692992):0.6949531741):0.752783861,((t2032:0.00182481599,t3081:0.2864647491):0.1948995111,(t1642:0.1777490822,(t4802:0.6750068944,t4822:0.9309473911):0.224436945):0.2201781396):0.1557234703):0.4853850524):0.8829760635):0.6089727767,(((((t3936:0.7378455061,(t2897:0.04230677406,t3949:0.1348307354):0.05313787027):0.3296164989,(((t3846:0.6774868004,t1854:0.8158837501):0.9599367611,(t138:0.8174523709,t3594:0.4482949905):0.665435472):0.6561711624,(t2941:0.9655892011,(t450:0.3526771103,t1975:0.8324833733):0.5348272631):0.5999133568):0.6406183194):0.6497490988,t1146:0.3918073028):0.8136717316,(((t1196:0.589256949,(t2107:0.9461472598,(t1078:0.08970693359,t2101:0.07596769161):0.5243451649):0.6314494654):0.7436189905,(((t1968:0.4095283882,t515:0.3888608604):0.1900956177,(t3762:0.4513915717,(t4458:0.05377129768,t2264:0.320200067):0.644646083):0.363355296):0.635781954,t272:0.844973563):0.1486973958):0.6539557155,((t2591:0.3961518875,t3425:0.4591378972):0.3509312924,t3772:0.7132560285):0.5538575212):0.7374948005):0.5583165744,(((t3437:0.5564842799,(t46:0.6903763306,t4938:0.0871099371):0.8569924438):0.01571567683,(((t4370:0.05613637832,t3307:0.2048978412):0.119604972,t3120:0.2856470875):0.7677832902,(t3407:0.4573363201,t1322:0.3836230936):0.1984996686):0.1137238124):0.6174768275,(t4730:0.03508606879,t2523:0.1745297287):0.7856323458):0.0273360936):0.1670195637):0.6524346559,(((t3667:0.9109485524,(t1152:0.6157433027,t1939:0.1478762547):0.4132024939):0.4341737363,(t945:0.09898446617,t4841:0.752896589):0.8002180101):0.8601645206,((t4478:0.544979098,(t4165:0.9513458982,t3528:0.274565479):0.6006933185):0.7104016929,t4635:0.8616838527):0.4258347338):0.161475023):0.3792145243):0.8458303383):0.1501436951,(((t2056:0.06283738953,t130:0.7353540463):0.3818885721,(t3431:0.4171808963,(t3289:0.6297804855,t812:0.6145251342):0.3355520647):0.3989941175):0.4776869144,t1610:0.5099943201):0.2747247163):0.5808562574):0.1173578743,(((((((((t633:0.7878978646,t4257:0.2668918124):0.058420585,t3858:0.3574693177):0.3867737106,(((t713:0.6144745413,t225:0.7666423055):0.9468104623,t3854:0.4185951184):0.8965026038,(((t1046:0.03403013782,(t4872:0.7471951458,t2751:0.3780142374):0.9674805063):0.4954787777,(t1620:0.3943154635,(t1835:0.3586856071,t766:0.1593085309):0.4414879608):0.6448796524):0.7341502274,(t321:0.06573951617,t78:0.9645420606):0.9772567479):0.1642606161):0.5015719032):0.009362907847,t843:0.9306513865):0.9098140819,t4133:0.1726960964):0.1627949611,((((((t3506:0.1555005824,(t2291:0.2415448872,t2218:0.6229850096):0.9834204547):0.0766422525,(((t2740:0.4675714979,t3267:0.5807356378):0.07195233577,t1839:0.3615350355):0.6904575126,t2152:0.0539963597):0.7748350685):0.5079670688,(t3105:0.762835742,t2725:0.783796466):0.9584548178):0.4012050428,((((((((t1295:0.4164196772,t2954:0.6073149303):0.2107647646,t2483:0.2546401017):0.5298662703,((t4601:0.4726096373,t2845:0.9699278455):0.1594873061,(t628:0.7959083684,t472:0.5038546089):0.2468013354):0.2405225963):0.2042652881,t2719:0.3846707568):0.9881216274,(t263:0.2444275522,(t1540:0.002669940703,(t2890:0.4910230506,t3113:0.1967816416):0.2833629383):0.6500252585):0.9340893866):0.4419223468,((t2604:0.4997490097,t2689:0.1908188728):0.397136529,((t1230:0.1887639058,t2606:0.6170728437):0.6400903785,t4779:0.05593576096):0.2662141183):0.04496151092):0.4585990051,t1598:0.4488763234):0.6067683478,((t1386:0.1602443401,t2467:0.2520359186):0.07220733934,(t2764:0.7036789996,t231:0.1407988446):0.6025765829):0.04527508747):0.4518896716):0.440780648,(((t4427:0.7291558969,t392:0.9098402502):0.324533311,(((t241:0.876763883,t617:0.7776813521):0.4155768962,(((t2014:0.2481024927,t966:0.5110307785):0.9683095682,(t797:0.006647384958,t3194:0.5751418984):0.7965638384):0.6731909767,((t1383:0.08988061198,t4905:0.3317841429):0.4784023084,t4354:0.4192292877):0.130420354):0.4951545659):0.6413437445,(t1254:0.8138528576,(t585:0.9455490054,(t2662:0.3696457266,t547:0.1125562578):0.9929754266):0.1361916685):0.5261483763):0.4947226157):0.6115640856,((((t561:0.3359035361,t4271:0.7219727971):0.6940529414,t2105:0.5294406756):0.796080841,(t2287:0.03226945712,t2982:0.5769619986):0.3669758472):0.7853436787,(t4773:0.6322223386,t976:0.6932521139):0.9681658773):0.1409364128):0.6987744232):0.03979036654,((((t1929:0.830397408,((t1051:0.02677565673,(((t3498:0.6135061702,t224:0.1955287277):0.6820306573,(t4776:0.1175722748,t1251:0.7412909265):0.2617493835):0.3223988065,t3377:0.4271865219):0.04042899003):0.6740744531,t1572:0.6555063031):0.9782287439):0.1330191679,((t1757:0.7890949878,t1039:0.573943957):0.5220484147,((t4961:0.8228227056,(t2732:0.02673610114,t1188:0.4030509922):0.5501612416):0.03512003901,t557:0.6656008158):0.9630228325):0.02071409463):0.03710257704,((t4058:0.8863248506,(((t4144:0.08660087571,(t3256:0.1293852923,t125:0.4784358384):0.1715465183):0.3329215932,t445:0.7716247349):0.2207148937,t69:0.5560288329):0.5100735258):0.7043115939,(t3136:0.5859558817,t133:0.8939665779):0.7172426854):0.5837200184):0.8319789858,t3069:0.5247035872):0.8014201785):0.1257841513):0.8516854437,(((((((((t3721:0.2272730847,t2524:0.7267203361):0.2073178284,t426:0.0979973576):0.2254999974,t958:0.755704436):0.3981743976,((t291:0.4075606614,t3639:0.8241939354):0.8926953345,(t2251:0.5661321448,t2157:0.7918906126):0.1668497066):0.548846822):0.9142585853,(t2184:0.07892514626,((t842:0.09095380595,(t2309:0.2447845724,t3066:0.3357016074):0.1155967922):0.6148679897,(t4920:0.067015148,t3388:0.6636415964):0.03361500497):0.5691323599):0.4181748333):0.4819492891,((((((t1844:0.8891954913,t44:0.1263200638):0.2889231648,t689:0.6517695789):0.9710279307,((t3043:0.3420520991,t4734:0.1014613379):0.08970208839,t1928:0.1404719288):0.2330005949):0.5202469786,t3527:0.3073822088):0.7455507996,((((t2873:0.993842721,t38:0.8716940666):0.9702959431,t4451:0.811977949):0.3911699778,t4219:0.8243935166):0.8491760052,((t1723:0.6181446293,(t3320:0.4218083268,t2378:0.5709121886):0.5222427058):0.6756005385,((t3117:0.4757901349,(t1667:0.03000710253,t1563:0.2898409474):0.7336020495):0.8879907588,(t1795:0.05369648803,t3545:0.5110886015):0.03172167391):0.5453401254):0.3470024748):0.2373066347):0.1232472439,((t2045:0.09022754454,((((((t2221:0.06582522835,t559:0.3750349162):0.9823703724,t2857:0.91887657):0.3647457191,t4769:0.4148848066):0.2218487649,t2599:0.2610909671):0.117852998,(((t3623:0.2997155234,(t2384:0.2523896666,((t4418:0.3204169911,t4210:0.6035194506):0.1483694732,t2862:0.1070660686):0.382917634):0.1145242173):0.8360710598,t831:0.6974025108):0.9243301519,((t1145:0.1953724711,(((t2462:0.02957250248,t2420:0.7349497643):0.4452858581,t4450:0.5839691546):0.833532796,t4169:0.975084123):0.6798284068):0.9351414097,(((t3315:0.5507207906,t3260:0.01705354918):0.7518011779,t518:0.9714504066):0.5806051695,(t1564:0.7447547193,t4506:0.5968745956):0.2476783139):0.4340931242):0.4297363679):0.7803913141):0.6385204501,((((t2904:0.7438715228,t3027:0.7617073336):0.2069750784,t2889:0.1843298285):0.4033136379,t4479:0.7930720742):0.2364758458,(t313:0.8746632556,(((t1462:0.01935528778,(((t4791:0.8398026014,(t2301:0.3197807434,t1038:0.4504582621):0.8727893296):0.8012714216,(t1551:0.227017323,t3554:0.8384136984):0.7671152134):0.636139391,t2369:0.3707443958):0.607876146):0.9290237429,((t4206:0.8721011363,(t229:0.5312422623,((t288:0.7717724796,t552:0.2046030276):0.4665413366,(t4193:0.6593752354,t3097:0.06107479008):0.4428279179):0.2099730223):0.9599901461):0.4818946796,t1997:0.4558941461):0.1404252206):0.8787578936,t3216:0.716959503):0.1899293421):0.2787984787):0.9139637328):0.8343013593):0.6285247961,(((t1028:0.9913741208,t88:0.7136295764):0.165553801,t1565:0.7906226674):0.06138058635,(t2731:0.9538947833,t641:0.4709784789):0.8748916266):0.02064642939):0.5488434534):0.516574932):0.5289227734,((((t1755:0.9561335959,(t3279:0.7558561666,t43:0.4442185273):0.2835182364):0.7017092011,((t1186:0.2810068065,t302:0.8027740314):0.5610312901,(((t4574:0.890135447,(t3265:0.2827178964,(t1623:0.9504110247,t1030:0.7499874001):0.8045510368):0.2839150925):0.6179945848,(t3904:0.762395394,t3763:0.8478405366):0.9658707301):0.2542268585,(t4927:0.7666204537,t2354:0.8009463164):0.20428839):0.918566271):0.3655805727):0.8375776128,t1797:0.6570838718):0.796043894,(((((t154:0.8041998569,t2016:0.5771751257):0.2804442178,t4055:0.6514532836):0.7740242362,t4161:0.2860580245):0.2508791196,t2860:0.1996267657):0.7764814005,((t1424:0.378427458,t4587:0.9449776406):0.5118783186,((t4857:0.5625643381,(t2643:0.7123449885,t489:0.7666595853):0.03049568855):0.1289974691,(t49:0.6853816502,(t1391:0.7640693565,t532:0.3948962637):0.809085659):0.3425395396):0.9466729695):0.8294895785):0.04192236811):0.04168931209):0.7047592879,(((((((t22:0.1062093936,((t1608:0.5267388825,t651:0.1217423705):0.5899810842,((t4287:0.5227646604,t2521:0.5014049015):0.8743285942,t1241:0.9668636306):0.4575376154):0.03729532915):0.2942101231,(t1444:0.4691511907,(t4198:0.3791185436,(t1334:0.8677457774,t1740:0.6761152355):0.8855947256):0.2441808309):0.5557023743):0.9297091982,(((t2994:0.07313275663,t3835:0.590317568):0.6060474706,t4380:0.03807822103):0.7266947783,t1214:0.1094867818):0.5525156616):0.3165107092,((((t1689:0.8687409447,t482:0.2549425911):0.5812358698,(t303:0.4390058708,t3353:0.5086138793):0.3762504952):0.01645651134,(t4571:0.04501585197,(t4575:0.2743993851,((t3727:0.6485891575,t3872:0.6394018608):0.08933446277,(t2164:0.4854876618,t4368:0.9923337493):0.6518029273):0.2402326344):0.7225597147):0.1010865043):0.5105761525,(t1875:0.06032745005,t816:0.5409573084):0.008607164258):0.2561405229):0.3016850099,((((t1523:0.8558590831,t4840:0.294845476):0.2742795113,((t2859:0.7109305528,t616:0.1376374094):0.2209257833,((t4358:0.7087082041,(t1416:0.1961699151,t4523:0.2965116177):0.8787157112):0.8280309597,(t3902:0.05308877141,t3852:0.4151797926):0.235782797):0.7114830553):0.9277588653):0.7383752821,t4561:0.2057104243):0.03686085646,t1609:0.9181670593):0.1685556176):0.05538826296,t2031:0.7053138402):0.7046458928,(((t3226:0.0102605545,t2117:0.3897832988):0.6513349356,t3401:0.5885010974):0.8343021218,(((((t4934:0.1817306485,(t2123:0.7340407188,t2454:0.6104232282):0.1886806476):0.8746837541,t4986:0.2951512481):0.4614188874,t2195:0.2262022588):0.06382826786,((t3501:0.9004146492,t3446:0.5758411724):0.9606548897,(t347:0.3655740756,(t3622:0.758681197,t3665:0.9648439176):0.5645550238):0.4364375959):0.9828783802):0.4975256296,(((t2803:0.2527835162,t898:0.3529181536):0.1843255248,(t27:0.6882137144,t4046:0.02794211241):0.7162194757):0.4623265967,(((t4609:0.814070601,((t4062:0.1450867835,t4828:0.09026327729):0.4442945814,(t4771:0.737855226,t4404:0.05066533503):0.006584957242):0.5666926212):0.302958742,(((t1585:0.564349792,t664:0.09179626498):0.966538725,t2147:0.5980658231):0.8023969457,t2508:0.06598238577):0.691382552):0.442074684,t3294:0.64118041):0.6807374794):0.7692557098):0.7764729986):0.442435506):0.8189105245):0.9433969869,((t4929:0.04752131505,t2012:0.9867965637):0.9948739554,t2455:0.04473616625):0.1120181805):0.1489457684):0.6889146112,(((((t476:0.2264062176,t4684:0.2312254407):0.3889585189,t1644:0.7865779568):0.9279815371,t4113:0.5871377555):0.8840677089,t3847:0.4483252489):0.2803470888,(((t2512:0.6852147239,(t35:0.06016547373,t908:0.6581263512):0.5731073944):0.385760973,t3849:0.333136999):0.6110870452,(t534:0.8452889619,(t3517:0.4495121788,t2119:0.4179212325):0.663919671):0.6861887071):0.2076667706):0.7435675948):0.4227561087,((t3788:0.3898302007,((t1119:0.4965937783,t3589:0.3181228347):0.9094769817,((t256:0.3326093024,t2085:0.1543390448):0.7657084488,t2248:0.6890798693):0.588582275):0.5533623886):0.5621604042,(((((t709:0.03085775184,(t2111:0.2596664443,t2810:0.6147328482):0.995723933):0.3582475041,t4889:0.9339648089):0.5444564556,((t3832:0.783544641,t2767:0.104425156):0.4558330271,(t986:0.6722481819,(t4871:0.1085624942,t1604:0.7791883154):0.4663334307):0.395669759):0.3671582295):0.3892007871,(t86:0.6587422066,(((t2113:0.9756236989,(t4881:0.7471210759,t3655:0.3530885589):0.469051074):0.824929679,((t1921:0.3051331437,(t4021:0.9797260971,t2208:0.8523167975):0.4366853214):0.5225139929,t3580:0.0625502828):0.3908204229):0.3321972652,t2066:0.4542768572):0.8504227449):0.9353928568):0.07003547181,((((t3838:0.7098855111,(t4554:0.8891642259,t2288:0.3118351218):0.3736097808):0.161281171,t927:0.3298595808):0.3784930583,(t781:0.6679566738,t849:0.6259508296):0.1130717548):0.2727484729,t4065:0.9912141578):0.1440196547):0.2147689259):0.7043501181):0.07248589955):0.530497049,(((((((t3769:0.6466846738,t672:0.1033081454):0.6266500417,(((t955:0.2273717627,(t3352:0.5579966898,t1339:0.01964822924):0.3156101445):0.3591132183,((t1745:0.1167644225,t632:0.5405399653):0.6658276576,t3050:0.76286367):0.7346331722):0.5164513744,((t1878:0.4324891842,(t4087:0.3476188192,t2584:0.7207941094):0.6210414676):0.2790375431,((t1996:0.1042214141,t3344:0.5070581199):0.2879082335,t4645:0.8558622005):0.4922160259):0.03505943157):0.6815868346):0.4029777001,((((t460:0.4704633558,t172:0.2671403964):0.686816779,(t2323:0.1825801868,t1010:0.1803271479):0.6091568777):0.2921220942,(((t1724:0.3313620081,t4464:0.007205237402):0.6120055735,(t1894:0.3802170276,(t1420:0.2573057683,t1697:0.7206129674):0.9375002088):0.1827242058):0.1120939164,((t2794:0.4763097398,t85:0.1901003367):0.08367978805,(t2781:0.5407154397,t3262:0.04304612358):0.2294769997):0.8126289344):0.08153182222):0.3462535993,(((t3823:0.5676665765,t3619:0.7623456235):0.05221901299,(((t875:0.8342969231,t4510:0.01280949032):0.4805617358,((t3237:0.868317923,t2516:0.3559935484):0.2391033964,(t96:0.2634183168,t3078:0.5413043841):0.4104156578):0.6433246587):0.7133132869,(((t916:0.4961097147,t1710:0.5212233118):0.3681945752,(t4060:0.4648038673,(t494:0.4682004587,t4558:0.2630725261):0.9225172596):0.8050615776):0.4532664379,(t4079:0.1588851346,t2602:0.09221257851):0.4393938517):0.03791397042):0.9239859581):0.2933065468,(t3022:0.6358551804,(t729:0.2333274339,t3212:0.1765346471):0.6913841155):0.148070453):0.357521517):0.1009253073):0.1452931252,((((t2548:0.2236656782,(t815:0.8213350857,((t1055:0.8640718691,t4634:0.6664643853):0.7207097509,t2589:0.2309196405):0.4839087222):0.6354335675):0.837207197,(t1926:0.3605173982,t4132:0.8602008261):0.2074110394):0.7975981371,(((t4650:0.6114310657,t1480:0.5927743833):0.7775010511,t1815:0.7387591815):0.093524531,((((t3952:0.3417194458,t3025:0.3344225893):0.3593524352,t1458:0.9698847521):0.6445904365,t4591:0.5833607835):0.8164989168,(t462:0.1087539014,t2621:0.00215459778):0.5499014652):0.5343262032):0.2478236798):0.3348515588,(t4627:0.4438257637,(t3544:0.4862496899,t3644:0.3581603325):0.6639149655):0.798017679):0.06999673764):0.5962453596,(((t1022:0.3090051059,((t1052:0.3328659611,t2115:0.3883270295):0.9024150395,t3579:0.1267078514):0.3432275653):0.5559913013,t90:0.4690397023):0.2694137404,(((t2452:0.2666485338,t4151:0.8441270906):0.1241979003,t3658:0.1196388621):0.2930851004,t1613:0.8959609733):0.9758957163):0.3256739243):0.318873889,((((((t959:0.3091299231,t3548:0.6373040627):0.877055404,t2240:0.9956788805):0.05453642411,((((t1110:0.4505025849,t3478:0.9588692153):0.9506132605,(t1262:0.7979903871,t1798:0.8417904859):0.8500949503):0.03177290736,(t3616:0.8490903568,t4107:0.7680919052):0.01018177648):0.1415458149,(t2280:0.03998941439,t4367:0.738323306):0.3047480215):0.4852210497):0.4157199706,t3568:0.01037793071):0.7840320186,(((t786:0.07067953167,t4476:0.2343864425):0.07798373513,((t1366:0.9256579953,(t3280:0.9086080925,t1168:0.00243516732):0.9665693233):0.9192321084,t3394:0.2588717935):0.07120910799):0.7155589776,t583:0.6690537904):0.2810324449):0.5849530844,((t3933:0.6101319897,(((t2788:0.637666493,((((t2432:0.1340591561,t312:0.9343538997):0.903231292,(t4100:0.1527151116,t1677:0.2576732035):0.5319194936):0.6589867629,(t4550:0.559303558,t144:0.0886226478):0.1319019839):0.8748646763,(t1561:0.9570078254,(t2179:0.4851890197,t1477:0.6949043816):0.8570922611):0.8941170226):0.1368041006):0.6804844884,t4378:0.008004796691):0.9268674203,((((((((t3696:0.4309293772,t2573:0.8052610301):0.7855452653,(t868:0.8255533013,t233:0.8835596314):0.3080080952):0.2007256106,(t2789:0.5761511396,t1800:0.8347072047):0.428101057):0.005547654582,t227:0.8791453352):0.4778623141,t4974:0.6944202664):0.7070723181,(t3749:0.3686779568,(t1276:0.3139098885,t4949:0.5140190755):0.0808090663):0.7302102458):0.2413145641,(((t1091:0.3533034031,(t3310:0.1232876047,t2116:0.628836398):0.4255135322):0.007262541912,(t905:0.7864620169,(t2060:0.5953573496,(t4565:0.7373510567,t4296:0.7121330944):0.5411335668):0.229283927):0.6448725243):0.3898677453,t1247:0.005315080052):0.5133132506):0.3200960087,((t4624:0.2680101108,t4426:0.8724194732):0.3928657519,t2448:0.8980655884):0.8232885108):0.6005251962):0.492095921):0.0661570658,((((((((t1892:0.07490668516,t1300:0.7155320027):0.6556291892,t556:0.5938144561):0.9076140674,t1495:0.8580564891):0.1105535126,(t1553:0.6643014061,(t3084:0.1838265697,t114:0.9746948618):0.6744990337):0.2293225429):0.3844142871,t175:0.04850835819):0.4151965023,t933:0.563647782):0.2607754976,((t4508:0.7912175111,((t3634:0.1404989895,t952:0.1977472259):0.04340601573,(t680:0.9767462285,(t2898:0.9942323398,(t4667:0.008750366513,t3204:0.06591424509):0.1737183398):0.0230799003):0.6838204332):0.06915117893):0.1759667194,(t4805:0.3751401543,t1421:0.5349217146):0.08212707122):0.01960918121):0.9615102094,((t1818:0.02239792328,t4493:0.8646344917):0.02888329653,(t1422:0.5962255253,(t332:0.5765972435,t3947:0.9994824568):0.3736898096):0.725945405):0.5272825065):0.9426176234):0.462855007):0.4692642924):0.6231946275,((((((t4584:0.3446172159,t3324:0.2740713169):0.6993737789,((t4416:0.7008554828,(t745:0.7204409756,t4466:0.03135086969):0.6014024185):0.4866102792,t2289:0.1004670695):0.5513202175):0.259507034,(t1784:0.4614843968,((t401:0.3483010333,t1318:0.1402908785):0.3985952279,(t4764:0.8497869105,t663:0.05895327055):0.05699259927):0.2789120059):0.5578124062):0.4990431448,(t4197:0.2207745444,t1190:0.05646117101):0.3738585685):0.7549968797,((((t1701:0.3322706043,(((t1905:0.3732423827,(t757:0.4035894233,((t3242:0.7552078962,t2403:0.5936345106):0.1508447032,(t4178:0.61984832,t3728:0.7685682925):0.3778034118):0.7662743039):0.2142664462):0.117607801,((t1023:0.9588375415,t4861:0.1600330039):0.2320436148,(t3035:0.8584500449,t3125:0.6009808923):0.441564067):0.8285019302):0.6594157638,((t3217:0.487779933,t4642:0.6032790754):0.9606279701,t438:0.6125991954):0.8090999648):0.8741976963):0.5877119387,t3193:0.3671189328):0.2184245391,t3979:0.1792570727):0.3983644147,(((t1165:0.7263523617,t890:0.2832219426):0.07846978703,t4454:0.5662037719):0.1181382251,t47:0.9923386106):0.5804408917):0.220135994):0.007112539373,((t2868:0.2712685561,t3937:0.275737674):0.1641175735,(t4056:0.768098864,((t42:0.2365621892,t3567:0.6706970748):0.1907827624,t4983:0.8764561741):0.7494066874):0.4942596904):0.5996063228):0.1864717037):0.862709363):0.2909562169,(((((t4664:0.9304566951,t1417:0.05257266574):0.9538545979,t715:0.7549683796):0.2979818108,(((t639:0.7151184981,t3234:0.547814091):0.8006778057,(t790:0.01991344942,t4124:0.256110417):0.537971291):0.2506390414,((((t4552:0.5155060624,t4201:0.8163021519):0.3725403999,(t414:0.4584031648,t3991:0.8911658113):0.4593221471):0.7854233689,(t2841:0.4291381338,t292:0.8226222498):0.9112290922):0.9897106818,(t3476:0.4077175015,t1132:0.6122271449):0.8860064833):0.01477308688):0.5107039129):0.888281065,(((t4854:0.1313291453,t526:0.07222869736):0.2509720894,t670:0.4472576221):0.766625968,(((t1793:0.2389741635,(((t4863:0.9820780463,((t4926:0.9223933427,t1884:0.3346033227):0.05232663313,t4001:0.1193057916):0.8076331932):0.7739556783,t3549:0.6004829931):0.8882187577,(((t2239:0.3772761128,t2250:0.2095319733):0.1052006851,t1240:0.6904255881):0.6335157263,t603:0.5203551303):0.9040458892):0.1352078444):0.8636435727,((t143:0.8836502237,t4280:0.3276579389):0.01877145306,t775:0.5463361186):0.05106935976):0.9104602209,(((t734:0.5162792953,t4704:0.7455891052):0.1655608963,(t3857:0.9605378576,t4953:0.9289893166):0.1238029888):0.8203120271,((t2296:0.7620274387,t2969:0.766865572):0.4544611631,((t1011:0.3972623609,t1225:0.05121889338):0.9187192614,(t3898:0.9651806743,((t3403:0.8659662877,t2570:0.08471553889):0.323113448,t4566:0.2656651421):0.4838454106):0.1887069202):0.7914218155):0.2909786284):0.6412001203):0.3802099496):0.3064949738):0.880862786,((((t3341:0.3332408047,t2515:0.1978242584):0.8614687233,(t1461:0.861720603,t3843:0.9288190815):0.3957614873):0.69364743,((t4812:0.7317738635,t3652:0.3375414284):0.565154887,t2544:0.7843509533):0.4330095451):0.7054571616,(((t2541:0.4933483594,((((t3923:0.4715760476,t2755:0.8865578156):0.2502611224,(t119:0.3522350364,t2989:0.1754289723):0.6067595079):0.2132098943,t975:0.212742205):0.2965624079,(t1775:0.3828190726,t665:0.05458779749):0.005589244189):0.2234592594):0.6484751042,(((t375:0.8568919525,t2041:0.7244076359):0.4808880824,((t3774:0.7240769605,(t150:0.3972217536,t3011:0.7937627106):0.9788834793):0.4871354501,t4080:0.988321912):0.3351018999):0.5312720148,((t383:0.01207726309,(t3499:0.2568179797,(t299:0.8896732756,t385:0.2647393481):0.667279324):0.3519880599):0.1533683233,(((t467:0.418780687,t2498:0.6713116074):0.5403868537,t1035:0.2980335103):0.06631625933,(t106:0.5664194273,(t204:0.8044231657,t1527:0.3726060388):0.8450954028):0.09663877636):0.2212472481):0.6314126034):0.9628856818):0.499548621,(((((t3434:0.02921271767,(t661:0.6966851011,t3641:0.342011869):0.8244890231):0.5888657626,((t508:0.4760097784,(t1681:0.2457318238,t4181:0.9818324908):0.6673686649):0.9187927989,((t3020:0.1534836716,(t4135:0.1878414021,t596:0.764049435):0.2463693663):0.0377906668,(((t355:0.830097822,t683:0.631418912):0.7811487305,t4278:0.08528722147):0.2779802149,(t1863:0.1677124577,(t4683:0.6648355403,(t3429:0.4877056612,t3323:0.3100777254):0.02398575284):0.6693547082):0.8219353205):0.462421733):0.8788085955):0.9803359851):0.4286797429,t1942:0.7418841412):0.6818343839,((t475:0.8252504314,(t2655:0.9207634099,((t4729:0.1058409377,t3681:0.647373857):0.5800706963,((t4958:0.6524147743,t2040:0.5286924397):0.2672304942,(((t95:0.1119787451,t3608:0.8565463028):0.8246072433,(t2847:0.5987734403,t4444:0.9840675676):0.2915324722):0.392334224,t4148:0.8568997411):0.6183971874):0.9127077053):0.5213467709):0.7230928952):0.1333552203,(((((t2232:0.2205803443,(t3557:0.2978168651,t1506:0.3850140183):0.06996091548):0.6449486064,(t431:0.6946327062,(t1324:0.1068797028,t4391:0.292209873):0.004252291517):0.1702999678):0.8252241763,(t2062:0.9994597856,t2175:0.213354528):0.74027936):0.3237326844,((t1544:0.5699626633,(t1460:0.4113790395,t899:0.08304075035):0.54801297):0.1936414354,t3798:0.4422166443):0.06655659946):0.2460256666,((t2391:0.8475717674,t4343:0.3584974871):0.342210504,t1050:0.8479827428):0.1725088675):0.8290164387):0.634862249):0.4533321522,((((t4281:0.5646155793,(t2067:0.5218651907,(t653:0.7031459841,(t1104:0.9791517481,t2919:0.5508994241):0.7258873556):0.2585143552):0.8625433333):0.5460079182,((t2441:0.8395343409,t2244:0.1548981154):0.9797443727,(t4162:0.2197723729,t2792:0.9625409972):0.3465276305):0.8702985502):0.7059335867,(((t853:0.8620653683,t4390:0.03419136116):0.9327281343,(t1361:0.2163218269,t1491:0.5646170527):0.6218528966):0.7057344399,(((t567:0.8550386804,t791:0.2068807057):0.7368793134,((t637:0.08879817254,(t1638:0.3268671932,t3064:0.4647028318):0.5596295425):0.6141556352,t473:0.798001904):0.473791338):0.4576970944,((t4782:0.3626284255,t4323:0.1962700405):0.8224465291,(((t3657:0.9684312907,t3861:0.1475586966):0.5796871684,t1967:0.01208710275):0.5235993129,t1469:0.8825382099):0.03749012272):0.5428821018):0.5300997873):0.1519905326):0.8845694123,(((t5000:0.6542834463,t4596:0.3955448049):0.1582832257,((t2502:0.1293041811,(t4757:0.5799618098,t4341:0.6409835743):0.4716792221):0.2549489588,t4796:0.2944373179):0.7562957068):0.4390988136,((((t1177:0.7042823255,t405:0.6745223184):0.3263553947,((t2043:0.7949909673,t4933:0.9306182039):0.5296911064,((t1889:0.7580958125,t4804:0.3421752078):0.1580273192,t4112:0.7942998982):0.9547303247):0.4788878018):0.1389126801,t4463:0.8718663051):0.7158308181,((t2362:0.1835035582,t753:0.6578063713):0.7754279808,t497:0.3557105674):0.1888224029):0.5325981814):0.3214960583):0.1916291704):0.7138409382):0.9680554771):0.5993425802):0.7299016058):0.4045579233,((t4353:0.1107426004,(t2706:0.1409204477,t1158:0.494018099):0.1074948939):0.3680671498,((((((t2471:0.477228865,((((t190:0.6279092438,t1833:0.4570953285):0.1860131605,t1653:0.5785431438):0.4694334727,((t4139:0.3854422728,t4799:0.8529077165):0.5567523644,t599:0.4512025889):0.6665724323):0.7083741869,(t3756:0.8943691009,t1423:0.009874930605):0.5635304069):0.460964622):0.6377704144,(t2681:0.6104491751,(t3057:0.05157451704,t1943:0.2220654311):0.4718299431):0.3266741713):0.3740884187,(((t1870:0.04568355624,(t3139:0.5958024331,(t578:0.3606470337,t1651:0.1003955645):0.6571315897):0.7115604032):0.3396621484,(((t4109:0.737479209,(((t1338:0.7933644499,t2687:0.8810002108):0.37037879,t2578:0.6125737969):0.2917059367,(t3225:0.02518131561,t3175:0.04199181893):0.2581833464):0.5016670155):0.8108378814,(t4449:0.6662035922,t4537:0.5077769589):0.5667670546):0.06295551523,((t4660:0.6646387281,t2071:0.6016214364):0.3256749455,t80:0.7274544344):0.2224477578):0.6685227843):0.9571340487,(t3366:0.2648028,(((t1273:0.7439932788,(((t3218:0.448566528,t326:0.2594898166):0.6261468036,t4205:0.4008740869):0.266143691,(t1449:0.315638005,t2625:0.6102447689):0.1348877717):0.2770228032):0.9484447937,((((((t1648:0.0710146015,(t1244:0.8875683583,t4277:0.9055026341):0.9926929721):0.9354329193,(t3848:0.4621008357,t2233:0.8691533813):0.6097526776):0.9697218987,((t3571:0.3002396666,((t4421:0.7226957385,(((t248:0.6331399546,t3469:0.5242302737):0.5434815402,t2039:0.8367426426):0.7528521668,t4073:0.4847606472):0.9986144644):0.4991449337,(t2555:0.5412526568,t3782:0.4691321114):0.1567828935):0.3965436046):0.5747563448,(t2155:0.2956574522,t3276:0.562233102):0.4276773022):0.9195994914):0.2654775479,((((t4029:0.8716457572,t4551:0.5098797474):0.1592948956,t3625:0.8528882326):0.9010232496,t3706:0.8191902256):0.348420091,((t2506:0.5884979342,t3419:0.4183517862):0.4663748727,t4839:0.4952596903):0.05719872564):0.965208919):0.07442370267,t4869:0.3335710757):0.2371746358,t3537:0.384530978):0.2411411754):0.3683480818,((t3442:0.4070422184,((t3971:0.04083079286,t4697:0.2762315597):0.3199129221,(t1403:0.1763692936,(t3371:0.1556898048,t2249:0.6660126662):0.1512764152):0.803617235):0.473045357):0.9722921816,t991:0.603489609):0.4174242041):0.7283605086):0.5506962622):0.09124358324):0.9464069572,(((((t2400:0.9705576329,((t2104:0.4208654726,t2335:0.4269797488):0.9772462484,((t4952:0.8789142785,t2436:0.2013784337):0.801069631,(t4877:0.3199565767,(t2671:0.1940777826,t3420:0.3889506406):0.8898991027):0.2412282587):0.7203557712):0.8552576851):0.7778617404,((((t1896:0.6637700479,t1754:0.9356380405):0.8179022677,t919:0.9822418476):0.5630755189,(t1787:0.8135371506,t4669:0.01248314418):0.2223929283):0.02853894513,((((t859:0.3312765246,(t2126:0.6147523939,t3775:0.7615870694):0.6444736742):0.2489153333,t3427:0.9375454322):0.2014391206,(t4028:0.3361945648,(t1144:0.5410954524,t3736:0.1249863813):0.3851700076):0.15557622):0.5414525308,(t2793:0.6484910289,(t3360:0.8046605496,t876:0.3311111818):0.4460008682):0.5736540004):0.1743675778):0.8788767331):0.7930357086,(((t990:0.6368215128,((t2545:0.8928038324,t295:0.07223393559):0.6814632406,t730:0.8002198073):0.3362920096):0.8674865395,(t2427:0.01845167112,(((t1486:0.9996755584,t1252:0.2028291838):0.9960522573,t2042:0.04575532489):0.5237805296,(t1972:0.4059354018,t935:0.6507278581):0.8355227534):0.1522516466):0.4286695353):0.5273843405,((t4255:0.4543393997,(((t4420:0.2971841071,t545:0.3332000007):0.01865118812,(t94:0.7683817795,t3692:0.8107102367):0.5053617253):0.6078841612,t2787:0.8389514058):0.6196948041):0.1145944379,t3026:0.5628551843):0.1219715753):0.5296312575):0.9698127317,t4187:0.8519778538):0.07975409017,((t1351:0.3331482511,((((t4072:0.5070701337,t1471:0.3583125344):0.6869560846,t399:0.5557919878):0.3168950486,t432:0.6126366067):0.7207304907,((t4781:0.6262775937,t1314:0.7919575688):0.8411403566,(t4763:0.5874064958,t1649:0.6437614192):0.8352220107):0.02254280518):0.3803870066):0.7237869536,((((t3202:0.4554702342,(t3423:0.4873491582,t3169:0.2562238786):0.4213889998):0.6376321344,((t2224:0.7993498789,t825:0.3101734177):0.9971639239,(t2779:0.7520597123,t4047:0.4295145557):0.5429032641):0.3204396437):0.09376283642,(((t2204:0.868632175,t2907:0.0110831738):0.169708889,((t605:0.9111692719,t358:0.567261819):0.05806265236,t2482:0.6795250291):0.7371255949):0.6812536654,((t3836:0.3588431552,t316:0.1484292119):0.04866353865,((t1456:0.4573692996,(t3685:0.4049420615,(t1732:0.03175419802,t63:0.7370199659):0.04544956447):0.4321082784):0.4295368057,(t513:0.718055706,t4810:0.7073802983):0.2262096833):0.8606561651):0.8932628231):0.5201998306):0.7852239942,(((t1256:0.9519475286,t4925:0.302369585):0.2312221755,t2229:0.9986906303):0.6951793965,(t3428:0.9815349209,(t3122:0.9759477731,t2349:0.1864664955):0.696283869):0.9521299626):0.7049105288):0.6249511011):0.3216903589):0.8987392392):0.2801516475,(((((t4496:0.4463092897,t752:0.3158174364):0.867041653,t4775:0.7598986188):0.8731350999,(t3168:0.178622514,t1313:0.3941091052):0.3326715976):0.2848390217,t3072:0.3147836681):0.446998256,((((t3792:0.8984242238,t1058:0.1500867321):0.7425107306,((t780:0.05250215554,t4097:0.2417020369):0.7772540979,t289:0.1198037756):0.2988460825):0.119683774,t1758:0.2234687479):0.6866154983,(((t1857:0.1844939264,(t498:0.3224776275,t2642:0.8830989553):0.4936092405):0.3093467366,(t1200:0.9315602698,t4465:0.06757676019):0.1046388031):0.8706031714,t655:0.01064087218):0.3891302701):0.1128686606):0.4859765146):0.835902421,((((t2279:0.9717465041,t3886:0.4193502069):0.06555824843,(((t1605:0.7753756978,t1730:0.4917756831):0.5128283896,t2998:0.8065010493):0.7042934997,((((t3982:0.9509645812,t1510:0.3448941733):0.829875448,t718:0.0721514686):0.07321393304,(((t3895:0.7212343053,t4436:0.4256257957):0.4148385255,t3919:0.8034451739):0.7761058277,(t4874:0.2431940178,t2654:0.4809309542):0.4536632861):0.1276760825):0.8771003252,t826:0.9029375031):0.5793730048):0.3272070892):0.8232686038,((t574:0.6935549122,t4431:0.3369199848):0.6660736643,(t2632:0.7172376872,(t3284:0.2890637992,(t3760:0.610869026,t2661:0.3828362881):0.9132228284):0.6985175717):0.2757551523):0.305714546):0.9497868454,(((t3281:0.7757596613,(t1041:0.927387987,(t21:0.4005575341,t4104:0.8487283036):0.09940630058):0.1363720093):0.9446371587,((t1980:0.9734664827,t2615:0.5329524074):0.943363965,t3946:0.4273997399):0.4193501268):0.5490211681,((t3397:0.2102524454,t4616:0.03972084937):0.3606578496,t4088:0.5346075103):0.5938322914):0.6471076494):0.4116916063):0.5551993193):0.5818428893):0.4733014414,(((t56:0.4686750816,(t4171:0.3328116143,((((t67:0.9993218847,t58:0.3014979505):0.2153624303,t4801:0.858343744):0.1397100568,t2424:0.5627536532):0.3531578565,(t4487:0.120865172,t4896:0.9840020852):0.722391112):0.584701688):0.08989368798):0.3315123618,(t1762:0.9500000593,(t4136:0.7692101826,(((t3585:0.1911480248,(t1325:0.001689209836,t2255:0.8488115035):0.9699812687):0.4517220745,((t1841:0.7408196907,t1873:0.1788257684):0.8215829337,((t2877:0.01625809073,t298:0.3340431126):0.5077751833,(t1587:0.6160635191,t1063:0.4995452394):0.6997666017):0.4765272953):0.8427055916):0.7527390639,((t2694:0.630414933,(t2865:0.3308303428,t3804:0.4585468143):0.5944606748):0.9974351265,t3929:0.9625774261):0.6486500097):0.8115323854):0.09416588559):0.8292487813):0.5469847133,(((((((((t2992:0.5683556173,t4259:0.3949927753):0.7805599114,t1175:0.04091445007):0.7643867584,((t3821:0.02768375655,t3006:0.1913647617):0.4203181083,t3475:0.9063835873):0.08342644386):0.9979251395,(t4433:0.5105338693,((t1521:0.3995661624,t4803:0.2436534266):0.2399714079,t4991:0.9149852279):0.5945335242):0.4999193475):0.03605821752,t3650:0.4262857866):0.3821511313,(((((t4728:0.8763825349,t2640:0.8909392343):0.8235181617,t3939:0.06163756945):0.7023730883,(t3670:0.8489563302,t45:0.8891752213):0.9062200298):0.7284130137,(t4214:0.6071781716,(t255:0.5471948399,t3329:0.07519407175):0.4449009895):0.4435284184):0.9209371316,((t2214:0.864842118,t1959:0.1228758341):0.3623794119,t1991:0.3001923957):0.8381461618):0.5013161055):0.03621138702,(t4845:0.3836973961,(((((((t264:0.1797379714,t2920:0.06323192385):0.9472994932,(t178:0.8134361336,t3555:0.4632465078):0.5426755052):0.2946241021,(((t1479:0.2834749671,t2206:0.1258117519):0.7001927397,t285:0.8636411829):0.3024032288,((t4602:0.4241112387,(t307:0.1070842601,t3141:0.4348691227):0.6733420957):0.0474547029,t940:0.6841482823):0.4393613203):0.8206952848):0.08065264951,((((((t2743:0.1904597802,t631:0.3801400191):0.4537953029,(t2708:0.8371253381,t2886:0.004927282687):0.7707951961):0.3148066546,(t2626:0.745920066,(t2370:0.9215266744,(t4516:0.2544125484,t4012:0.9781097556):0.4198086429):0.5622178058):0.7167179745):0.1032030664,(t702:0.763994524,t3503:0.1553396252):0.431600013):0.4082977704,t1661:0.3882870362):0.5601166503,((((t1040:0.499051779,t2421:0.9045748545):0.4754377627,((t4185:0.6205980275,t4640:0.047475409):0.4838943111,t3601:0.7463765482):0.9137264625):0.8396863625,((t2738:0.7547164068,t3981:0.6234005406):0.1647720113,(t2832:0.8119530433,t4411:0.6729323159):0.4709452786):0.1100061161):0.8806596005,(t2950:0.2572114377,((t2558:0.7671095899,t525:0.8266024466):0.7879298865,(t3277:0.07909034076,t348:0.1474553389):0.2294729133):0.473996164):0.03797697159):0.344315456):0.5689797066):0.3359961072,((((t3451:0.07902769349,t3001:0.3875313066):0.5004862808,((t3860:0.844207234,((t3249:0.7483110789,t1195:0.6853311474):0.3935448329,t4613:0.4835625468):0.8231310782):0.8616584544,t600:0.2031736805):0.6895686598):0.7588504981,(t148:0.09319290449,t4586:0.3463730731):0.9267917315):0.2067353737,t2595:0.6939287838):0.6630084696):0.3517247469,((((t1358:0.2051085916,t1142:0.2535161141):0.2356792837,((((t194:0.1963996682,t2990:0.8241801145):0.5477449715,((t2215:0.8922409483,(t359:0.2819857213,(t1475:0.7335701424,t4572:0.2278105463):0.9821699557):0.9990439347):0.7445889062,((t3130:0.932181898,(t1003:0.4691574103,t3114:0.9672080709):0.06537476392):0.5752563705,((t4502:0.5084266593,t4366:0.403658591):0.2622919441,t3286:0.0406470811):0.5171034969):0.3138342754):0.5583873198):0.4776753755,t447:0.6068464008):0.06063132593,((t2957:0.2039489527,t3729:0.4641009655):0.6983884296,(t65:0.5302408147,t2418:0.2530338576):0.7596179815):0.1529862811):0.1936920679):0.3463638735,((t1694:0.789749603,(t410:0.8857452367,t2577:0.204111401):0.2676870029):0.5145677184,(((t1259:0.6243996972,t2191:0.1395968294):0.1414316138,t3051:0.9098383149):0.6104961869,((t2746:0.9354026457,t2804:0.925767374):0.1337771374,t3056:0.02450954658):0.5843916459):0.7518114641):0.1798938902):0.423137611,((t3533:0.9757340511,(t72:0.01517983759,t382:0.5286995219):0.4901269246):0.5750538178,((t1150:0.7687767656,((t366:0.6142901366,(t4511:0.6463828427,t2479:0.7865037511):0.3651600857):0.9743084444,(t1141:0.4077304071,(t4:0.7421213654,t3024:0.4245137293):0.4052562108):0.6585915205):0.1432655065):0.3668315208,(t943:0.914228024,t346:0.7226156169):0.229525069):0.2749038998):0.640280901):0.5096835378):0.3947588347,((t3578:0.6678591974,t1539:0.02688177186):0.8730778901,t2997:0.5442402717):0.03954109619):0.7367512153):0.3130241984):0.9557205813,(((t934:0.08885042346,((t4943:0.7681824565,t4666:0.7884132648):0.430543829,((t2567:0.7613236192,t4647:0.8441665513):0.8276139072,(((t345:0.485455778,t882:0.04423073493):0.3159170146,((t3987:0.8490459814,t3224:0.6555161292):0.4186672282,t1387:0.767411242):0.07535287458):0.3255701531,(t3088:0.1736841297,t4687:0.1593686373):0.0143699469):0.06147860666):0.5972477924):0.6454618159):0.3578858965,(((t3399:0.6922279608,(t1227:0.07279453869,t3994:0.1977977045):0.9250212964):0.6493315734,(t3614:0.9628730402,(t477:0.4931149457,t2153:0.6466388721):0.278322249):0.5116253912):0.5139884478,t3709:0.2226705174):0.7410245829):0.1560418631,((((t4709:0.01547131012,t1124:0.7757657599):0.6755204608,(((t4741:0.1844353909,t1712:0.2946070931):0.3033433531,(t903:0.2345348301,t1951:0.6515262525):0.02518605581):0.1926513275,(t323:0.8523304395,t62:0.08897882816):0.5348312724):0.7108556966):0.7834853258,(t1048:0.04900486208,t1149:0.9334965798):0.4003091659):0.2707316291,(((((t707:0.5182382076,(t20:0.7646884294,t2109:0.1980254753):0.4493309145):0.7037261934,t4177:0.5034425973):0.5418781326,((t4355:0.8896170026,t4814:0.8395685484):0.8667432563,t4247:0.9097422885):0.4744975835):0.139687004,((((t197:0.7592019648,t1827:0.5059122564):0.6455280827,t4243:0.3151332475):0.7940043439,(((t2015:0.9685657995,t188:0.5177651295):0.9996646901,(t4674:0.06534428243,t3868:0.1392651426):0.7662261429):0.4299562648,t4049:0.7989851125):0.7915994257):0.4046109545,((((((t2910:0.2856124884,t3998:0.4131231478):0.3555277931,(t2473:0.007324372884,(t3230:0.5106419667,t3108:0.674084597):0.1768954021):0.5667304955):0.5571086986,t997:0.6583933989):0.1964697065,((t1786:0.9011179721,((t516:0.7434655449,((t4887:0.6546093866,t2154:0.09627265506):0.7461024655,t630:0.007537527243):0.9425626635):0.9086317783,(((t89:0.7400260819,t334:0.6808794844):0.3554692685,(t764:0.003195799189,((t2723:0.6121856633,t2373:0.3929359764):0.2481850327,t1702:0.3423850278):0.237114317):0.4051361245):0.09202447906,t2122:0.7053843862):0.05282155401):0.07046277798):0.249602237,t912:0.03067528619):0.07358160033):0.6664595972,(t1066:0.7569250409,(t1305:0.4392006425,t2583:0.1490234821):0.9629560602):0.8277951095):0.9652523915,(t2446:0.6310957014,(((t4340:0.7684437605,t2322:0.6377697303):0.1788574064,(((t4530:0.8596319782,t1779:0.2722145903):0.3804833062,t560:0.6932128028):0.9835307521,((t4752:0.2737035069,t4982:0.09819985554):0.6147408606,t634:0.9432918408):0.4080122488):0.3075683757):0.3831631818,(t4422:0.9733662456,(t380:0.2677505114,t158:0.3671286749):0.9330587434):0.6576022322):0.4208260037):0.3222646452):0.9280143622):0.1717364278):0.4577977227,((t1000:0.5509319867,t711:0.07187527837):0.9277023722,(((t2861:0.3340854507,t4130:0.4591609144):0.1053898169,(t2262:0.1149932325,t4693:0.2332712759):0.2090749755):0.9027315977,(t217:0.9636097224,(t2928:0.5128834809,t1096:0.1159280853):0.1984668444):0.7048599408):0.6615817256):0.3667517549):0.03848553356):0.5243290702):0.8572676014):0.6570304416,((((((t1490:0.6932693394,t4813:0.9867736525):0.8887978443,((t1114:0.5809965285,t2037:0.9336156386):0.04703436582,(t4792:0.7670502604,t1033:0.2572370144):0.2232324991):0.576036897):0.2489234114,((t1090:0.6681642598,t1861:0.7592822921):0.5409648407,t4700:0.6785265177):0.05193778384):0.2892193224,((((t1910:0.3160585102,t4923:0.04208368249):0.04808597988,t2314:0.9112550071):0.4262476773,((t1718:0.9596122832,t1410:0.6763408249):0.7451057229,t3584:0.3915688666):0.9577314954):0.8923585019,(t1317:0.9794028152,(((t3409:0.9630143095,t1543:0.1187219149):0.2824911382,t2676:0.3542244716):0.4642618545,t1672:0.1254665174):0.8097363436):0.5053133694):0.2160847241):0.9988232758,((((((((t3546:0.6127355194,t2852:0.08237288939):0.4432123553,t4853:0.6185161674):0.638206308,(((t2029:0.5011937697,t1898:0.4220412152):0.4177122805,t3518:0.6557646575):0.5064452945,t2590:0.6847039601):0.02987745591):0.8142041245,t2608:0.6898910287):0.1739860389,((t252:0.5513642298,t1556:0.4358791208):0.475605414,(t1748:0.06122780219,(t4118:0.7664316038,t1077:0.6857726299):0.7227829446):0.9400820734):0.9176695209):0.2050848741,((t995:0.3843363277,t1044:0.5294049557):0.03525020485,(t2682:0.4704738713,t836:0.9912010645):0.002895758953):0.6543571826):0.3725051805,t2460:0.5794375308):0.8704791195,(((t1700:0.01301623811,t1451:0.1506979994):0.7094372038,t814:0.8712445896):0.7178737819,t11:0.873640296):0.7535745106):0.7136196867):0.7952476251,((((t1092:0.3115337621,t4213:0.8566735301):0.3055560649,t1961:0.6281866122):0.1655700796,(((t918:0.1747456768,(t2102:0.9304132541,t834:0.6489289068):0.5543183968):0.4556021083,t2892:0.07734527602):0.8816009036,(t4751:0.2212550647,t4491:0.03246501926):0.1210206342):0.5378542719):0.7218904784,((t1831:0.7088573559,(t3039:0.2050984807,t706:0.7538442865):0.9405935137):0.1461368694,((t909:0.0614246903,((t4294:0.7530829513,t1202:0.3808521081):0.03547592345,t1284:0.7092968402):0.9835932779):0.1483054191,(((t3542:0.5183108682,t884:0.01049827249):0.8388102937,(t4849:0.441805477,((t2798:0.05319192004,((t1987:0.9769116782,t1021:0.690025053):0.2116704141,t3038:0.9827959649):0.09709312557):0.4772203767,t3626:0.4265433326):0.01943905489):0.9944709251):0.6617160912,(t3417:0.331042337,t213:0.8466180656):0.3969420043):0.7329561443):0.87057892):0.4604139021):0.4813063571):0.7990057706):0.6415172855):0.6678200508):0.9383532861,(((((((((((((((((t2962:0.7262138692,t3922:0.8583351264):0.7161026634,(t336:0.7422789659,(t102:0.74303062,t4434:0.8937873503):0.1427770783):0.02465950488):0.9909834159,((t1509:0.1205776036,((t3561:0.934114797,t2456:0.61937402):0.2097503061,((t2828:0.8452374968,t2924:0.3701922887):0.4136367294,((t1882:0.9889265688,(t2065:0.4537541952,t2079:0.7174955611):0.8356350283):0.9128047386,t999:0.2084656192):0.07642332604):0.5967560955):0.8393798631):0.667187131,t1811:0.680862278):0.8853018477):0.4850144743,t1920:0.7151014295):0.2848867627,((((((((t3326:0.5353890422,t3800:0.3645006984):0.2478848353,t2796:0.6690276565):0.7904291577,(t1102:0.00570087065,(t4971:0.3237898755,t3602:0.990154071):0.4400586628):0.550433977):0.5442886134,t1330:0.5556930776):0.6117003686,(t1319:0.4745831352,t1282:0.5457444051):0.7737812246):0.1190525598,(t2124:0.07137664082,t1316:0.9582177997):0.1256239337):0.5778165422,(t4954:0.6034199249,((t3450:0.2082994911,t4386:0.2360969407):0.5511637467,(t257:0.3070545059,(t827:0.4293418007,(t678:0.5941290427,(t4336:0.9044811127,t2246:0.7943649262):0.9168136851):0.1483359498):0.594744635):0.2570373758):0.2839867861):0.7414448969):0.4724285717,(t4007:0.1859351087,t1332:0.3810958802):0.2426472993):0.492919263):0.07921248046,((t2871:0.661856913,t3029:0.9956787496):0.614281669,(t3913:0.3594356859,((t2500:0.6831667242,t4149:0.6431639136):0.5516189961,(t3961:0.8166393847,t3855:0.2546768542):0.4254611351):0.02258943021):0.658599227):0.2355531412):0.472023821,(((t604:0.452792451,t2148:0.8389070204):0.3583359655,t4405:0.4204529207):0.659835666,(t4352:0.8666897214,(t3553:0.5559384008,(t3819:0.08926707553,t484:0.7154611591):0.404522259):0.05359331914):0.3632904361):0.3948235237):0.5181460318,(((t1874:0.1355053952,(t964:0.7533758597,t686:0.9844604735):0.183650122):0.7069136717,t1696:0.4060957308):0.05870199832,t2808:0.8396733731):0.866898672):0.1832010448,(((t1232:0.08006161731,(t2701:0.8607525362,((t1637:0.6824927994,t954:0.7245764658):0.8207018448,t3551:0.7166925969):0.2768487402):0.4727787913):0.5248614741,((t2613:0.3654149936,((t2648:0.1286852288,((((t3259:0.5070074962,t1664:0.9151285114):0.2723700923,t353:0.07942079986):0.8792005605,t1893:0.1564026419):0.1172787147,t3586:0.31520587):0.4191328976):0.4746167548,((t4382:0.668854695,((t4266:0.6942115454,t4759:0.9251362102):0.1276626752,t463:0.5920489666):0.06901807222):0.8891850021,t1917:0.9551467802):0.7135965317):0.6093363722):0.3366824749,(((t1927:0.1652037762,(t3010:0.7443171921,(t4774:0.05417106254,t2605:0.3119480584):0.6296363084):0.6548634963):0.3114851988,((t2530:0.5482004059,t239:0.9850254925):0.7656948289,(t3973:0.4502616085,t222:0.3419384907):0.1624722034):0.6988789586):0.2249992865,(t4125:0.4914448904,((t1067:0.6593904477,t610:0.3369132897):0.9780704966,t1596:0.237529248):0.3897448229):0.6126249847):0.9516359342):0.4012620279):0.4237613117,((((((t1118:0.3783206646,t2108:0.7804531916):0.9224953155,(t384:0.6595169653,(t2089:0.03323150403,t3395:0.4163023452):0.709629087):0.6090891897):0.2694969382,(t1580:0.003898847848,t4179:0.9587801204):0.8697528949):0.7935501849,((t645:0.001531203976,t4092:0.8965186332):0.6515337583,t2222:0.9239231932):0.370236512):0.810224117,(((t652:0.4089869517,(t4833:0.5379008662,t4460:0.9477532159):0.6428246435):0.8888959237,((t2850:0.9468437084,t4555:0.773919052):0.172683717,t944:0.631241061):0.6874550991):0.3906486121,((t4732:0.6644497479,(t1356:0.828384636,t602:0.7148917774):0.03186064656):0.2766948789,(t3257:0.0134123133,t3378:0.02960873791):0.5065867337):0.9603377432):0.7573593766):0.3788352862,((((t1121:0.8400318578,(t1727:0.2598242431,t723:0.5820624123):0.902386379):0.5028996905,t977:0.8005011298):0.3318134772,(t4322:0.269030743,t1753:0.5229965521):0.8806119692):0.797340415,(t1680:0.5867746982,(t1617:0.7191714298,t1949:0.6596775411):0.7487695187):0.4226935252):0.8843422038):0.3909669449):0.5417169568):0.5114382424,((((((t1567:0.1844823088,t3599:0.9039775273):0.4835802736,(t548:0.03273510071,t4767:0.02672401746):0.3378277521):0.3577588,(((t424:0.08024800848,t4745:0.3147766502):3.427267075e-06,(t2007:0.911275578,(t127:0.6719994016,t3707:0.6370994784):0.8615536501):0.08571552136):0.608506822,((t963:0.8954207071,t4004:0.1710815181):0.7203015303,(t2082:0.9753508174,t3123:0.635486145):0.6631354715):0.7841217713):0.6585065254):0.8198231205,((((t1428:0.9879597069,t4520:0.9947917201):0.02603928093,(t3178:0.7134850451,(t2961:0.2175829681,((t4918:0.3267469991,t75:0.806449634):0.9262028558,((t3587:0.2930043011,t2903:0.9516623588):0.4193006081,(((((t389:0.3402687062,t969:0.5061908925):0.3697433439,t308:0.4861857144):0.5600478824,(((t4122:0.3142874816,t4808:0.9828627198):0.2153276552,t1413:0.6939961817):0.1179870605,t3358:0.8280851066):0.3168632302):0.7464490433,(t3176:0.8260539107,t563:0.8996708137):0.9911272344):0.3439247569,t53:0.7261300127):0.7146047754):0.2551055667):0.6233245106):0.9600828567):0.3450316775):0.5784728285,(((t1181:0.7961972628,t3096:0.2175641307):0.0989612001,(t1880:0.4179928249,t3261:0.3475408361):0.2301712115):0.1828998581,t4536:0.8773570918):0.3965672867):0.5203566179,(((((t296:0.4962522339,(t3642:0.4497568575,t2899:0.8129758721):0.3301557752):0.2441319616,(((t3841:0.5856999112,t2538:0.2656358804):0.7877510739,(t2917:0.7939528148,t4232:0.6278766962):0.1803098298):0.05187540012,((t505:0.257752409,(t335:0.8851803045,t2076:0.1084350201):0.7093391018):0.04393044533,t1999:0.9639111937):0.2097887732):0.7898364442):0.3349640444,t2983:0.9945538465):0.3651028003,t873:0.4657309745):0.2077373657,t750:0.06851134798):0.05651681428):0.02990587684):0.568570222,(t408:0.8290724331,t4618:0.3050913746):0.974504231):0.8530586013,(((((t1766:0.5785169625,((t2444:0.1964719016,(t1662:0.2404880368,t4648:0.2487684961):0.2832237349):0.1989366617,t854:0.6079415688):0.9256657488):0.7512660285,(t4360:0.9631684893,(t1629:0.3435779579,((t3595:0.2225473409,t32:0.8150938721):0.7657266038,t4820:0.3309884691):0.5334544):0.8536345856):0.8840599447):0.5045049391,(t4967:0.03526027827,t4393:0.03504588688):0.2806280362):0.3065179011,((t2182:0.3809294947,(t4762:0.889537808,t2327:0.4205300524):0.9629419369):0.9261945221,(((t1742:0.6586132899,t1:0.2439815288):0.4315093237,t1466:0.7518830958):0.8714073719,t4490:0.7333032934):0.5044749463):0.3279135083):0.2239434922,((t4685:0.5753700216,(t3609:0.6175053166,t214:0.5655104215):0.6993877119):0.6711862364,(((t3889:0.7590449103,t4357:0.6798005202):0.3475025736,((((t3301:0.2741813918,t2312:0.8178372607):0.1876533707,t3827:0.1749639947):0.4963001544,((t4173:0.824564669,t3635:0.7417331673):0.4160173705,t4743:0.05443758378):0.5306879554):0.7629743188,t128:0.5647710063):0.2857035312):0.1444029463,((((t3345:0.334779449,t4270:0.1058641498):0.2625456776,(t4725:0.5735976107,((t742:0.3297777141,((t4057:0.2443992328,t1140:0.1056225006):0.3865051235,t2880:0.4282674969):0.4648150541):0.5310326878,(t4707:0.5831282181,t4230:0.0363113198):0.5300266079):0.3989459372):0.2982925221):0.9989453731,(((t509:0.4984098547,t275:0.1400559484):0.7930502705,(t3486:0.05330740102,t3464:0.6512927921):0.6183371663):0.132246179,(((t4600:0.4332387527,t3596:0.1654799755):0.1100643626,t61:0.06889371714):0.9352715786,t3724:0.4764409203):0.3504803083):0.8416362784):0.3266389561,(((((t1655:0.3338650414,t103:0.8236319672):0.9929849051,t4597:0.7570586994):0.9568472649,((t2434:0.2504198209,t3839:0.7122473572):0.7889866326,t1250:0.5060904166):0.8359238699):0.7995930989,(t3372:0.4739475425,t2639:0.1556260188):0.1401465081):0.004005366005,(t4240:0.4544786301,(((t3411:0.5287708368,(t3748:0.8777483352,t4050:0.799628072):0.6812146788):0.4289193612,t3890:0.1908239357):0.4363218157,(t212:0.5147207784,(t4267:0.1932296476,(t4880:0.6118474281,t996:0.8419796473):0.6801335325):0.6654939319):0.233021626):0.1669444793):0.3458538419):0.01115980605):0.9437996801):0.7577234146):0.6465956764):0.6471493801):0.5583716554):0.7520835288,((((t4084:0.7113987007,t4628:0.2698982207):0.1747126551,t3278:0.8913992182):0.6786976752,(((t2528:0.4973286414,t2230:0.1247115331):0.2626977852,t2991:0.2733335379):0.1595955738,t2474:0.6849966929):0.4124169736):0.5120637666,((t280:0.281440116,(t240:0.6452526185,t3448:0.8434458328):0.9662224487):0.00692372513,((((((((((t4504:0.3090105972,t1302:0.7995785354):0.7904413466,t3005:0.5781695005):0.2197984329,(t2383:0.1823836593,(t4719:0.6322632323,t801:0.477838997):0.1493833216):0.8419143513):0.9379060357,t1170:0.6712732548):0.9263909035,t4643:0.4157575618):0.4467396941,t1984:0.7724634516):0.2890088405,t1817:0.3480296144):0.3469312706,((t4722:0.7305953342,t4249:0.6763996407):0.9420758062,t4843:0.8965585527):0.5335896797):0.01741760666,(t4835:0.8365049786,t1792:0.5501905924):0.2631628548):0.3988401878,((((t16:0.6834248656,t3463:0.3617509233):0.8271554811,((t3997:0.196211996,t93:0.1453959302):0.439272328,t3246:0.8807860925):0.6056077126):0.4260622188,(((t1957:0.2868695825,t1174:0.2309518459):0.2613003873,((t839:0.7853680588,t2727:0.9920329151):0.11601342,(t3283:0.2833106688,((t1288:0.7023590382,t2931:0.6141899168):0.8169669176,(t3956:0.9682801757,t2492:0.08359932806):0.3023653438):0.405985462):0.7040732002):0.7462452657):0.2885388869,((t594:0.8468178732,t2088:0.4128975002):0.8556786396,t783:0.4798944306):0.6337389685):0.7739790755):0.1640336085,(t1194:0.2368415657,t1162:0.9484809532):0.1104401278):0.4034753377):0.1689618062):0.1142551003):0.09222533344):0.1329308471,(t2333:0.007825135719,(t186:0.4937455179,(t4794:0.06020901701,t4344:0.7841492801):0.2287012585):0.2842475143):0.2768523567):0.5299976587,(((((t2036:0.4405954094,(((t166:0.4513484389,t577:0.9506866713):0.06507066754,(t3822:0.9107432063,t2680:0.5976844975):0.980326389):0.01877636719,t4870:0.8322180631):0.6260457765):0.8190441623,((((t4282:0.179717391,t3710:0.8906076108):0.9608471682,(((t901:0.7763043579,t2942:0.6476838898):0.7039241516,((t4172:0.4447395764,t2199:0.1728869642):0.9063360847,(t1337:0.785604873,t3443:0.1969642164):0.04977966822):0.1300333319):0.5817552099,t4063:0.8917331693):0.6318063152):0.7867649812,(((t4915:0.5278157443,(t1006:0.4145518111,(t2433:0.2928249028,t2496:0.4507393125):0.9773183155):0.7569449318):0.1637509586,((t4401:0.2987637655,t2980:0.3483995374):0.0258481456,t108:0.6204921142):0.173995943):0.1980753115,(t4314:0.143935276,t512:0.7094376604):0.02067261352):0.5105723352):0.4587925454,((((((t4000:0.3878473961,t1955:0.9217705303):0.2358263177,t453:0.06399881095):0.5610117144,t2754:0.6788395494):0.9287288031,(((((t3127:0.892684886,(t1865:0.3275158557,(t1924:0.7168268529,(t4533:0.2141097956,(t2069:0.6426789684,t3426:0.3880048452):0.01012898097):0.1743045854):0.6319946114):0.6271233344):0.2919093622,((t1340:0.4825708452,(t2394:0.3374720199,t2143:0.9809464659):0.8276730704):0.3819046619,((t2001:0.1712828206,(t1459:0.5572053397,((t1992:0.9427959051,t200:0.8078974748):0.2230089966,(t1253:0.2885210831,t4644:0.9449514681):0.4743002446):0.2081758261):0.6107601211):0.7672076034,(((t4373:0.4224551204,t1245:0.486102323):0.9678493661,((t4387:0.6867055648,t2657:0.1356155784):0.3089737417,t1209:0.9855114771):0.02427270915):0.8964985707,t3384:0.6749291152):0.3100609556):0.5930570129):0.9383101412):0.2625636309,(t236:0.1012587028,(t2684:7.105595432e-05,t510:0.7905280346):0.8733937622):0.8222653079):0.5823246415,(t1344:0.9254428018,((((((((t3118:0.1040853884,t3457:0.6976789827):0.2812245579,t1105:0.1523652102):0.5247684622,t1716:0.9184710039):0.4347472272,t1956:0.139966673):0.03697826923,t794:0.4474921427):0.3198264765,(t3968:0.01561442623,(t3697:0.03309637238,t48:0.2392675329):0.8297406249):0.1214160703):0.2677891972,((t2697:0.7228317864,(t4272:0.6955559249,t3052:0.1212348503):0.510660446):0.8216106254,t273:0.04460524395):0.285070807):0.09515914181,((t1370:0.8247850279,(t4483:0.6413038895,t3247:0.6255824168):0.7946476543):0.2671518866,(t2562:0.4926836079,t3258:0.7683827931):0.5191153679):0.8641255347):0.434407952):0.7386618992):0.1003132293,((t3754:0.1528475615,(t1437:0.2521358016,(t3273:0.4569258506,t4160:0.4420627088):0.9036253279):0.3137168845):0.9214993231,t622:0.1753153994):0.6875868337):0.9079641439):0.8159785327,t2894:0.8683465328):0.3144849443,((((t3500:0.2174643609,(t429:0.9429278637,t2475:0.09661523276):0.6467378302):0.4284414572,(((t1788:0.6325364669,(t2259:0.5643249038,t4663:0.8346780883):0.098181471):0.8661162907,t2048:0.9545170122):0.702393227,(t142:0.4987407257,t2238:0.9545663558):0.9399609426):0.7665764655):0.2638789965,((t4110:0.9450869875,t1134:0.5860108251):0.7755987796,((t1823:0.4399406891,t2034:0.07218393381):0.3540829711,(t1185:0.215702889,(t2635:0.03400844848,t2729:0.7401463999):0.3975910922):0.0471498901):0.3823134245):0.8467795982):0.453222641,(((t4471:0.5070170932,t1260:0.9572457285):0.3352085319,t3198:0.9076079237):0.2212412681,(((t2493:0.07784569566,(t4987:0.4467527189,t396:0.1993106157):0.9489312868):0.1130697392,(t1711:0.2374455857,(t4689:0.8031826077,t4134:0.8644662038):0.1860905394):0.884776301):0.4594913532,((t1535:0.6797542591,((t2299:0.1952631101,t3467:0.9276725044):0.6601518267,((t4694:0.7917074778,t3941:0.1939830135):0.793794516,((t276:0.5535281657,(t443:0.163111296,t2638:0.2404224884):0.9953644106):0.8714757748,t601:0.9922141591):0.8936089175):0.3500113913):0.3907055203):0.9470289228,((((t411:0.8892813064,(t1731:0.1137634676,t427:0.7484523198):0.7715137694):0.05781302624,(t2771:0.3454575201,(((t84:0.1340223446,t1998:0.3526036164):0.6273540517,(t2955:0.09404921532,((((t2717:0.9525596115,(t2093:0.7015653446,t3219:0.06627323548):0.7692494278):0.7032709303,(t3487:0.06485897745,t270:0.06158223702):0.4189626474):0.1120371916,t2761:0.9931760672):0.3868831932,(t1201:0.6044189965,t1329:0.5739250337):0.1408602663):0.9467953723):0.4454013335):0.8951859141,(((((t3505:0.2784547431,t4094:0.3499765312):0.9306254296,t3191:0.4602360178):0.6705352091,(t3381:0.8893911785,t33:0.05029384769):0.1421050171):0.8701545307,t3864:0.5503185035):0.1820352292,t3227:0.1809635703):0.4946118339):0.5582480875):0.9797163103):0.3181473331,(((t2669:0.7735680772,t2823:0.3494082035):0.02417387511,t3519:0.1478207435):0.8570490258,t1047:0.04962586565):0.6861325419):0.7532274844,((t4978:0.4982785615,t3773:0.6828479297):0.2767159804,(t3495:0.6784780018,(t1131:0.6584981559,(t2934:0.6086825014,t4423:0.04016133258):0.1421606252):0.5449224976):0.04323421023):0.4579206966):0.7816028884):0.2307484918):0.9516916894):0.4918339697):0.9123380941):0.869848578):0.2785552375):0.1004747536,((((((((t3491:0.4335138886,t4748:0.1400322514):0.2525995702,t4002:0.09436890576):0.02209823276,t3817:0.6024864793):0.3975430417,(t167:0.1525500917,t3515:0.0578564601):0.1135589082):0.4366631401,t2356:0.612857216):0.03312625643,(t4157:0.7319163128,(t2951:0.8195105705,t3744:0.9934666695):0.002051943215):0.9257125922):0.1120324801,((((t523:0.3683013392,t2777:0.9105640552):0.4414764908,t2406:0.4191308771):0.45298629,t3343:0.1453825799):0.1481226557,(t2730:0.7202222662,(t793:0.511303504,t1157:0.1419341005):0.6160562474):0.6027775933):0.9096826667):0.3534672654,((((t3653:0.2392133458,(t4547:0.8674878811,(t3529:0.5105062064,t3410:0.7225932346):0.5938546609):0.9204834765):0.3003669134,t1345:0.09367364761):0.3876323588,((t1855:0.9383467061,t893:0.9251530587):0.08135259617,t4688:0.8466076404):0.1899984921):0.7431564101,(t2399:0.9017488202,(t2057:0.7648492239,t4221:0.654516336):0.7387904963):0.7905760899):0.9576236228):0.2468244559):0.6560781917,(((((t1220:0.02463638806,(t962:0.1923351241,t696:0.5418487466):0.1910762563):0.9249072359,t4527:0.9588721867):0.2319914654,((t3444:0.5096210332,t2028:0.07436235575):0.03401017957,t1582:0.8321982867):0.4663031856):0.2608651796,(t2693:0.6563487633,(t2553:0.6295327423,(t1429:0.1556336989,((t2247:0.09405928664,t2149:0.2680600684):0.3460829179,t3101:0.8610302578):0.471151703):0.04652113724):0.727698961):0.4072797264):0.7943767714,((t237:0.7817081171,t3228:0.8807131613):0.4931400535,(((((((t3741:0.10361787,t4174:0.9064539389):0.2694355219,t2140:0.5610431139):0.5672331613,(t1323:0.5220925177,t1371:0.9451466422):0.2540856332):0.3813403731,t1513:0.2345292515):0.5498658898,(t4184:0.7509594737,((((t1098:0.9460167964,t1112:0.5436120615):0.8802956603,(t787:0.6914048451,t3539:0.5194457916):0.03716892167):0.1108253072,t169:0.01069202903):0.8384362198,((t4738:0.8546245962,t1812:0.5348012799):0.6461196237,t762:0.1153883126):0.2637670825):0.862475428):0.1050547645):0.4176039489,((t2586:0.7499921029,((t1500:0.02044343459,t2027:0.3348222179):0.3237804268,((t3689:0.6078839707,(t2853:0.6352343571,t3757:0.2663663013):0.2798020944):0.7826251891,t3266:0.6487117496):0.6497461733):0.3159767149):0.6204324234,(((t2970:0.7001024887,t3470:0.3267781525):0.4574851508,t1516:0.9740719567):0.4813881828,(t4592:0.981659618,(t3562:0.9862664249,t2513:0.2776642111):0.5735258013):0.9303701306):0.7627139403):0.8158748376):0.4813487413,((t3799:0.6550234298,((t2265:0.5691744401,(((t1012:0.3639946983,t4032:0.2513800713):0.05077923788,t642:0.111116068):0.332600015,t3008:0.7463258731):0.8327168939):0.0596749594,(t2059:0.4885736173,t3031:0.3480907534):0.426270128):0.3755060467):0.03681151336,(t1950:0.2029797365,((t3305:0.5627737804,(t4714:0.6405647639,t4908:0.2053580165):0.4354477008):0.559139661,(t703:0.7935215533,t1208:0.6252977592):0.04521982861):0.2336645124):0.06484406511):0.4471674964):0.385971332):0.6591193636):0.7686109957):0.1250434432,((((t3061:0.3440663607,t4042:0.3157220664):0.675226545,(t59:0.628866812,t1089:0.4707864656):0.4955711076):0.1480921553,(((t861:0.6624049346,t2443:0.224396925):0.2847800031,t1283:0.05358729139):0.7775096421,(t3104:0.2491828208,(t2842:0.3309956335,t1611:0.6659108354):0.2361551465):0.5764183176):0.8832583958):0.5620367536,(t746:0.3837163656,(t2412:0.6583311439,t2269:0.7888609043):0.4374579969):0.9452234805):0.5297878124):0.6996258383):0.3415587887,((((((t3718:0.5074331015,t2091:0.9762871319):0.1811176061,t2469:0.1494129056):0.8782151802,(((t4495:0.9885551857,(t4654:0.6719454667,(t4538:0.8634280069,t3969:0.8553444003):0.8008243486):0.6771141468):0.4766978489,((((t2908:0.5818098409,(t4858:0.4159208869,t4474:0.05818697251):0.7745204403):0.9324367635,((t1558:0.4736906223,t4044:0.665242705):0.2250373163,t2389:0.7295064945):0.3089176943):0.2673065483,(t4696:0.1253566248,(t18:0.5562083421,((t180:0.8087127898,((t1147:0.4080226736,(t1615:0.1755405967,t4203:0.5146258022):0.1837007273):0.5720820408,t3174:0.1923566561):0.7669331504):0.08475451125,t4756:0.881315602):0.5069478815):0.4342382322):0.7497944215):0.5907712085,((t4237:0.05716132326,t3576:0.7684986677):0.08016940788,t1749:0.7509441033):0.3090163285):0.2411447875):0.3455243844,(((((t3364:0.34792628,t1129:0.5038216433):0.5441117203,(t1860:0.7143098817,(t949:0.06636277982,t1734:0.351874568):0.9854714014):0.4995358372):0.03393022786,t2063:0.05341036804):0.9451750594,((t2210:0.7825913473,t1744:0.6990753382):0.4731194037,((t3023:0.1679361237,(t4780:0.421774203,t2307:0.6262942585):0.2137286193):0.3505443342,((t4637:0.2698253188,(t2722:0.3325449296,t2816:0.546037917):0.3756804399):0.4088706858,t3091:0.4856662734):0.6109957881):0.9863933926):0.9387679112):0.6389107239,(((((t4981:0.8478533158,((t1487:0.5215197825,t4661:0.7316549162):0.2389087351,t3552:0.9588676286):0.5110373397):0.1714820098,(((t1018:0.8430333142,t1840:0.01911786478):0.042871841,(t4753:0.7676874339,t1548:0.08773567271):0.8203522854):0.2735698586,t829:0.5314179112):0.9354918224):0.6734886642,((t3013:0.01223673136,(t2630:0.6517426269,t3621:0.5579286371):0.1238641946):0.8461402038,(t3563:0.03705857205,t2192:0.3473468565):0.7967079529):0.6902407985):0.8339096378,(t1377:0.89918656,((t1231:0.4139542195,(t2741:0.4605228261,t39:0.3945815491):0.05059540644):0.5795721228,t31:0.3798495962):0.6751602513):0.1613921041):0.2757078975,(t4468:0.06827956578,t1328:0.7412680518):0.709372744):0.9975278385):0.1847563968):0.7529182183):0.6685144533,(t3391:0.8265393139,(((((t3303:0.02300456353,t2791:0.644107152):0.9647951617,t415:0.555972304):0.3688408323,(t2745:0.4287488409,((t3930:0.9095302937,t1912:0.9525530094):0.5336323192,t2345:0.3841391159):0.7811806232):0.5653265091):0.2391818666,(t3611:0.231048631,(((t81:0.1912265182,t2835:0.1174680332):0.3640010895,t1111:0.6997918475):0.4389217969,((t1271:0.7103070596,(t4196:0.2140415381,t3593:0.7422047108):0.1657404907):0.90213965,((t441:0.8982148496,t3370:0.8179885081):0.8231075644,((t3151:0.4392819994,t4127:0.5257334935):0.8790064119,(t2132:0.8195359597,t210:0.5704154421):0.8011257984):0.3606936478):0.6137814664):0.5840160763):0.6357191191):0.9892603077):0.890853995,((t4676:0.6922533268,t923:0.2501822175):0.9310205204,t984:0.8551313698):0.7925177705):0.5400493077):0.168681616):0.488126782,(((t4389:0.3458877853,(t4457:0.7923225379,t1632:0.7491949643):0.9259447595):0.4408739002,t1008:0.1913873628):0.842608558,((((t3693:0.3857462199,(t3879:0.7568280797,t3145:0.9899931664):0.7439075795):0.2408439193,((t501:0.9941636659,t1660:0.2161669778):0.8060132959,t350:0.8570286797):0.6125214973):0.4928399154,((t2876:0.6508947839,(t3959:0.8732843283,t4525:0.9837955022):0.6316757964):0.6923866039,(t51:0.8951506307,t569:0.2466581692):0.8383802888):0.9068373493):0.5322197783,((t3974:0.4531708441,(((t1802:0.2676001659,(t4413:0.8194835559,t271:0.5992415494):0.1719379958):0.2776597468,t2325:0.2783769357):0.09932349063,t2574:0.9280326176):0.2778331882):0.6306165769,((t4394:0.5079495979,t4010:0.5211637116):0.8248848014,t481:0.745801439):0.003628396662):0.3672056266):0.1515814501):0.725278693):0.8877521958,((t2480:0.05927129113,((t4755:0.3830011629,(t1138:0.9597058615,(t4686:0.7104720634,t4706:0.2416778356):0.9450298094):0.3729218752):0.8104782165,(((t3065:0.7320153033,t2491:0.09692441858):0.7762306281,((((t2139:0.04145612777,(t99:0.4728451278,t3297:0.8104012054):0.5694008139):0.2352781035,(((t3943:0.7746164536,t4048:0.4115498853):0.3138031936,t3248:0.7205174111):0.6524397761,t521:0.76257078):0.8780235185):0.4367877501,((t3646:0.5080549561,t1042:0.9520941379):0.1351560694,((t2627:0.8308105904,t1557:0.367066717):0.3223976509,t1970:0.6444319007):0.8107497683):0.3334985899):0.3917275881,(t835:0.25590939,((t258:0.1984209584,(t2353:0.5758404543,t4342:0.9567582933):0.4421464668):0.7643002984,(t4345:0.05998407467,t2620:0.9994047221):0.2421732962):0.7181127602):0.1561494695):0.2364479394):0.1835280061,t2996:0.1042222406):0.3432798772):0.2137563182):0.594169474,((((((((t3876:0.4912681824,t2425:0.8930247265):0.1938128436,t4143:0.7781412113):0.4296156678,t3566:0.1208547582):0.03143707709,(((t452:0.06286405702,(t4937:0.005857448326,t1706:0.08070565574):0.09601403191):0.6065935327,((t695:0.6426736352,t3739:0.01534578111):0.1614832615,((t613:0.6584661799,t1169:0.9065664147):0.5477463342,t646:0.002258092165):0.7034766371):0.6349377504):0.7388612218,((t2647:0.2590813586,((t614:0.2705418812,t4377:0.9876070665):0.09514414286,t3173:0.9704576891):0.932730657):0.3506622994,(t3473:0.2270562372,(t3985:0.2409176808,(t362:0.1109369814,t3042:0.08021445014):0.4879289952):0.7511877683):0.6035878267):0.6341009482):0.5159756786):0.06471874192,((((t2900:0.8801715737,t3342:0.2295453155):0.5562548172,(t496:0.01051674131,((t4096:0.9140247838,t3781:0.2054861542):0.2853377257,(t3932:0.9571212442,t1532:0.5884533862):0.4747523509):0.03889815626):0.865502869):0.3659548238,((t3099:0.7026090627,t4726:0.5867526701):0.2676256995,(t2481:0.6185686886,(t4293:0.0931499924,t1364:0.6392615368):0.7653348001):0.3459384607):0.454394215):0.13702334,((((t981:0.5379725944,((t1489:0.703229161,t2172:0.6340708009):0.2020587332,t3331:0.6049142927):0.3474886045):0.7989596224,t1095:0.01419937564):0.9042303436,(((t155:0.5791752611,(t4746:0.5270857823,t1352:0.7720973161):0.6022324318):0.8390010761,(t3335:0.849161898,(t4605:0.6174221081,t1583:0.6836379617):0.668287229):0.7570138166):0.5464033855,(t118:0.7986642586,t4024:0.6063636034):0.9490999258):0.8020704146):0.06805515499,(t1226:0.1323996638,(((t1279:0.1436497921,(t101:0.1362354541,(t2227:0.9049487426,t2696:0.09585445956):0.07034533867):0.4185555575):0.4856618694,t1086:0.1113172413):0.1319267359,(((t4679:0.2609236098,t2597:0.9943507127):0.1028425321,t2784:0.06530996389):0.4709627007,t2258:0.7218955809):0.3901349038):0.2361333712):0.2148582321):0.1405034994):0.9984194201):0.5310845601,(t3752:0.1649888011,(t2933:0.3763812126,(((t1179:0.02401749091,t4330:0.6266003756):0.2760254466,t2700:0.9479821592):0.4909517767,t1354:0.3978155542):0.6772292887):0.5740000149):0.3634165446):0.2295289459,(((t2350:0.2958781703,(t147:0.9509503511,t2183:0.5479835363):0.9483288983):0.346765172,t286:0.5826510598):0.2440643532,((((t1618:0.1451511809,t4988:0.8338828273):0.4419743095,t2728:0.8145442226):0.2333698769,(t3556:0.1644302213,t468:0.5517940556):0.4053673719):0.1781886325,(t1128:0.02794138598,(t12:0.1076982603,t1176:0.5385575735):0.1209488767):0.8491805126):0.7731262511):0.05186388362):0.8715103425,(((((t3150:0.1637162974,(t1737:0.2143738649,t1236:0.3574736102):0.4401737268):0.2282431084,(t4305:0.6562011095,((t1294:0.2199306951,t2047:0.7256445154):0.1831500421,((t3231:0.5772739297,t3730:0.7509981773):0.7497914117,(t1285:0.9890013656,t2849:0.8784068828):0.5872603415):0.9109540004):0.6312700519):0.37108106):0.685623752,((t413:0.4824377436,t1164:0.3325604142):0.5884122383,((t3361:0.4001204052,(t1218:0.7332760252,(t1419:0.2719845292,t182:0.2631517912):0.2184543661):0.6036438742):0.5644708413,t245:0.2534804579):0.4855942002):0.5280457262):0.09601887665,(((t4402:0.9676652874,t2260:0.1953467079):0.8273638566,t66:0.08008735953):0.3203009174,t2334:0.1242184814):0.9871498325):0.2194295281,(((t4461:0.5394940658,(t425:0.7549582496,t3925:0.3060626234):0.07645645272):0.416362369,((t1171:0.4600801102,t1126:0.8090640323):0.3192197173,((t207:0.9006679945,t542:0.9207556369):0.4294410599,t3468:0.01797916531):0.210681879):0.8657100152):0.8573118509,((((t4827:0.7512464493,t524:0.4392646465):0.9863706282,(t4392:0.0353441725,(t4675:0.5735928619,t1229:0.7767823657):0.5324997245):0.9831927998):0.04009053577,(((t1533:0.6484861458,(((t2656:0.5628228772,t2901:0.5876230062):0.1013386501,t1127:0.189776608):0.1568047907,t1780:0.1946117289):0.728710887):0.208614113,t4356:0.8870425483):0.01077792165,(t2911:0.5290552003,(t4166:0.1420111237,(t3910:0.2031622138,t2360:0.0537654683):0.7086734292):0.6999085587):0.6530871815):0.8440217196):0.7041317283,(((t1101:0.4845554559,(t693:0.9521668649,(t2358:0.5665360778,t811:0.2816568827):0.2386645305):0.9701437727):0.5956941347,((t4892:0.2736582246,t1805:0.7756556629):0.9486975286,((((t1794:0.810251985,t915:0.7599247298):0.6739438279,(t2653:0.2679301894,t378:0.2808274301):0.2985125012):0.4846359501,((t4023:0.006146292435,t4534:0.415595189):0.3088320256,t4701:0.01817663573):0.3650710427):0.9421814401,((t1249:0.6625316611,t1888:0.5578232217):0.6212042996,t1770:0.51330514):0.1874328551):0.5932504237):0.7712079845):0.537988364,((t1268:0.8784598566,t1393:0.09459985723):0.8256800091,t823:0.6652170031):0.4114926523):0.4071907259):0.2386713631):0.9695122116):0.5738056689):0.5871799232):0.7020032196):0.5270228449):0.9141213698,((((((t1856:0.4554060134,t3435:0.7338342601):0.8591136402,t3496:0.2724766836):0.9792074515,(((t4131:0.4493588384,t1438:0.1424898391):0.0169189279,(t1346:0.4873448659,((t3564:0.8928217215,t2855:0.5168398467):0.712301655,(t3109:0.4267446159,t2429:0.6532436686):0.3163490046):0.4123994086):0.6535635055):0.1853193012,(t636:0.6641721975,t1482:0.6693586495):0.8676144062):0.9786236645):0.2879595733,((((t3853:0.04455169896,((t3348:0.8287277913,(((((((t3255:0.617343432,t727:0.2154106735):0.1012310933,t568:0.6011281984):0.6217951449,t1577:0.7009667819):0.6624103796,t4192:0.2826758164):0.9119206527,(t2437:0.1889628186,t606:0.9565844375):0.7889055966):0.1195758537,t1643:0.3837734144):0.459727319,(t1117:0.2698911282,t2618:0.5926116046):0.8330705573):0.1032648038):0.414168054,t1206:0.8492447692):0.7138323134):0.9698224976,(t2881:0.2783603433,t2666:0.09769286518):0.4015298686):0.5426371822,(t1597:0.5330211655,t2976:0.5537147741):0.9786706397):0.875983658,((((t3953:0.1110261881,t1575:0.1038666626):0.7970567138,t3373:0.9463112121):0.3151886098,(t2256:0.1085454128,(t293:0.494775947,(t4485:0.1427976664,t4598:0.733915071):0.7007157537):0.956928045):0.3789895251):0.6441339992,((t635:0.2422891704,t2867:0.09260289278):0.08963877847,(t1335:0.969954977,t2245:0.1232232768):0.5454997937):0.4868543586):0.9475524505):0.5203342964):0.07464472926,((((((t1307:0.7448745144,t1549:0.8074504638):0.04834123538,(t1439:0.0464704372,t1143:0.5817480646):0.4043240268):0.970479633,t3538:0.4797786004):0.5141336827,(t3055:0.4362680903,t699:0.9673258108):0.4752776776):0.9283996534,((t2702:0.7561031333,t3119:0.3125612405):0.1533818038,t1454:0.4214225169):0.305761968):0.0512400975,(t1606:0.1635172532,(t4819:0.7113592383,(t4859:0.514698758,t4403:0.1776181532):0.6947528103):0.9120983917):0.2337281129):0.6036511629):0.4587940099,(((t1571:0.2347469344,((((((t1885:0.4648549536,t2205:0.3423239004):0.7271133899,(((t2943:0.733401611,t3524:0.4820557779):0.8062739889,t2161:0.6277431529):0.5578693014,t2002:0.5098305431):0.9197717921):0.3901361739,t4103:0.521268836):0.897731225,(((t2893:0.9821053573,(t29:0.6714318432,t2527:0.1848702407):0.3035584206):0.2247526785,t4011:0.4433292681):0.7310939576,(((t3330:0.04318036791,((t8:0.08348144381,t3148:0.7158740116):0.09420429147,t3250:0.004358904436):0.5794243214):0.7331423769,(((t4333:0.4577489819,t3915:0.3274542151):0.3613349234,t3060:0.1739946923):0.2744481734,(t3778:0.2019108806,(t4783:0.1555155762,t2150:0.642218383):0.7774707614):0.2836122969):0.349599502):0.4391534119,t3810:0.57538392):0.3221794753):0.5986578595):0.8557625245,(((t451:0.1744205549,t2300:0.2394699859):0.9904182642,((t4256:0.1891314115,t1156:0.4642362955):0.1918831174,t3870:0.3518105119):0.02302495181):0.8487849045,(t4932:0.685475261,((t4588:0.4764426483,t756:0.8637069417):0.05062302598,(t433:0.6106292214,((t2305:0.1798949142,(t4826:0.6423080359,t650:0.542726764):0.9022707359):0.567137663,t1237:0.5889266245):0.4441852246):0.7752515059):0.3152752926):0.3289108421):0.004907134222):0.9948941171,((t2922:0.332996178,((t2178:0.4801534307,(t3659:0.3371630004,t987:0.4654632651):0.4621898753):0.6625878876,(((t3416:0.909577426,t247:0.6051569113):0.3323170394,((t195:0.6771594041,(t3110:0.2685421258,t2118:0.9539791781):0.3602705286):0.6442660717,(t36:0.8065310852,(t931:0.4053850835,t690:0.09342274955):0.9165674765):0.219859709):0.8406578354):0.5010006183,(t2759:0.766777148,t1483:0.5716585668):0.8970591493):0.0163023218):0.2714568165):0.178097826,((((((t772:0.07234980096,((t3970:0.2556516523,t1774:0.7473114163):0.7367113307,((t487:0.3291204767,(t1915:0.6737416417,t2375:0.2043516154):0.475276906):0.6535222342,t2457:0.08902432304):0.7175145592):0.5115761126):0.6376107286,(t1045:0.6093795286,((t1441:0.2389455924,t2594:0.5455076403):0.5832318075,t846:0.4566403269):0.6144008227):0.8709627425):0.001059436472,((t177:0.3155212936,t4964:0.6736288012):0.195589324,(((t2438:0.503904656,t4529:0.5337130895):0.5037660524,t4274:0.1082925878):0.1722046456,(t573:0.9777321906,(t4346:0.6133071729,t1820:0.1443039007):0.911746575):0.1935084583):0.4993587444):0.5274296873):0.6773992765,t3404:0.6927224961):0.7624552012,t469:0.2176305975):0.326663807,(t544:0.6165601171,(t4816:0.9676665168,t1488:0.5706706215):0.7112443757):0.9025811823):0.4540297256):0.3950548787):0.07468537451):0.3215730707,((((((((t3597:0.2778795164,(t4768:0.3640320725,t3365:0.01658568741):0.3654491922):0.7050910292,(((t3962:0.05825421191,t4901:0.2232397597):0.349669602,t4518:0.4222123367):0.8222987463,t3332:0.01361004892):0.8684816549):0.457746349,(t539:0.07964168349,t4301:0.8961682636):0.478665082):0.2231043547,((((t4469:0.7857592576,(t3184:0.7019001909,t805:0.2204649528):0.8314798789):0.8302122941,((t2064:0.4600917122,t4311:0.1571462958):0.4913400623,(t3751:0.5141741321,(t209:0.24431416,t581:0.8388730425):0.3050636274):0.5205267509):0.9991438405):0.6470655424,t1369:0.3032506888):0.593077288,t4930:0.01240562112):0.8656959101):0.5042608958,((t1389:0.9001046948,t3516:0.9083159235):0.524610267,(t3786:0.3953087821,(t951:0.8998011777,t582:0.5606955183):0.580449735):0.6469036601):0.4011064649):0.590336259,((t3390:0.8315982851,((t917:0.5988220992,t2895:0.304213068):0.63178022,t1024:0.4064287215):0.1356475712):0.2176654835,((t2782:0.8343843911,((t54:0.7095737844,((t2187:0.6702263686,(t4364:0.9226403139,t883:0.717911659):0.2681436751):0.7453207106,t961:0.9855663897):0.6642666061):0.7799918016,(t4505:0.2609919675,t3131:0.6107438086):0.9097181126):0.6873232839):0.5555929479,(t3485:0.8560693327,t3433:0.2038849513):0.08809814625):0.08785267221):0.734919569):0.3240610184,(((t1154:0.2515835371,t2739:0.3910329333):0.7558204362,((t2707:0.7294976402,(t134:0.4851843589,t3190:0.2765906989):0.5612602818):0.04161430965,(t2283:0.9866168618,t116:0.2518849722):0.1378967855):0.9956789967):0.7314204467,((((t799:0.07380788494,t3631:0.6021298333):0.07271333132,t4691:0.4143983892):0.125977909,((t4439:0.2216689812,t1834:0.2722222121):0.09730051388,t4372:0.6841965835):0.2620185714):0.6279848493,((((t3382:0.8331882693,t3319:0.5416606984):0.1421982232,((t4750:0.8010217831,t1331:0.8270491208):0.710337963,(((t3369:0.6017719919,t4903:0.7916275745):0.5099746892,(t1397:0.6708954154,t830:0.5889599104):0.8337304106):0.6273544645,t1242:0.3373929246):0.3176791298):0.6009680098):0.5922259644,(t3059:0.5643526746,(t110:0.04729210632,(t3135:0.05593224615,t2995:0.2531147534):0.9050108318):0.4461702101):0.9784323771):0.4462039103,t4568:0.5483387862):0.04373953515):0.9025261765):0.9756627248):0.1298916589,((((((t2622:0.48191215,t3018:0.7444168539):0.1396671874,t1255:0.9484746438):0.5644144597,((t4379:0.2571523439,t4472:0.9462082535):0.1031161232,t3867:0.06826604926):0.5223095182):0.9378964424,(((t4054:0.7025092912,t3049:0.3619596492):0.3767808862,t1274:0.3648959694):0.3768779512,(t892:0.3261858011,t4153:0.2276025296):0.481593007):0.9884560702):0.5641372495,t3380:0.420128898):0.9593211666,(((t176:0.7291272117,(t330:0.2285093896,t1434:0.8140266149):0.2461815048):0.807351124,((t55:0.4705730958,(t2660:0.07369258977,t4108:0.8956295552):0.2999411812):0.4361851562,t3291:0.705221473):0.5625782227):0.07907431177,(t3790:0.715761242,t2672:0.904522578):0.04355101241):0.07673324202):0.1443478838):0.03750915523):0.8366761184,(((((t3880:0.009498371743,(t4866:0.829416381,(t77:0.2253937863,t2572:0.443762416):0.2815836424):0.9043423578):0.5083251945,t1180:0.809090744):0.1056817512,(((t2167:0.2688501936,((t377:0.4173331284,t1433:0.4375145175):0.4050415305,(t2169:0.01456472883,t2726:0.3118140332):0.1912904955):0.2985302159):0.9331593327,(t3660:0.6137267263,t2776:0.7787270402):0.9159004963):0.472194792,t1704:0.5341559888):0.03054457088):0.5855971854,((t2404:0.536513949,((((t1440:0.09305174975,t1703:0.4803197079):0.9445625183,t1502:0.7432476904):0.5563949964,(t1015:0.8012833111,(t1224:0.9997541034,t4018:0.9197692787):0.3193182859):0.5101394474):0.01780749904,(((t1399:0.6819714301,t1743:0.9916607647):0.1207272294,t1219:0.4478181247):0.7536255666,t2518:0.5641920085):0.6884802077):0.8074946171):0.6158743305,(((t129:0.3399521485,t3711:0.7580136268):0.7707588433,(t751:0.7696820339,t1310:0.2450120503):0.7058143062):0.02370283753,t3288:0.053807355):0.05112718837):0.9079942876):0.8495567185,((t2675:0.7575813993,t2054:0.7147112186):0.8014610172,((t3931:0.1651508578,(t185:0.5115830449,t1178:0.4925294972):0.9022826573):0.1235455943,(t262:0.7000839368,t1257:0.5507976615):0.6789103413):0.09495306131):0.3843114113):0.5105817989):0.5146451734):0.2653607025):0.9136875945,(((t4500:0.8369823263,(((t2972:0.7158841167,t2780:0.284190929):0.987707369,t759:0.01065959362):0.9310221791,t1953:0.3465204393):0.925167802):0.05986686749,((((((t1148:0.02572913631,t3722:0.5591894875):0.07132194657,t4593:0.6294633904):0.6290661243,((((((t2338:0.549685539,t1069:0.06180818868):0.4545175545,(((t3021:0.8694861883,t4617:0.08560915734):0.3727767158,t4159:0.3482113048):0.05875081173,((t1993:0.3553766415,t1207:0.5449401096):0.7918334526,t4304:0.3689052828):0.8009147546):0.09462153586):0.5054381862,((((t219:0.3396998795,t4846:0.5325069129):0.414721299,(((t2836:0.04239083803,t4832:0.3427956109):0.3126660329,t2580:0.7316503034):0.03463238687,t767:0.3322009149):0.4818835603):0.1121127263,t3999:0.2678172956):0.3720021686,(t3046:0.3483302249,t4223:0.6817101769):0.8559712677):0.8345867596):0.9626000368,(((t3942:0.1884689068,(t4615:0.4740338752,(t1465:0.4349414709,t311:0.4843829516):0.6494075418):0.2483379648):0.3801422291,t1280:0.5939206914):0.3179876846,t1595:0.1686464569):0.3168983343):0.1343455976,(t782:0.07591431658,((((t4842:0.1188244692,t4482:0.2925068098):0.07987042773,((t1721:0.9066852052,t1299:0.47622155):0.3871370147,((t3865:0.09764161357,t265:0.4461827667):0.9143252866,((t2704:0.674680145,t4633:0.8897380566):0.1852220371,t4824:0.5553345073):0.1240355191):0.06238064193):0.332126034):0.8693814105,t2134:0.6806587656):0.8521507066,((((t929:0.8002988698,t360:0.8183032291):0.009845185792,t1985:0.9126624789):0.8641686465,(t3359:0.38005118,t4917:0.5200587453):0.1385276746):0.16935671,t374:0.233729441):0.8459879947):0.8638308253):0.8059760581):0.3825276541,((t2519:0.06507771066,((t1197:0.8521167859,(t2476:0.3362973782,t1519:0.01831251686):0.2568920124):0.9586757338,t1671:0.22752981):0.2835618311):0.4942988891,(t4155:0.1929033052,(t2863:0.5156508496,(t448:0.2467189515,t3181:0.4397426683):0.1473059917):0.3392687084):0.8834627275):0.9323639253):0.7571982117):0.146434682,((t828:0.1404300332,((((t4227:0.1090566274,((t3725:0.6568177217,t4760:0.2437198397):0.6209602875,((t2824:0.5036426929,t2532:0.577140965):0.9957042299,((t4594:0.3095171417,((t980:0.006717170589,(t113:0.1866171919,t1814:0.3099160981):0.4332461294):0.3120039708,(t3471:0.2818450702,(t1708:0.8126967642,t776:0.1572455452):0.7730102895):0.3881945733):0.9498954143):0.6194212488,t3620:0.9156248278):0.407178387):0.1257237506):0.8613045986):0.4482021052,(t174:0.4459921096,t2207:0.6330357408):0.4553039488):0.7233551547,t3917:0.8036865431):0.9706450701,((((t3196:0.1340210841,((t3668:0.08909374918,(t397:0.4095169359,((t64:0.0420454077,(t1581:0.5624408533,t2053:0.6458594268):0.4943832883):0.6234029923,t3098:0.4803616086):0.5210689649):0.01183941821):0.6345238178,t2888:0.2901899479):0.7297252673):0.414621393,t840:0.8789100484):0.3682488038,((t4606:0.7089055008,((t3211:0.8094896381,(t769:0.7427877439,(t2293:0.2129344461,t2170:0.2581404191):0.6707271731):0.6197249177):0.2246985422,(t2024:0.7475114388,t2402:0.8309086787):0.1947811497):0.1233669175):0.7182466106,t3076:0.6135416909):0.6066994325):0.4392030942,(((((t3532:0.7363093009,t4740:0.6918825521):0.5426328494,(t4477:0.340933698,((t2986:0.9069751448,t2644:0.3677296161):0.7604535611,t1538:0.04005089682):0.5092387351):0.3587896966):0.7300251429,(((((t3777:0.5113722524,(t2783:0.6496069378,t4374:0.9223218246):0.6801467591):0.4291814344,(t2724:0.1584493206,t4540:0.041419046):0.4929437148):0.6988639643,(t608:0.5963644218,(t4453:0.5223036893,(t1989:0.3655173599,(t3633:0.3952991699,t2664:0.6562355424):0.9897218072):0.1945227834):0.6095784998):0.9909649452):0.02491518413,((t2044:0.2806361623,((t400:0.3591469496,t4789:0.7693813005):0.003038908355,(t809:0.1642546973,t1579:0.5120665554):0.46266476):0.1571634088):0.4460494511,(((t3185:0.5098795667,(t2790:0.7727807839,t3896:0.0366510991):0.3252120435):0.2835461679,t4972:0.6557989831):0.7941733068,((t2953:0.6719913266,t120:0.9234735344):0.2988974361,t4226:0.748288108):0.0006843348965):0.406105007):0.4261288217):0.1521647731,((t3720:0.5797700428,((t3160:0.1059150773,t4509:0.3929493751):0.5921763538,(t1427:0.6143258237,t419:0.7646313587):0.8615653722):0.3002199887):0.8456335361,(t3203:0.7404808493,t3592:0.1301935143):0.2416811101):0.2492846239):0.3009291091):0.5577313779,(((t2324:0.6973792133,t1029:0.2406649673):0.9089854714,((t4212:0.6869469616,(t4907:0.6388354555,t511:0.7353082828):0.5581969724):0.4012417477,t1336:0.575420161):0.5371567395):0.07368898857,t1736:0.9952599034):0.03336502588):0.7481436345,(((t2281:0.07256022957,t3337:0.6702866182):0.6520519743,(t79:0.7701018879,t2202:0.6437998023):0.09681177209):0.1554714723,((t2837:0.4657903819,(((t3327:0.1342333052,t798:0.9098021449):0.06409947365,((t3456:0.5288823529,t4682:0.8587517438):0.340582605,t2096:0.4005519615):0.2754574022):0.2233461717,t2988:0.1913614455):0.2467669721):0.8861047777,(t2692:0.07277175086,((((t368:0.8347213021,t1692:0.8053751253):0.2060074015,t3133:0.3197813483):0.5429385421,(t1239:0.6283798432,t1172:0.6885980866):0.1762647002):0.1264597785,t2812:0.9299414023):0.6506422285):0.5663623146):0.2927152955):0.591703448):0.2231672609):0.4563438375):0.9006128618):0.5514444432,((((t844:0.5720536876,t1946:0.6459459441):0.4623575248,t1407:0.2443437532):0.3069188646,(((t743:0.3695427808,t668:0.5327903118):0.7709483253,t4435:0.7198622294):0.9679794593,(t230:0.7378858384,(t4115:0.3709415938,(t3497:0.6867466671,t1890:0.7687915664):0.4324218887):0.2445589292):0.08001586422):0.357253839):0.1187253331,((((((t2918:0.2836341027,t3742:0.9865829756):0.8742365201,(t1447:0.2723757359,t4629:0.825598248):0.594071619):0.7224193772,(t4245:0.525883226,(t2145:0.08124206588,t294:0.1318612602):0.4427436804):0.437288142):0.2044635166,((t4522:0.7843593648,t1056:0.3918762994):0.9108479111,t4578:0.0546629501):0.7616028071):0.9593883697,((((t4715:0.8633079221,(t1443:0.997992194,t4164:0.3304667904):0.316386963):0.1026790114,(t3140:0.7829170411,t3541:0.4355274951):0.6869436326):0.1847779641,(((t3911:0.1359237181,(t1836:0.7276318881,t4891:0.5542378575):0.4816342618):0.6672778998,t2458:0.06779606198):0.5070403109,(t390:0.5007870004,((t2929:0.4699226981,t117:0.6878304353):0.608078378,t2190:0.4522648994):0.7273431094):0.6967197198):0.7575234787):0.3502995032,(((t1990:0.8247298382,((t1108:0.4754039778,(t586:0.7978644543,t2587:0.5099183416):0.1292700137):0.9669083608,t1418:0.3218551388):0.1109355083):0.1727054426,((t904:0.6865722307,t3079:0.3358588398):0.7931844026,(((t2556:0.3654822772,t1895:0.9362867798):0.301499065,(t337:0.08531545266,t1099:0.9171976291):0.2881839571):0.9741957404,(t4098:0.7821968084,t3897:0.6990451512):0.9611030989):0.1725696758):0.8070128013):0.2237789028,(((t1654:0.5596507511,(t1343:0.3806105421,t4397:0.08181013237):0.9760157776):0.638342805,(t3209:0.8896464566,((t2068:0.9345775826,t3019:0.1845085828):0.9683633423,t2304:0.9180472342):0.03846808639):0.9712639223):0.4561148998,((t4636:0.9530295967,(t1198:0.236275549,(t3885:0.4316755361,t422:0.07642566622):0.8550094459):0.4796622358):0.0280244872,t1151:0.9451876087):0.4585058074):0.9793507231):0.5831834909):0.8233530656):0.2641680946,((t2636:0.224543985,t2321:0.4922982377):0.386644474,((((t4856:0.3644424952,t4350:0.2913451123):0.2318404017,(t4665:0.8624050699,t881:0.7815753685):0.8935352324):0.1867722482,((t3489:0.2088805153,t4931:0.402618679):0.8080928561,((t3647:0.8394976901,t2800:0.9883021594):0.7658070254,(t1899:0.7681623884,t1060:0.2660722579):0.2057109757):0.4731678287):0.3794782853):0.4154082837,((t4286:0.8137694835,t555:0.3828139727):0.07427591737,(t4873:0.6539772099,(t4708:0.6638824544,t2905:0.7580100039):0.8293501411):0.09703705669):0.6259755625):0.4635938986):0.9592518804):0.8922702463):0.148444203):0.9343399641):0.3050308751,(((((((t2699:0.8012237339,t748:0.2971138852):0.203262157,((t2710:0.2165209348,t92:0.4777823789):0.8662169997,(t3737:0.6127622721,t1159:0.7211965011):0.03905109409):0.03947924264):0.9639162105,t527:0.3113428475):0.7297065221,(t1845:0.4124485485,t1353:0.7987751556):0.132463662):0.4291678162,(((t4384:0.5566129005,t2679:0.08691043104):0.7908748419,t1501:0.9259745886):0.4439405294,t4670:0.1426056463):0.3734167689):0.02125661052,((((t3367:0.6992029927,t3600:0.9555784948):0.3446867869,t2882:0.6148914178):0.7194906557,(((t3244:0.3919181186,t1120:0.8059376527):0.7269515335,t1213:0.5385679661):0.6660441153,((t1476:0.4000402284,t1635:0.1271797798):0.9407927128,(((t503:0.4732806697,t1566:0.7955614415):0.9320799538,t3874:0.8879348787):0.2920600104,(t3992:0.8307674127,t3045:0.8317342969):0.975092506):0.02935594879):0.771670348):0.2577193296):0.2427873353,((((t2261:0.412747598,t3747:0.02048412757):0.2675705592,t4432:0.9606487877):0.377837684,(t3040:0.8763132908,t1918:0.6285465527):0.9045121861):0.1957078567,((t2714:0.09086301387,(t749:0.5729200717,t649:0.1088098537):0.9886608864):0.6518330537,((t4123:0.449544901,(t162:0.5558426406,t202:0.5067394613):0.7176455399):0.1929503132,(t2649:0.729774453,t2579:0.919938473):0.08137630462):0.1766410554):0.4118976456):0.1122404407):0.2474422581):0.5146801604,(t4957:0.436951132,((t3087:0.5753574779,((t4815:0.8729810186,t4417:0.7263636144):0.9240609789,t3926:0.7476318018):0.7443793921):0.9512390504,(t2542:0.02293844242,t3484:0.724450717):0.6332095882):0.05870819837):0.2594692421):0.8810820857):0.1495012052,((((t2252:0.937488304,((t2504:0.9867848356,(t2317:0.943743147,(t4888:0.4339599803,t681:0.5157751769):0.8072077632):0.6645195126):0.7800397463,((t1301:0.102808954,t1994:0.9588635338):0.3589918974,(t765:0.2438202074,t1869:0.6499144204):0.7221982933):0.4669387878):0.5752985966):0.5863504931,((((t215:0.5727722896,t1904:0.8428073172):0.9647256706,(t3651:0.8852861626,t3089:0.4912025551):0.7162498443):0.1018998672,(t3299:0.9717466864,((t4258:0.5321303899,(t2733:0.7834940294,t2624:0.5684673015):0.7836041322):0.3227238562,t3421:0.5157056013):0.2613937748):0.7485891818):0.9887640327,(t796:0.07456954941,(t3813:0.3176633122,t2365:0.7265721126):0.665169504):0.772024906):0.9593439985):0.7177256555,((t1359:0.4387016946,(t357:0.6234506357,t3455:0.8149569801):0.8773031712):0.04695808911,t4651:0.1699312807):0.4375447084):0.4826788681,(((((t3128:0.933858467,(t3643:0.8054314826,t598:0.114755739):0.5425080012):0.8853158632,t4189:0.197608998):0.489264078,(((((((((t2176:0.8866862964,t2243:0.349186938):0.9862535028,t3213:0.7484109309):0.1792391839,t442:0.5261107686):0.8789361974,t2549:0.2448483566):0.2558014186,((((t171:0.9020560577,t3387:0.08420457202):0.5089216139,t4310:0.6069775461):0.2443324302,(t261:0.02017428586,t4514:0.2950399467):0.4964784621):0.1228583416,t2517:0.8857710287):0.3793380805):0.7143672667,((t3935:0.8670415848,t819:0.4806494319):0.2669103718,((t1414:0.3823591999,t4852:0.2762575715):0.2110284897,(t3152:0.4858272923,t3292:0.8382191358):0.3459897891):0.9452719593):0.1013912975):0.4099577169,((t3313:0.4846970714,t344:0.4913905775):0.7632178282,t3408:0.9770317287):0.3806704285):0.5981262724,(t4893:0.7027347006,(t504:0.6147600787,t2834:0.2072892771):0.03294551815):0.284631381):0.9845412469,((t735:0.4133740352,t2887:0.01353460038):0.2697750623,(t2310:0.8320460271,t3502:0.238386709):0.5066075828):0.8053682547):0.03046746785):0.6776652182,(t1160:0.4759279299,(((t3534:0.03743700986,((t2159:0.5152246291,t4997:0.835828562):0.5466340648,((t1191:0.2370719807,t2514:0.9385933217):0.8111704974,t3311:0.4988263033):0.5932488206):0.2217038346):0.06951630092,(t121:0.9985843864,t3102:0.2786764631):0.4404369935):0.1021271741,((((t2932:0.4698367058,((t3453:0.5631490678,t1707:0.9427366545):0.8003241699,((t3809:0.987141568,t3856:0.4675174854):0.6096208931,t4811:0.3921885728):0.01340659382):0.2553897954):0.4692299028,((t2318:0.790962084,t1547:0.9971508456):0.5075981405,t2094:0.2794549004):0.3334353804):0.9001811063,(t869:0.7318383113,t2121:0.1157797934):0.08645348134):0.6470715425,(t2141:0.6988364281,t1287:0.2330084988):0.5787462727):0.8766039493):0.06377180503):0.7309218945):0.684688176,((t1435:0.4589002668,t2472:0.1006257879):0.3694287753,t4737:0.8384478185):0.8062062371):0.7337007218):0.8959746282):0.8670469248):0.5744035253,((((t1515:0.7314148745,((t857:0.6937058407,t4188:0.3531461365):0.7961301573,((t3479:0.6725545558,t879:0.333635187):0.7967187157,t2858:0.7669005336):0.7693218684):0.8914538585):0.3327511635,(((((((((t4260:0.1064754152,t998:0.2574049702):0.195351867,t1093:0.3095765836):0.4667647046,t1769:0.3787756967):0.6617973789,(((t2825:0.9686296114,t4425:0.9006350124):0.3569231806,t2623:0.3120621273):0.573712192,t3615:0.8227911168):0.8945066028):0.1887728923,(t4659:0.5387571999,(((t1234:0.6469551385,t2711:0.8976277004):0.9942138637,((t2600:0.05442509754,t4275:0.1009387353):0.06102215615,t1223:0.3689330923):0.6042885287):0.2318796492,(t2329:0.1417985647,t4900:0.8538807179):0.341736882):0.08622415038):0.2224209411):0.103392605,((((t1634:0.4097096338,t1935:0.7501708716):0.548618919,t701:0.6546011576):0.1871454117,t4910:0.9149184108):0.9415250302,t74:0.7760634047):0.2042345528):0.829071708,(((((t4526:0.8667801642,t3322:0.3327974682):0.3578662928,(t290:0.188197478,(t2678:0.501363337,t983:0.1387528805):0.4288190794):0.143288722):0.7430509485,(t2752:0.1769741932,t1526:0.5169562961):0.8769162884):0.3189639973,t3188:0.9392928444):0.1430957802,((t2736:0.6635686026,(t1525:0.5984657467,t4407:0.5212928792):0.6164866213):0.2275492107,((t437:0.2526989372,t535:0.9105512251):0.3766164875,t2004:0.9699793241):0.09454207146):0.1359472563):0.3433801532):0.998804736,((t2641:0.8065798981,(t1394:0.123316949,t2866:0.500028698):0.03182609659):0.1365225806,((t3990:0.9320991945,(((((t2241:0.9083765817,t4216:0.06096653733):0.5385114108,t4251:0.3897182744):0.8903810428,(t2637:0.856557284,t4844:0.8184581583):0.1662196741):0.5301939086,(t936:0.4075053111,(t3094:0.05678880122,t3210:0.7504768339):0.828516084):0.3805793433):0.994455982,(t3368:0.5961451025,t1355:0.25489164):0.368018769):0.2507202199):0.2045819678,t2343:0.08113888884):0.4123190469):0.1976829055):0.8548034646,((t4191:0.7787076291,(t562:0.9033107206,t223:0.120892674):0.2589873418):0.3255200025,(t3735:0.9830148052,t4744:0.7531497411):0.804399993):0.8374731264):0.2969495396):0.7432577119,((t2477:0.940511015,t4202:0.08635751321):0.8326536708,((t4641:0.947972625,((t2489:0.2104190751,t2049:0.2205914403):0.4576844366,t439:0.6949845278):0.5420915766):0.6452192722,(t220:0.361704899,((t309:0.5279205495,t4146:0.7172448467):0.6132897523,t1668:0.6722036903):0.6506566028):0.8454663011):0.8304784249):0.2191784678):0.1775699225,(((t4090:0.6610624751,(t4215:0.1174950518,t4066:0.3435489342):0.03925399296):0.02184642665,(((t773:0.7976296747,t571:0.8299787033):0.3189818368,t1027:0.4338484704):0.1923920785,(t2025:0.7595946065,((t4623:0.6131265915,t877:0.1377695815):0.9687020318,t2668:0.6022374979):0.2372806836):0.987158336):0.4156237831):0.4042463067,(((((t546:0.6415574434,t2703:0.7609884674):0.7671890552,t1682:0.04103107867):0.02491541416,t4968:0.1276262009):0.1602516724,(((t971:0.6840931494,t4589:0.1188302776):0.1964416706,(t1512:0.4864058353,t612:0.2189840165):0.1026609538):0.8504436119,(t1457:0.7207864879,t4867:0.269663167):0.08948907792):0.2246571155):0.4927230231,((t1233:0.7077212846,t1221:0.3091154352):0.8350778508,t1830:0.7694225563):0.4522555959):0.9972205139):0.8277567942):0.5453875163):0.9862852611):0.609162159,(((((((t1261:0.8384260379,((t982:0.5945289328,t4758:0.3562018974):0.2427385356,t249:0.4056981746):0.7873854758):0.663503859,(((((t2000:0.9372161333,(t492:0.02706951881,t3820:0.524569914):0.2986664067):0.1111951028,t4332:0.02516701957):0.5346297799,t1155:0.114448624):0.631635915,(((t3700:0.2104276384,t3085:0.6688902327):0.3719157665,(t1641:0.6124643874,(t3132:0.1862125634,t1713:0.953584472):0.3054009029):0.5287357476):0.8384651057,t778:0.2240596258):0.2584256351):0.4515862034,t168:0.26872435):0.8658130984):0.8653871731,((((t4248:0.06238186383,t3794:0.407754425):0.6435149044,((t3149:0.4210521155,t1106:0.06191094941):0.7227248035,(t3945:0.8332250125,t97:0.815079035):0.9505549623):0.6193752047):0.7521885494,t2114:0.5959006359):0.9367237729,(((t3649:0.3690777873,t3312:0.6809168311):0.467814554,((((t369:0.4171270917,t3581:0.09447962162):0.4209646427,t393:0.2813234788):0.5762603432,t2686:0.1352388605):0.1256183407,t153:0.3658573376):0.5721252512):0.4771786497,(((t4051:0.9766817172,((t363:0.922010578,t2690:0.7955336913):0.7220247625,t4005:0.1658760691):0.7942039482):0.8030758812,((t3240:0.3127969161,t1518:0.855935558):0.06681084377,t666:0.115505697):0.5532432431):0.2473491449,t3074:0.5111969933):0.7523526787):0.4829520467):0.7182613621):0.9667968703,((((t1166:0.9580510824,((t4320:0.3563710554,(((t1591:0.8031230553,t656:0.7498661198):0.7078606791,t2086:0.0478030527):0.3088571164,t3785:0.5887087022):0.4599189693):0.01474047452,((t430:0.08852729038,((t3009:0.9386205161,t3080:0.9559022344):0.02579208068,(t4658:0.796111753,t2582:0.5118807643):0.5194544471):0.9710456664):0.3613994175,(t671:0.8261138103,t3669:0.9812037053):0.1859337194):0.1932827758):0.6496146901):0.3893976216,((t722:0.9433857107,t625:0.6256659487):0.4065536226,t4963:0.2028079275):0.9715239722):0.6343217012,(((t1979:0.1706170156,(((((t1936:0.4509160114,t3920:0.9356836039):0.5483713523,t1396:0.3265940335):0.6742770716,t1379:0.04072640347):0.3895931081,((t973:0.1888134347,(t2557:0.2812815872,t2535:0.08426499134):0.6898595081):0.337251103,(t3053:0.6475397204,t4855:0.6543969095):0.5885763231):0.08010148373):0.5287671254,t2974:0.1998683193):0.1185810096):0.3795859911,((t988:0.1751254715,(t3028:0.359916352,t3957:0.8019847393):0.503323273):0.8285343531,(t3894:0.03016225714,t1726:0.9830852142):0.1546443275):0.4455288732):0.950395176,((t403:0.7802719872,((t657:0.8533000886,(t3520:0.01778965606,t2005:0.4891612378):0.2043992765):0.6761122681,t3628:0.9852999472):0.4273264022):0.03824152448,(t409:0.7434687773,t2485:0.06917876774):0.3052965698):0.3023526508):0.9513719429):0.9452355229,t436:0.6515739495):0.3669611928):0.8413599627,((((((t2186:0.07278544805,t4865:0.7648233511):0.9057840202,(t2087:0.7882120516,((t1025:0.09697709489,t310:0.6350561862):0.1325417957,t4180:0.02870662673):0.03483329038):0.1877279298):0.9212437507,(t1908:0.9230633206,(t459:0.8965792398,(((t2225:0.1210412814,(t891:0.05017616483,t531:0.9493207065):0.8657229147):0.806501017,(t1650:0.8200322816,(t1235:0.1592528375,t111:0.7393423545):0.8525960937):0.4725027331):0.3224223452,((t1688:0.256521483,t2328:0.2419971554):0.2918403777,t676:0.0827699767):0.7778679132):0.4097151277):0.2327106078):0.9283119862):0.1433613857,((t4723:0.6426415988,(t4681:0.476801927,t196:0.04449872836):0.3522045633):0.8927918347,t4542:0.309316216):0.3803462666):0.005959084025,((((((((((t1306:0.6625155385,t3154:0.5788743051):0.6942271364,t4595:0.6462783546):0.3458511811,(t1761:0.6431226134,t3514:0.474795742):0.4640164622):0.4682177082,((t500:0.04715898959,(t2975:0.1638396068,t728:0.390467606):0.4828180331):0.5105539772,((((((t3966:0.3171784254,t4398:0.1057330519):0.9652270568,(t1291:0.04129211209,t2509:0.008624878712):0.1047771352):0.1940012944,((t974:0.3166699824,t1281:0.1223041036):0.04144057841,t3054:0.06751919142):0.3227787917):0.2302918162,(t3199:0.3355228603,(t3116:0.5217979888,(t2271:0.5784333204,t1699:0.165101069):0.7486459159):0.5733405876):0.7615066201):0.5743735849,t900:0.03434542147):0.5279786836,t4105:0.669628117):0.8647306643):0.9980840066):0.9034371506,((t1286:0.2711363558,t1026:0.2862989199):0.2237319851,(t6:0.07400500076,(t4022:0.3678018562,t4868:0.9102277698):0.3532601434):0.9931948367):0.4737812679):0.4719534579,(((((t3907:0.9337142562,(t1074:0.8272509351,t1009:0.1715387641):0.4012475861):0.08845907776,(t3015:0.6320556086,(t1426:0.9724213271,t1919:0.4914185717):0.7823729701):0.6979540864):0.3046035231,t2308:0.2255856206):0.6747009351,((((t1161:0.6109603324,t1944:0.3022694523):0.9122979813,t2189:0.09345585224):0.3761903755,((t1363:0.3734101611,t2216:0.8175721234):0.352427487,((t1474:0.1373761806,t4539:0.5819771958):0.4617048386,t2020:0.09170772671):0.1470321757):0.08851658273):0.7731607505,t3671:0.9741039332):0.7446371799):0.3371518401,((t4904:0.8478458903,(t464:0.4815310899,t4033:0.5043772548):0.5269971234):0.6340927281,((((t82:0.5138032995,t2267:0.7280561368):0.1798360252,t3458:0.5755626212):0.7099283771,t4786:0.6382588882):0.906457162,(t2819:0.7697539437,t1272:0.5290390612):0.9816745485):0.5191024602):0.7031795261):0.7615391072):0.5544314543,(((t3862:0.8909274847,t4473:0.8033184893):0.6809424772,(t872:0.9600820695,(((((((t137:0.2896960774,t3934:0.5574270131):0.191988342,t367:0.2278354552):0.1320280619,((t3572:0.8455217613,t2603:0.8748849405):0.6991057321,t3207:0.2626712676):0.8769331847):0.4287475673,t1578:0.2641025337):0.1135635301,(t2379:0.2068804726,t3535:0.6995875342):0.9549149347):0.1021306932,((t4761:0.9229997268,t4599:0.5993401993):0.4876780952,t1385:0.3649725451):0.8509466222):0.1570861537,(t3223:0.9277206375,(t2651:0.6944487377,t878:0.01476062997):0.1592911712):0.7858840495):0.6738772807):0.772159942):0.5585829311,(t3806:0.1493738552,(t1785:0.07195074577,t1292:0.3258715849):0.1357428702):0.6573399028):0.9519632084):0.938945764,((((((t2766:0.8301594898,(t3590:0.7848921847,t4955:0.8075683445):0.490549589):0.1866396705,((t1837:0.4750313454,(((t1599:0.9633741183,t1555:0.592134688):0.6933864525,t2470:0.8934372724):0.716476371,(t3687:0.4126009156,(t3177:0.9111810306,(t4071:0.9000572613,t2311:0.5913071034):0.7428332102):0.4588288118):0.2644274982):0.3406621828):0.4788396843,(t4703:0.7337097768,t254:0.1605558367):0.5408725387):0.2482185345):0.3822573805,(t3393:0.9401557567,t3812:0.08276883047):0.1854269009):0.1283881173,(((t700:0.904105593,t2278:0.2956481948):0.4576966716,((t2357:0.5898458927,t1741:0.7654539822):0.9976518925,t4876:0.06166582438):0.4478008831):0.2087158132,((t691:0.2371950878,t2821:0.7662359269):0.733832093,(t3908:0.6222557665,t1907:0.4776410062):0.4917978144):0.1147178791):0.8110312023):0.6142546136,(((t2110:0.4068866046,(t480:0.3765701388,(t3412:0.9884318956,(t4448:0.9171438639,t3047:0.6033808696):0.8318900899):0.6741980033):0.3691143668):0.8100946425,t2552:0.8348319742):0.1272541964,t1764:0.3008317272):0.1648190578):0.4921144247,(((((t4847:0.9897845511,t2013:0.1150005404):0.5742557517,t2100:0.7985914436):0.3909685558,t3903:0.1316637693):0.4614736927,t1054:0.7908657016):0.686560662,(((t3565:0.3815201928,t4830:0.4485210071):0.6334995779,t1576:0.4580239502):0.1816751629,(((t70:0.9830735284,t2468:0.2578766143):0.6959347576,(t818:0.5240444785,(t1531:0.7257173958,t3432:0.09622520069):0.7333663937):0.9187780591):0.1743005598,t4027:0.8889026763):0.7262699015):0.1681794105):0.7337998389):0.2630934385):0.04755302961,((((((t4671:0.7442549909,(t874:0.851199711,t2137:0.4230890083):0.236699943):0.7341618135,t4951:0.7607454644):0.733840876,(t2083:0.04760704539,t2163:0.9825374899):0.110005053):0.4061617136,t673:0.1749120215):0.2873740415,(((((t2956:0.5960845023,(((t838:0.7908684884,t34:0.8710529318):0.7887487609,t420:0.5159612976):0.09884430328,(t4754:0.4050655863,t2135:0.1973387431):0.8113414184):0.3808289689):0.4271105549,t520:0.6433347128):0.3988169555,(t2268:0.2554176836,t2874:0.5965922703):0.3500775318):0.8563295116,(((((t3559:0.8817794996,(t376:0.7587874776,t2773:0.5207101298):0.6402119787):0.07355266227,(t3828:0.1502614792,t3636:0.04580697953):0.3912103709):0.3825840859,t789:0.806178628):0.3769880026,((t4486:0.1778437102,(t3309:0.7644431598,(t979:0.3594361143,(t483:0.5529396688,((t1988:0.930516538,t243:0.7716484417):0.719999789,t1600:0.3973555677):0.2252132939):0.3861434972):0.7647850735):0.64130017):0.2068860447,t2749:0.6730570218):0.2033656833):0.3669486181,(t3758:0.04888459831,((t3392:0.05379986833,t287:0.1515188674):0.4357672776,(t2768:0.672232467,((t1070:0.754875151,t2537:0.5540782376):0.9425495574,t896:0.3974358556):0.761196075):0.8987171769):0.6668362382):0.7504312079):0.714615427):0.5349522748,(((t885:0.991088066,(t2342:0.7582814777,((t1357:0.01434732298,(t822:0.01482085302,t2833:0.2260544251):0.03959428356):0.7109994227,t2774:0.4716866894):0.154644626):0.5385832968):0.7989087331,t747:0.6955810771):0.1711032062,(t2575:0.000914911041,(t1497:0.8655935668,t52:0.3589440908):0.9986947051):0.2012554437):0.4113488775):0.1969548052):0.9451415432,(t4381:0.9372320261,(t465:0.6130136971,(t1455:0.9961758414,t941:0.5282937884):0.4460495182):0.9203294376):0.4709608646):0.06176591269):0.5089698557,((((((((((t1938:0.4506780782,t1614:0.5386361699):0.4906993643,t717:0.9253239811):0.2576218052,(((t2843:0.7066054549,t4948:0.5413063662):0.7606470541,t3802:0.4189484511):0.4081786035,t1877:0.1427718799):0.3787695817):0.679254065,((t1720:0.4620407138,t1053:0.1842573939):0.5910443421,t2607:0.579850161):0.01609432581):0.2693065526,(t4117:0.3403181308,(t1868:0.5548204016,(t938:0.6436053249,t607:0.3325492998):0.2859567988):0.9038223624):0.04803479696):0.2625273401,((((t141:0.8432537839,(t1133:0.9733708412,t1933:0.6802309277):0.266705412):0.2081605464,t4297:0.560999989):0.3155386958,(t2831:0.2022556351,(t416:0.2114946467,(t4036:0.03007476823,t4168:0.7299505037):0.8069894302):0.9685357288):0.6955827714):0.8897945664,t4252:0.373821028):0.07343759807):0.05587284337,(t2909:0.02543121763,((t423:0.229050504,(t201:0.756670919,t283:0.9636300642):0.6951249554):0.5098641352,t1311:0.4837725968):0.9555919468):0.08219328779):0.8008514971,(((((t2533:0.4055384023,t2030:0.148349392):0.7950930458,t4890:0.4953629647):0.8163733261,t4607:0.1730753644):0.9381625033,(((t2103:0.6955712952,(t2359:0.9607756922,(t3158:0.8080149069,t2576:0.2177550667):0.9614209188):0.5909861373):0.8428447549,t2051:0.950081232):0.4309321581,t4973:0.2176441564):0.8256461716):0.3971608372,(t1014:0.008585044183,((t2935:0.9731717333,t580:0.07845517201):0.8090646733,t4262:0.02521286788):0.9061860852):0.7562892493):0.736085708):0.8657910067,(((((((t1962:0.2764282033,t4897:0.7748251103):0.6094185496,(t1514:0.6496626851,(t4966:0.1292345531,t4553:0.2425795549):0.580833938):0.7858646319):0.3833850457,(t1425:0.3251084911,t457:0.5218465829):0.2789259893):0.5587647394,((((t4622:0.6883039747,((t3460:0.05981955654,t3955:0.02930417261):0.4105365505,t519:0.2245433386):0.6105637196):0.2866736674,t554:0.2343207463):0.4727460204,((t576:0.1660534365,t644:0.0896624194):0.08148735994,(t4626:0.07391705946,t4408:0.9781142285):0.3282505835):0.5720485121):0.5328586821,t2074:0.1049654153):0.2173090093):0.2013262406,((((t3582:0.0847752043,t4361:0.7883528466):0.158219846,t4497:0.6677595):0.01568840491,((t187:0.4613066034,t3906:0.1082955068):0.1333021545,t1588:0.7501992306):0.1213510714):0.6527330419,((t349:0.9946205963,t4531:0.1951954158):0.6257631481,(t2376:0.6568077942,t4129:0.4123618789):0.2328181588):0.6667658798):0.6299884976):0.7562745872,(((t3004:0.6346339073,t597:0.620633553):0.357741521,(t161:0.392554003,t4980:0.1988916437):0.3960133914):0.9987790298,((((((t1080:0.4690474747,t1560:0.8713968219):0.7616307552,t3824:0.539999132):0.4185333471,(t1630:0.07416326483,(t4086:0.784059162,t3909:0.2223871201):0.9800751642):0.9936017287):0.7710696119,(t338:0.8138993238,(((t1709:0.7687785625,t2879:0.07628527959):0.2422069267,(((t3629:0.8109712144,(t2826:0.6724647165,t4306:0.3102253233):0.08095337986):0.003892149311,t2142:0.8593762042):0.1722882625,(t2765:0.2857820976,t3282:0.5430412453):0.7162899601):0.5053268173):0.5977762137,(t4795:0.9500683988,((t4662:0.6225377815,t1902:0.9040065701):0.2117790605,t319:0.0185329353):0.6059972842):0.09835731774):0.5309695606):0.4371532651):0.223250227,(t105:0.8974860974,(t3466:0.9317452747,t2090:0.4536145667):0.1194735467):0.3637679331):0.08956605988,(((t4941:0.8772421756,t2165:0.03166935919):0.559009165,t3591:0.1495880217):0.2169536962,(((((t1891:0.803135732,(t2316:0.6746139228,t3531:0.4359723686):0.7968061415):0.7120484267,t2144:0.3050360023):0.05563539895,((((t953:0.883375485,t4140:0.9473110309):0.568522935,(t2747:0.07006613328,t3683:0.9785314517):0.08892077953):0.9886806344,((t2848:0.7819524945,(t1002:0.03516155784,t1568:0.4864505634):0.02226023027):0.205183605,t3543:0.9967197163):0.4352953082):0.9203425532,(t1400:0.8890176164,(t1838:0.07583892904,t2949:0.8483078515):0.6295645658):0.9362367564):0.9273173481):0.1270740789,t2125:0.6900143288):0.3252531351,(t4919:0.5798449882,((t387:0.6013426958,t342:0.799759612):0.8069799629,t2440:0.4402328641):0.2240310528):0.5152902934):0.9833524481):0.2571283078):0.06278632535):0.9387170237):0.8079462145,(((((((t4099:0.1476075589,((t1983:0.7779976348,t151:0.4868836931):0.4398316417,(t2566:0.08049368626,t3719:0.5150664123):0.7501435229):0.2025247561):0.3240894505,(t2977:0.2991218856,t3164:0.1800260302):0.2126411742):0.6573841551,t638:0.3981222135):0.7963230368,(t3815:0.04988141055,t4254:0.4772721485):0.2407619844):0.6450764593,((t4459:0.2004495608,((t682:0.2478228027,(t1799:0.950177189,t1954:0.2891972372):0.1529001885):0.5758770576,((t1847:0.2166110654,t149:0.08891969919):0.4781909729,t2560:0.003338381648):0.8941823482):0.1064330689):0.140564396,((t1574:0.09573221439,t3726:0.4771131929):0.6398814209,((((t4409:0.5210505298,t3274:0.6702788989):0.9170753672,t4128:0.5519625407):0.7459044706,(t2628:0.5195446874,t2978:0.4412475321):0.4762363236):0.9895288527,((t1804:0.6409427726,t1909:0.9536479027):0.4142348801,t2677:0.9229315934):0.6280904433):0.9825764794):0.7517568402):0.2207729982):0.5286096432,((t902:0.7106647748,t2273:0.05418361002):0.7888616519,((t391:0.8722135094,t2386:0.005610590335):0.5780541729,t2106:0.6971302987):0.5430430495):0.8371044574):0.3885032986,((((((t2709:0.2680609454,t4315:0.4109451238):0.9607587198,t4208:0.598536036):0.6926135665,t4190:0.836377145):0.06919118483,(t1043:0.9763812278,t824:0.8327646335):0.6247991573):0.7759258393,(t3438:0.31062946,((t4119:0.3578151183,(t779:0.759313771,t2670:0.4942744419):0.5561653636):0.2905909219,t3983:0.7723350637):0.9930033083):0.8207229639):0.87709307,((((t4883:0.1614186526,(t1828:0.04244020698,t3290:0.4884343925):0.825237399):0.05893003824,((t541:0.6257399672,t4543:0.8651569076):0.1371917431,t2571:0.2904471124):0.3945463425):0.1984697715,t3789:0.7216464027):0.1664548849,(t1296:0.344513501,(t1384:0.633941629,t4797:0.5164134887):0.3241540585):0.998613128):0.6874893839):0.8716411465):0.09805524442):0.5344464518):0.7405186919,((((((t260:0.2507187428,t2177:0.7446666346):0.5740439247,((t3883:0.8917173557,t1481:0.001645839773):0.9502385533,(t470:0.9070524068,t3536:0.9270657683):0.6161576016):0.08720533527):0.09771193657,((((t1463:0.9468360564,t4312:0.003088414203):0.5892880079,t226:0.06393030658):0.1208121611,t3610:0.06126804301):0.09399031359,(t2173:0.8652643771,(t1932:0.3783735612,t3807:0.4121621393):0.6579337139):0.9226838665):0.5853285412):0.5338085641,t2543:0.03012549598):0.9833827191,(((t4985:0.4891514953,(t3115:0.01249902183,(t41:0.429718443,t4489:0.7454760836):0.4824624101):0.7208294566):0.505386624,t1849:0.8315349389):0.8382718307,(t1866:0.4240220538,((t1739:0.9013967407,t914:0.592842666):0.3357350887,t2181:0.6979792251):0.3281055274):0.8828910629):0.1753245231):0.444046498,(t3787:0.4238979246,t2592:0.5759206994):0.07062226883):0.8195602051):0.2778694844):0.6936475914):0.7454872169,((((((t331:0.001258900855,(t1842:0.8426810028,t4604:0.101900626):0.8921731319):0.910922359,(t2166:0.2822414548,t3837:0.8861320738):0.5661305289):0.2784726473,t528:0.6692687559):0.09026659932,((t4038:0.688137731,t3869:0.6507896513):0.7819873746,(t10:0.09832983511,(t1964:0.01783519634,(t3295:0.01810539048,t4429:0.3056126037):0.4684832208):0.3024870628):0.32408983):0.06554584857):0.6915811913,((((t140:0.538707447,(t1813:0.9737908228,t406:0.9146813895):0.0139404072):0.5179335913,t1395:0.1671022053):0.06630506739,(((t1846:0.1745882304,t575:0.06606488372):0.05121920654,(t3750:0.8074101179,t4218:0.8760551764):0.6904774948):0.5398307,(((((t1404:0.4372743058,(t1645:0.1120327292,t862:0.1870628074):0.8171742591):0.5200770739,t4727:0.6965951959):0.70958078,(t3808:0.6628386923,t1061:0.6355812792):0.6660233163):0.3928669863,t2772:0.9452636368):0.02398764086,(t2395:0.04642515653,(t1517:0.01527130464,t2851:0.6325122863):0.9750993203):0.004411956761):0.18187857):0.1194557415):0.4567462869,(((((t2958:0.04278241261,t4718:0.007581777871):0.1497793843,t1819:0.6422391315):0.07796975272,t2611:0.6261486476):0.7246003412,t3037:0.03563066735):0.2424288741,t972:0.8600773131):0.4660500409):0.4862626691):0.656405237,(((((t1683:0.8622007535,(t3948:0.7465377178,(t1375:0.7756871185,t2466:0.3822525861):0.3342086221):0.900145835):0.9607019848,((t2410:0.2016909418,t2407:0.3658506537):0.08481021901,(t4078:0.6641064496,t2891:0.2501286268):0.7556717696):0.9054881155):0.8113460585,(((t1472:0.2535919365,t2778:0.7864895789):0.7628663729,(t1524:0.2093437398,t381:0.9656464823):0.7172290673):0.2402605833,t1125:0.4865085741):0.830634614):0.05871765199,((t325:0.7927478591,(((((t4289:0.4740466215,t9:0.3509804467):0.4462221311,(t3507:0.7460888659,t4324:0.0004911550786):0.3317674983):0.5092310328,t4712:0.2344937809):0.8969759038,((((t3263:0.2383883861,t1309:0.52299398):0.7838380616,t553:0.6410430628):0.1125996716,(((t1453:0.891159622,t4831:0.6445487945):0.3789737951,t4239:0.7697436521):0.6862481597,(((t1312:0.9210228608,t1392:0.8763652968):0.3142630875,t3733:0.2371287656):0.2141387952,(t417:0.1014713002,t2075:0.9296532159):0.2078870747):0.8053944334):0.7505275842):0.4161074904,(((t3462:0.3079894737,(((t2174:0.8830183835,t2156:0.2814247061):0.2028957107,t4875:0.711153432):0.3652879107,t4902:0.1109696019):0.8968304184):0.805462094,(t741:0.5453510007,t926:0.272668076):0.6667220898):0.2711983498,(t3980:0.643249108,t3901:0.8473828575):0.5015894428):0.6294423505):0.6942864221):0.7828101702,t2499:0.5651829729):0.03756267065):0.7182828856,((t4325:0.3843760965,(t4371:0.6752849137,t2497:0.8321610214):0.7038713084):0.4947092605,t3440:0.09533341741):0.9511569033):0.9916526615):0.5126578892,(t4829:0.867901766,(t3624:0.09966382943,(t2340:0.8381469827,t25:0.229025753):0.9239300077):0.9157600983):0.8570160838):0.7212946026):0.05614505871):0.1347896699):0.179004024,(((((t4183:0.03498433414,((t372:0.5551300333,(t3780:0.1108565032,t281:0.7668217737):0.2132483809):0.2269229502,(t4969:0.5026890975,((t4921:0.861788224,t1981:0.8658422679):0.8223575565,t4328:0.9785240921):0.09359156899):0.7693149552):0.5861181945):0.4189771602,t3271:0.3634349667):0.8514026599,((((((t2411:0.8555425031,(((t1545:0.8805351097,(t2617:0.9539210703,t1945:0.02046124288):0.1986357721):0.4555004118,t928:0.9091663386):0.2866201417,((t2846:0.8774828447,(t2234:0.2789092725,t1982:0.5240008282):0.7394791341):0.9824146666,(t3511:0.58374977,(t1978:0.9524354003,t1897:0.2839191675):0.900977982):0.4702134603):0.8745409644):0.8755603663):0.4850559512,(t4784:0.8569935164,(t1238:0.3834544918,t1081:0.6871774152):0.7750321925):0.7383749511):0.2173684193,(((t737:0.04707947047,(t2315:0.0131335035,(((t1085:0.1442769289,(t688:0.4225948318,t4273:0.06332091894):0.360167366):0.6402588482,t584:0.5241686848):0.04551452445,t2705:0.3695312897):0.9571782465):0.8510107771):0.1973716598,((t3142:0.2662496693,t2585:0.176249011):0.1439375351,t4825:0.8061203554):0.5573487636):0.9281474296,(((t566:0.01602618676,t4519:0.8811376041):0.01258683391,(t3918:0.8847400728,(t4990:0.04467654042,t662:0.1359336996):0.5715997461):0.4379778684):0.003358415561,(((((t3044:0.7272873225,t2368:0.5111867257):0.7907419445,(t4236:0.5922578021,(t4307:0.79216597,t327:0.4962332442):0.4674668859):0.4607940794):0.3805021483,t3235:0.4245512364):0.1447227809,((((t4630:0.4166991047,t98:0.360728845):0.7336044635,t2006:0.1058789492):0.5290927105,((t242:0.03397620865,(t232:0.792025676,t2945:0.7342344609):0.8379193454):0.374333113,(t592:0.3060678809,(t888:0.3819731651,t2313:0.6949764304):0.6168746925):0.453013896):0.07009968231):0.219409008,t4898:0.07854854059):0.7888847322):0.1785667145,(((((t4231:0.5565471456,(t1995:0.1051999792,t4182:0.1068127737):0.30994653):0.3943217176,((t3996:0.5026715908,t1722:0.161997909):0.7489924845,t948:0.6202464416):0.872788925):0.9570072107,(t3859:0.4782486556,t1607:0.2676505139):0.7284537461):0.6724800514,(t1729:0.08233478293,((t351:0.3908062752,t4039:0.4162951536):0.4119176301,((t2546:0.6391671516,t740:0.8678360889):0.4036977089,t2431:0.6461039258):0.9415479084):0.1670047916):0.5186912704):0.9147595547,(((t1586:0.5844852191,(t4590:0.2725288377,(((t83:0.6073071496,t2820:0.790920946):0.2633254107,t3314:0.8223320653):0.6128519936,((t3195:0.3681525749,t1432:0.8363896655):0.1537865219,t4721:0.1421761694):0.5124192131):0.3938512199):0.98133908):0.8061394785,(t4521:0.3266725265,((t3888:0.9370659885,t4052:0.8451652578):0.109648067,t1342:0.5737883619):0.9790101415):0.5815594292):0.622520054,((t4559:0.1039693684,t1203:0.7782272329):0.02401937777,t4120:0.2803581553):0.7174880006):0.4154839823):0.4478431528):0.9217911873):0.2020240971):0.8064125488):0.6079810599,t1636:0.633023788):0.7194870701,((t4437:0.6211200338,t4924:0.1343696644):0.2199261931,((t4442:0.7164402893,t3645:0.004344208399):0.2203782578,t4668:0.07003209088):0.8123746226):0.1622884963):0.274353212,((t3068:0.02712885779,((t3355:0.3490721274,t2439:0.7126059581):0.1576623369,((((t1940:0.3227759032,(t968:0.6473166358,t1666:0.5610308102):0.07277433854):0.8237063163,(t3389:0.8605106589,t3287:0.4660388273):0.9579659405):0.063101626,t2297:0.5864622192):0.777613508,(t3882:0.9162227951,(t1401:0.1600445909,(t3347:0.5035923193,t4228:0.6511271535):0.6863384787):0.5673014047):0.5360833576):0.7150119154):0.327039612):0.2326927281,((((((t4154:0.04322365928,t4406:0.3913266102):0.5922087433,((t3493:0.3591241143,((t1115:0.1067133374,t17:0.4835998525):0.7195002707,t491:0.3722579293):0.4519817093):0.4757496524,t2417:0.9235911733):0.6281368455):0.8461889941,t4625:0.3438730766):0.1781335664,((t2827:0.002891597338,t4677:0.557582486):0.9978079449,(((t131:0.4150625707,t2361:0.6444458757):0.9158283938,t2631:0.6905614906):0.9060758918,(t4276:0.8273106213,t3884:0.5526829483):0.4579119771):0.08785636025):0.2332825679):0.8059410187,t3814:0.7399132957):0.9743419376,(t2525:0.05654286896,(t2601:0.8358298468,t3965:0.9118884327):0.7776587303):0.7113744542):0.5372428887):0.3227321182):0.6413298144):0.6290934349,(((((t4041:0.5228962381,t1627:0.001575568691):0.867952934,t3574:0.01398085197):0.8202219894,t621:0.9621890488):0.392775184,(t3540:0.1689535941,t2981:0.5189318226):0.9914790229):0.1864537774,((t921:0.3107247734,t2092:0.6435538926):0.6337020474,(((t4194:0.3539550863,t4836:0.748016512):0.008191016968,(t3430:0.3700572492,t189:0.08896864904):0.6616010615):0.556941998,(((t593:0.5615417028,t2712:0.1911697255):0.5061164447,(t2453:0.1456847051,(t3521:0.02072023903,(t2405:0.3015296606,t3239:0.9455536096):0.9027137982):0.5238690295):0.167392408):0.6318228755,t4082:0.1265681905):0.1759760506):0.7682713617):0.1285495104):0.7818435784):0.4639461278,((((((t1065:0.3649198841,t4462:0.9803476299):0.8572854886,t2802:0.9177466545):0.6354648578,t2158:0.8330703538):0.6835954026,(t4095:0.9640308581,(t3914:0.08639404899,t4878:0.2956093063):0.915058604):0.5658768245):0.3130760377,((((((t3513:0.4926985763,t4742:0.1713720474):0.03446522658,((t4944:0.03274010425,((t3881:0.1834514337,(t2133:0.9456696447,t4834:0.9915652992):0.7238658152):0.2551717979,t595:0.001773009775):0.4606404393):0.6174675324,t2416:0.5957349553):0.5638072698):0.5898884651,(t3940:0.4219544439,((t4348:0.3323780179,t2554:0.8437642804):0.1190564423,(t714:0.4813977727,t4564:0.0843265627):0.2549812302):0.002373819007):0.9509006022):0.766826828,((((t2382:0.4162371904,(t4211:0.3241011822,t2813:0.5665224805):0.922518908):0.5596825439,t1248:0.9273098833):0.3171474424,t932:0.2255773256):0.5681617181,(((t851:0.04758917517,t2490:0.5037428553):0.4807859433,(((t3328:0.607637825,(t1036:0.7380286271,t2413:0.6477231707):0.6884423664):0.3673243951,(t4438:0.8178327184,(((t1007:0.3420159628,(t3376:0.526880641,t4922:0.9159669683):0.7963963007):0.7091333291,(t28:0.8811329848,t1858:0.4192148934):0.2757636309):0.3399364909,(t1717:0.5030125584,t4517:0.2275296324):0.07253484474):0.9957819344):0.3820185855):0.4518062847,(t1062:0.2145344666,((t792:0.466161239,((t4562:0.9103371883,t2869:0.6373814903):0.3392106562,(t1537:0.5806159235,t669:0.8572312908):0.3112449534):0.4407875263):0.6739137808,t193:0.5247864882):0.01056895312):0.2867935977):0.1432283572):0.3998494314,(t2721:0.3427498876,(t160:0.3079014064,t139:0.8181248563):0.3993248921):0.3385442861):0.4512501115):0.1643289668):0.6372160485,(((((((t1809:0.2743742915,t1374:0.07588838972):0.163215315,t203:0.08692997764):0.9977863396,(t3833:0.05709264893,(t3950:0.930542568,t4698:0.5188817128):0.8798279769):0.5122426336):0.1084716122,(((t4639:0.3797747679,t3921:0.4872465627):0.4696924475,(t1360:0.1954839027,(t1542:0.09081384027,(t4003:0.5133987428,t132:0.1346397384):0.5166641658):0.1923890305):0.7222856875):0.3579873652,(t2844:0.6246628668,((t3617:0.2293634242,t4445:0.8847098707):0.5363692441,t3036:0.1581476999):0.2868516413):0.04240048537):0.9688619762):0.8906615442,(t4960:0.02161891223,t2522:0.3146707753):0.7744397575):0.9448928288,(t1763:0.175089522,t3632:0.5565167926):0.9762002903):0.2005018219,(((((t1372:0.3979482471,t3873:0.1809465187):0.1026419406,(t3405:0.6877240995,t317:0.2236579154):0.1948082095):0.6740453325,(t1303:0.6145745746,(t328:0.9974307073,t191:0.4738072644):0.6517810205):0.08336393069):0.6262360588,((((t2775:0.853323482,t4501:0.5678611014):0.8511348616,(t2735:0.2143761676,(t3349:0.5723843139,t2197:0.2797142311):0.428057692):0.6681310623):0.1016703749,((((t4545:0.8446875187,t4546:0.9586555904):0.910577344,(t2010:0.9690103161,t4507:0.6674968465):0.5998386431):0.4938527758,t364:0.6595143478):0.6369382769,(t4790:0.8872128958,(t4806:0.6912033148,t4081:0.3098049029):0.7651461118):0.2540600298):0.5727399732):0.2310874076,(t1646:0.6274053974,t4770:0.1649599127):0.1415438459):0.7863129312):0.1311176214,((((t4480:0.02431422216,t3161:0.4293422059):0.974740233,(t3413:0.2783182992,((t1378:0.5338815884,t3449:0.1499826792):0.9598109883,t3986:0.1572936685):0.5383016961):0.2119678424):0.1462068879,t2598:0.5835357963):0.5385114672,(t1590:0.08735408424,(t3866:0.6356867389,(t3452:0.4849746742,t1390:0.1203193658):0.6014444688):0.6266944509):0.813703368):0.85233804):0.4757138109):0.5299160364):0.5006791628,((((t2084:0.06776476582,t4692:0.1935749054):0.2812043815,t2593:0.7352479929):0.402071689,t320:0.3255034469):0.237820639,(((t3829:0.1052221477,t589:0.3303739447):0.5244130429,t4914:0.7424133946):0.2030139873,(t486:0.8172420934,t2257:0.3449840739):0.9405021402):0.1823198164):0.1331453652):0.1866337503):0.4044701776,(t40:0.7396954286,(((((((t3002:0.6852158969,t946:0.5584484143):0.7990920604,(t104:0.06902575097,t3494:0.8091755153):0.8689501577):0.1988386987,t386:0.2276676809):0.1649681414,(t2285:0.3375814916,t3699:0.9767505212):0.6940094689):0.7207634128,((t710:0.06697947322,t2786:0.9867426108):0.1962887687,(t537:0.5994637476,t3082:0.8050547156):0.7374164038):0.9975958951):0.8182417778,(t3422:0.7033255326,t4331:0.3326557186):0.2293707686):0.09979112586,t4793:0.923151321):0.349182544):0.8866503311):0.3725876003):0.1390460476):0.2280120251,((((((t1973:0.5681348487,(((((t3229:0.7805157392,(t466:0.3327716063,t4152:0.3817971484):0.4025114547):0.6651734647,(t732:0.5434208023,t156:0.5412850794):0.3237201551):0.1447944236,t2219:0.9102349412):0.4823831595,t1184:0.4072862128):0.2450032064,t911:0.2835888166):0.5377613746):0.1189110647,(t1122:0.2377422729,((((t1380:0.7807854339,t1308:0.2072210058):0.5720790091,t3222:0.8536675507):0.1517659891,((t2228:0.5361712012,t2138:0.4783078234):0.1864071768,(t1698:0.6768435568,(t2235:0.4367993083,t611:0.680078374):0.05972482287):0.5225880402):0.6123893568):0.8062817527,(t2967:0.7356431375,t615:0.6619515701):0.6836880955):0.9685213517):0.945118469):0.2978463729,(t3183:0.3086081978,(t3951:0.0532471654,t1760:0.03415024164):0.1994543918):0.4938798756):0.4231613828,((t4396:0.1972055018,((t771:0.1347459347,((t4220:0.4199971568,t471:0.05849102349):0.8688212584,t1265:0.2717101292):0.64392026):0.3391475659,((((t4928:0.9370449199,t2302:0.9355290595):0.8499020159,t4106:0.551449612):0.3629592422,t1210:0.2066633799):0.932586422,((t1589:0.8193950169,t4138:0.1252328053):0.2386185986,(t4512:0.05892444891,(t3508:0.3232453533,(t2987:0.8841003783,t1824:0.7019936033):0.5719101583):0.6873349322):0.3162428355):0.8067172014):0.8100727643):0.3706360622):0.7185237675,((((((t3811:0.02107680356,t3834:0.2368205786):0.5584657642,((t4017:0.3043216916,t4695:0.8510380911):0.2780501875,t4989:0.03551002359):0.6405151556):0.8730203134,((((t3236:0.8442994095,(t3831:0.2763952499,(t4528:0.2636937087,t587:0.9072120709):0.7550555654):0.7838176035):0.8740844177,(t4556:0.09125507181,t2374:0.007643377641):0.150889264):0.7683658656,(((t2633:0.7992068757,t2303:0.02005536971):0.6293278912,((t1470:0.7725120594,t1971:0.5546999378):0.8203329467,(((((t4470:0.5059940862,(t906:0.8871804071,t3215:0.4986258205):0.05140326847):0.5173163663,(t957:0.01117769373,t2021:0.3229423512):0.444803556):0.7477383774,(t1139:0.4625291994,t2331:0.5197766582):0.1441617727):0.2521233738,(t733:0.02769374033,t1685:0.8955042909):0.4162672008):0.5213008074,(((((t3684:0.08856470603,t4269:0.2662818376):0.5561260013,(((t2095:0.4003607864,t1365:0.9543576112):0.8567097997,(t3182:0.6542554642,t3067:0.5708788061):0.8750566638):0.7847285131,(t2294:0.4903427805,(t4936:0.8913810854,t837:0.3679510078):0.5707110872):0.9375580689):0.568499259):0.4040739257,t1049:0.2311931355):0.7079991836,t2985:0.9063880169):0.462552292,(t1782:0.8283266071,t228:0.8130566911):0.9290688548):0.6042019625):0.1213189084):0.6614465045):0.09665516275,((t493:0.1447340774,(t329:0.6704639406,t3975:0.03804422752):0.2193056867):0.5729851332,((t4499:0.7190933293,t4385:0.7394440654):0.7739107006,t3325:0.7804009675):0.7355467891):0.5388434415):0.5661206443):0.7432524425,(((t3877:0.4450442507,t2569:0.1403219113):0.9407719139,(t3357:0.02589368471,(t446:0.4004695402,(t4837:0.4379605721,t3424:0.3760701779):0.8215350939):0.3660148084):0.4466754429):0.1294153058,(t847:0.213489006,t2263:0.8867688843):0.8737378272):0.2284750796):0.2603327467):0.899828607,(t4295:0.05474576983,t4993:0.4926645362):0.2793980078):0.7331618185,(((((t404:0.2692052699,(((t2875:0.2962249662,t4970:0.5814666576):0.5213604465,t3111:0.6784509725):0.7169421795,t3791:0.07321951678):0.5351432806):0.8071425313,(((t1947:0.8714264557,t3927:0.8236694389):0.5862342878,(t2344:0.6687899644,t4415:0.5754951099):0.1105119165):0.3638015448,t3144:0.7900502102):0.40638194):0.9886402194,((((t164:0.03858151101,t4702:0.5568432503):0.8817469862,t2295:0.9559942377):0.04625429818,((t4481:0.5223434814,t3477:0.4114326495):0.1025199392,((t619:0.3386846958,(t1321:0.6779230861,t4467:0.4992252209):0.6786007786):0.02111861389,t2609:0.7472152608):0.8529291435):0.3655042709):0.9912209187,((((t2531:0.8329237453,t3771:0.4078233151):0.3647480933,t3317:0.1598947728):0.4351535088,(t3253:0.9512488341,(t2688:0.5810862109,((((t1135:0.2399002539,(t4851:0.3016948188,t514:0.5169305124):0.6658709128):0.3822170547,t4673:0.5276661867):0.7872603245,(t1796:0.6387306058,(t3712:0.2599982698,t2667:0.7856275097):0.6384053552):0.9936061206):0.6556682584,(((t2634:0.3005347394,t2906:0.2392392647):0.7344452916,t1674:0.8617159489):0.3230981717,t4548:0.5729222409):0.3774140733):0.3561273788):0.8041099315):0.08594133356):0.2224021014,(t165:0.3756423013,t3598:0.4661686651):0.2325377406):0.1371505742):0.5309759991):0.445349995,((t4101:0.733551767,(((t3333:0.2547055841,t4638:0.4853408784):0.1054612524,(t3245:0.3426477602,t274:0.4544992005):0.2099003398):0.5370014843,(t2807:0.2238084555,((t2799:0.7879253177,t2926:0.6336122488):0.2761631515,(t4009:0.02825549059,(t1826:0.7701620583,t739:0.3638238281):0.4894896767):0.5767587626):0.6722433202):0.1698046851):0.04157164111):0.276389024,(((t2770:0.4858519675,t124:0.6178175132):0.5646765314,(t4821:0.5258944745,t4261:0.6343801415):0.2452167862):0.9520252817,((((t57:0.1727824407,t1087:0.5076099653):0.6837265086,t3220:0.3042535151):0.2148457123,((t3638:0.8380151535,t315:0.1557895457):0.7073305079,t3300:0.4930051169):0.2224458547):0.1813289518,(t24:0.9768751285,((t1258:0.9233513507,t3588:0.5852432335):0.9873646139,((t2332:0.8053701464,t2510:0.4986264673):0.1329056283,t4419:0.5379136784):0.382758087):0.949759685):0.7425863803):0.2592336817):0.7542579756):0.7333036403):0.06144924741,(t454:0.1460019485,t1719:0.8226911214):0.03393371101):0.4070150023):0.8204112744,((((((t2292:0.748293052,(t2196:0.3429385978,(t2381:0.2443336579,(t2131:0.9729624165,t2507:0.8186007715):0.3065986007):0.2684547822):0.2586890231):0.3386101869,(((t3878:0.7278338587,t880:0.6147270235):0.9365753622,t4351:0.7659432772):0.2398818748,((t3374:0.8922289826,t2203:0.2460515513):0.9668913118,t720:0.7018498126):0.1789269715):0.1218655624):0.861960572,(((t4268:0.6640417795,t2017:0.8501156636):0.7050163336,(t3666:0.1271682917,((((t2596:0.909126516,(t795:0.4240204785,t2284:0.1429779688):0.4582759645):0.1643847714,((t3708:0.8361999497,t2213:0.6154832179):0.2140542411,t4238:0.9220388066):0.4539744849):0.1023058428,(t3340:0.03208618145,t50:0.8600961163):0.06177349994):0.5789808799,t2854:0.1229858967):0.6482802005):0.8009758056):0.3255659833,((t440:0.7058783264,(t192:0.8405572474,t1948:0.0250730163):0.1919589711):0.4904956671,(t4711:0.002739443677,(t4998:0.8118480658,t23:0.2753967468):0.6499649023):0.1824278953):0.0004081011284):0.09842019086):0.8613727053,((((((t2968:0.4552056591,(t947:0.3533337654,t2419:0.477048564):0.7395741297):0.6098678717,t388:0.4350367251):0.834151759,(t4428:0.8294762154,t2946:0.3494584237):0.02741308277):0.2273236816,((((((((t4576:0.2502201002,t4008:0.6835207734):0.878534012,(t856:0.9568232943,t1211:0.1636032383):0.02496005199):0.6843170957,t4043:0.1733671159):0.2716424514,t3905:0.5589628308):0.7930894364,(t3912:0.9776230401,(t4884:0.5602029441,t3825:0.5106574355):0.1959099527):0.01253564074):0.8810927535,((t1765:0.1217542361,t3569:0.5116582713):0.9754076379,t354:0.5338736952):0.7047852322):0.8536742842,t1952:0.6875796088):0.1722936532,((t3688:0.0747548691,(t1327:0.5918712898,t2450:0.4797310829):0.5419676448):0.461589844,t1859:0.5983966419):0.4327027961):0.7291815819):0.7381163279,t2306:0.3976691964):0.8085656778,(((t4064:0.8399842945,t4612:0.01232564542):0.8946208444,((t3510:0.7850880802,t2008:0.01812405279):0.5891563015,t1728:0.6169143638):0.8403662604):0.4019139621,((((t4292:0.6755691119,t2231:0.1982526924):0.3153131849,t763:0.3400511742):0.9811609718,((t3179:0.2933150055,t3415:0.6545480902):0.2167472199,t478:0.5715648343):0.6222857975):0.5494367834,t4535:0.1612658466):0.156308094):0.3467787418):0.591101317):0.3966062542,((t306:0.2069458584,t3007:0.3139159668):0.09378669481,(((t626:0.9913690069,(t7:0.03045098367,t71:0.7070615049):0.302808278):0.07598233898,t4532:0.6809716558):0.0894019336,t1094:0.5922333098):0.7046006129):0.938991586):0.2646000248,((((t4733:0.6874674284,t3187:0.2101472316):0.4669775963,t30:0.1291112311):0.07232087408,(t146:0.006808004575,t4583:0.5057103497):0.1183745698):0.1367379162,((t1083:0.9516856624,t435:0.7188548846):0.9037920733,t2023:0.4446699813):0.2243572443):0.9656858682):0.6086111453):0.6338212332):0.8866167373):0.1765589612,((((((t2423:0.1931195427,(t2534:0.8265983288,t3095:0.8243496944):0.9685227429):0.818495837,t218:0.9061410672):0.1691182684,(t2979:0.2181168981,(t4603:0.3930472101,t3385:0.7549666502):0.3901740469):0.7256010091):0.3328026992,(t4652:0.03233881178,t3107:0.302586054):0.7455977481):0.7628007988,(((t1266:0.983895601,t4699:0.9588194846):0.6439623553,t2185:0.6627187505):0.1626397525,(((t4383:0.1369052392,((t2663:0.6139643532,t4749:0.1409131789):0.4238608999,t590:0.9145384533):0.3411657589):0.8307616871,(t2536:0.1972671619,t1592:0.5925870582):0.6297995944):0.7798964998,t1776:0.4173541584):0.6133281582):0.3215427741):0.6857114967,(((((t2486:0.2534273854,t2275:0.7503685441):0.8798180674,((t2298:0.8103414394,(t2393:0.6213371884,t3630:0.09625232546):0.5807391966):0.2528979054,(t1690:0.2659385919,t2963:0.6608020701):0.7823831602):0.5500285327):0.904479346,(((((t4823:0.6681049403,t3012:0.4771814465):0.001934527187,((t3445:0.4074460873,((t2073:0.05536987958,((t339:0.5311197552,t2984:0.2774925723):0.947368707,t3414:0.8739840162):0.2069304287):0.03447289602,(t2326:0.02159788716,((t4175:0.7152250065,t2763:0.8634950928):0.1842890319,t2973:0.7143661538):0.5395991243):0.8536179641):0.2181519282):0.7373283878,t4962:0.3794792204):0.4467060424):0.4920569763,(t1665:0.3566718965,t4716:0.06141600548):0.6661270389):0.2428287917,(t3121:0.9890797352,t3163:0.447577527):0.3350870924):0.3109745036,(t2645:0.4413208286,((t4569:0.493079684,t692:0.9750845334):0.4223562998,t2665:0.3293162403):0.3930454468):0.3956420592):0.6204367881):0.9429676086,((((((t2430:0.5746829135,((t2387:0.1351242282,t2952:0.4281056658):0.7893913621,t324:0.2337104923):0.05011863238):0.8662156262,((t3627:0.2208117065,t3162:0.9804464094):0.01925483998,t1373:0.3310051456):0.6589865861):0.1584067957,(((t1097:0.7363349653,(t3461:0.1723615169,t4395:0.2414065036):0.9996072757):0.6580929365,(t3208:0.09314439818,t3308:0.3058410604):0.6357309732):0.2954049155,(t2435:0.4997220875,t4019:0.05289115245):0.9358429248):0.588728331):0.1243982443,(t3441:0.3734580029,t1528:0.03928226442):0.8957787689):0.8023243253,t2750:0.8384765207):0.8156214131,(((t135:0.3041282676,t3383:0.2028905882):0.2016111831,((((t2447:0.3909312193,t1810:0.5047113306):0.5381376615,(t3298:0.3905494898,t1071:0.4519690571):0.1127350007):0.03558178642,((t4657:0.8147869897,(t1783:0.4115657948,t455:0.8079520438):0.5385496116):0.3866672067,(t1388:0.7102561847,t4549:0.4729876851):0.6076403616):0.6183508371):0.09672315838,(t1852:0.9862481051,t2337:0.5894514318):0.1018238291):0.9706920604):0.5705323697,((((((t3418:0.630589288,t1275:0.552800958):0.626953651,t356:0.8769844605):0.8894203347,t2160:0.8381429922):0.06424663588,(t760:0.2627444521,t434:0.1229700453):0.9742061791):0.8042677077,t3447:0.187801935):0.9826317083,((t4996:0.6019638295,t2947:0.2761884399):0.04066902562,((t4335:0.04200083041,t2254:0.2211135556):0.7957244494,(t4040:0.1525988658,t1958:0.7806118259):0.1824159967):0.2243631266):0.2031931076):0.9341691528):0.8750062145):0.985997125):0.2238566605,((((((((t2748:0.5147456988,t1450:0.1291040692):0.282114788,t2785:0.05630724551):0.2112486598,t2551:0.2134872985):0.3733677457,t4141:0.5857856236):0.2292994589,((t3201:0.5778236259,t803:0.7630242116):0.3694555731,(t808:0.4002081675,(t4290:0.6134481151,(t2151:0.984259336,t2388:0.8453873931):0.5660084763):0.3215975091):0.5562006109):0.1521002862):0.995996942,(((t3077:0.9619504833,t4030:0.5840772144):0.4716762679,t813:0.9034354612):0.3038505164,(((t1750:0.6701757808,(t3041:0.2480250197,t3767:0.3457303897):0.4241412666):0.02314847778,((((t3350:0.01810546033,t4945:0.4614656721):0.6157923194,t3375:0.341232372):0.9711858954,(t3221:0.1464637632,t4317:0.7987553533):0.5066201154):0.3606127743,(t2035:0.405798197,(t3103:0.747801837,t1925:0.7850207575):0.5787311855):0.1865394271):0.7217439034):0.9823501331,((((t1570:0.8203902727,t965:0.1333067529):0.9048731402,t1130:0.9299169162):0.5035231116,(t1559:0.5921176076,(t3048:0.3960266709,t1136:0.3922457895):0.3909584808):0.9072791217):0.1079079628,(((((t2009:0.4942429264,t2099:0.241926902):0.02628811472,t2363:0.4211727823):0.7634884361,t1778:0.9654949405):0.9248338398,t4326:0.7726047933):0.4332441075,t2937:0.5632132294):0.795783286):0.7780447309):0.932150177):0.2160618624):0.1000490414,((((t506:0.2078445498,t2488:0.889951654):0.2628886627,t777:0.07831284381):0.7166936682,(t2130:0.5421452236,(((t4848:0.5781095168,t1277:0.1640591684):0.7114398866,t2885:0.2409533134):0.9576027545,(((t4302:0.3168382023,t754:0.6732195586):0.4850227439,((t4567:0.1100374947,t2348:0.7073331592):0.8015286857,(t3172:0.3295738439,t1442:0.4322834858):0.2422076396):0.1587184076):0.3766002597,t4150:0.6743513003):0.5084877575):0.403385967):0.5630816938):0.5360187853,(((t3379:0.5832756017,t2319:0.7406214147):0.4977829461,(((t3200:0.4904401915,t4083:0.7751636889):0.6517900915,((t2564:0.2804510219,t2168:0.3410268098):0.2062114044,t1966:0.4478792124):0.920973025):0.9055810389,t3302:0.2922328282):0.2938925768):0.2391921822,(t2652:0.7000637052,(((t1205:0.5777065705,t1020:0.5073747342):0.7099037117,t1594:0.887284691):0.9458675417,t1867:0.7512733827):0.8785453064):0.4940198082):0.624363808):0.4717816585):0.2085660666,((((((t572:0.379532306,t4070:0.6123310712):0.7303390841,t4524:0.1788888681):0.2872162133,(((((t1088:0.3532999163,t1264:0.4849720895):0.4018452859,t4014:0.03348378837):0.8097322928,(((t4557:0.8371669352,t3339:0.2984669444):0.9601581199,t721:0.5613423467):0.7700469464,(t770:0.8289872683,(t2505:0.0979451118,t2698:0.6768026797):0.5925026506):0.6979261627):0.33731049):0.4637486436,(t920:0.6574982002,(((t3734:0.8100221425,t2716:0.08582024323):0.03301334707,t860:0.5374278997):0.719058923,t1851:0.8791913262):0.6500799684):0.8640943333):0.2830932396,t3704:0.3622485015):0.3388570235):0.3210957269,(((t266:0.8070734625,((t2838:0.5118395165,(((t3805:0.9000154028,t1772:0.9251944011):0.09063745057,(t68:0.1498751407,t1616:0.7297034457):0.1283029625):0.8368552891,((((t1534:0.1605187713,t1612:0.3844619761):0.7593841029,((t152:0.4901892631,t4788:0.2173398167):0.1914218902,t647:0.2524600618):0.2133166476):0.3881987894,t4879:0.04109584726):0.3808377746,(t173:0.7821991851,t3891:0.7861911105):0.2433159414):0.9594336278):0.156430877):0.4771051374,t1216:0.5008824256):0.01799923601):0.2551365206,(((t950:0.3982100484,(t2220:0.5394569819,t4935:0.3081180819):0.1425234729):0.9693913762,(t2805:0.2292339762,(t2129:0.7433598125,(t845:0.2803928845,t3304:0.7964382907):0.2958148243):0.6521595784):0.6605819915):0.09217150649,((t2817:0.1206720865,((t3713:0.5433490546,t2211:0.2905615226):0.06590573862,t1931:0.9975637584):0.592594242):0.4749389139,(t1771:0.4169110474,t395:0.9526264132):0.7062237912):0.8062261732):0.9369536764):0.02601226442,((((t3776:0.3378560282,t2691:0.8420331138):0.593576598,t3779:0.9705707263):0.5653205868,t1341:0.4235697424):0.4848375767,(((t1530:0.8114094511,(t4895:0.9634755745,t3143:0.5191032316):0.9955240546):0.7111812828,((t624:0.4989298831,t3205:0.9874260847):0.8355864293,t565:0.8201916947):0.111524242):0.1626242092,(t4347:0.3448880739,(t3146:0.7751316263,t4492:0.9612808193):0.04527360504):0.0450264432):0.6674791523):0.673401268):0.8590438054):0.5752116239,((((t3745:0.3185973421,t4940:0.04776686919):0.07478568028,(t244:0.2252970303,t551:0.628346398):0.03934562579):0.6898457739,(((t4319:0.4919053507,((t705:0.1891188803,t4265:0.8596725466):0.7656589863,t850:0.3478679662):0.9094135438):0.6368377386,((((((t2614:0.6073432744,t76:0.5420673529):0.1312365578,t2914:0.171136348):0.8098387904,t3783:0.6902297013):0.6755238217,t2923:0.6870346002):0.6546089521,(t3016:0.2255403192,t4145:0.6210981531):0.4195707596):0.9879787012,(t2658:0.3318476437,(t269:0.1757362757,t1032:0.167580812):0.1086388414):0.9542251762):0.01195296762):0.282735395,((t1004:0.8293355212,t3605:0.2742171576):0.9109663779,((t206:0.9336392879,t4102:0.6185394912):0.4976757427,t1974:0.5364129208):0.75979211):0.3271802349):0.5149520426):0.8091221796,t2070:0.4599797614):0.6744872606):0.4556475009,(((t1916:0.6711884656,(t4850:0.4626499987,t1204:0.3298903366):0.7507011695):0.2849657605,t4334:0.5787722857):0.6108045524,t3698:0.07360934606):0.452089644):0.9232895148):0.5710479196):0.2088616579):0.1036200167):0.9835169467,(((((t858:0.06411242601,t3241:0.6167466976):0.1597068114,(t1541:0.4900703458,(((t2272:0.7899209,t4424:0.8265837955):0.1608338694,(t2253:0.08435524232,t1808:0.7128797399):0.9365540836):0.1442598519,(t3958:0.2045844104,((t4739:0.8380573962,t4678:0.4767366378):0.176096939,t4956:0.07749349973):0.8375570385):0.1329541006):0.8871714333):0.8702931136):0.3858317055,(((t1109:0.540027864,t3677:0.3816335269):0.8096927011,(t3396:0.803992528,t4016:0.7597528619):0.5524927927):0.0880539692,(((((t3268:0.1683249963,(t1825:0.7506772834,t4076:0.6392439331):0.9240729709):0.63906717,(t1669:0.4184154717,t1789:0.2789682061):0.8277164558):0.001375993947,(t1773:0.935941552,(t4992:0.1547551188,t3682:0.9978090229):0.7883346223):0.1852043264):0.5674477248,(t3944:0.8792232096,t3071:0.5456949391):0.4710590157):0.04378668033,((t2718:0.4613593251,(t4946:0.323215971,t4035:0.4105415372):0.3110346287):0.2111645374,(t924:0.04812495364,t887:0.8861145906):0.5511388378):0.3112093078):0.6461817189):0.2764341964):0.6193111232,(((((t1082:0.3545431283,(t112:0.9897957388,t533:0.6385162552):0.5577652529):0.3026472079,(t1485:0.9488416535,(t629:0.1344688442,t1199:0.7352579175):0.4240722817):0.7483503884):0.7632226036,(((((t3801:0.2241399826,t1901:0.2831259898):0.9354316208,t3723:0.3100351428):0.1441025338,(t259:0.7228130172,t2674:0.5040441325):0.5963819926):0.3485045191,(((t322:0.1461221024,t1554:0.3647905516):0.9185589897,(((t3803:0.6605719575,t992:0.4497298943):0.6843213777,t3640:0.65091625):0.6794951626,(((t3746:0.07412167033,t2128:0.2570761496):0.7678958459,t1746:0.9027049204):0.613769741,t2274:0.8372515563):0.7715896757):0.1552456077):0.1046929746,(t235:0.5276032682,((t1246:0.7057658478,t1452:0.4426586833):0.5321583336,(t1464:0.845162042,t2511:0.6405652175):0.462449142):0.03457865026):0.4133716801):0.8167003952):0.0843595576,(t4912:0.5801591789,(t4142:0.8180290542,t3480:0.8435524576):0.8996644802):0.5256826757):0.7885311178):0.9829331683,(((((((t1448:0.4981373355,(t1725:0.484600334,t1320:0.2279035149):0.1896492571):0.2846998344,((t3672:0.1743044206,t1153:0.5483347054):0.5445810712,(((t4309:0.3829213898,t3618:0.488213955):0.870641639,t852:0.4823688797):0.2478069675,t4263:0.8091821475):0.6262365412):0.7395905533):0.7988226681,((((t2737:0.3169172311,t1496:0.5678035892):0.7635582353,t1886:0.07464399491):0.8886354889,t1468:0.3775549468):0.995909086,((t1853:0.1006418986,t3826:0.2115952426):0.3963795118,((t4399:0.5926499446,(t2242:0.6836288641,(t774:0.9055709962,t4365:0.09117947379):0.2154549742):0.002541460097):0.6309567771,(((t3924:0.09210896259,t967:0.8954006704):0.9270673953,t3003:0.3667957832):0.6712272374,(t4288:0.6774673092,(t297:0.6477955675,t2971:0.9330043944):0.5347735363):0.4678568786):0.2744014449):0.5362804357):0.1896059418):0.9824639419):0.2910927257,(t4838:0.5823269445,(t1829:0.09542567679,t1494:0.9373426249):0.8674853742):0.4305811899):0.9418918968,((((t2408:0.8752732803,t2822:0.2914511007):0.4904692224,((t2339:0.706327063,(t1621:0.04946335545,((t4656:0.4187640275,t507:0.5830970602):0.730714587,t278:0.7531718691):0.3621049337):0.4529870385):0.4615745521,((t3270:0.7319646161,(t4318:0.5327777814,t1658:0.2603637984):0.5142575572):0.4667672347,(((t3243:0.01085781376,t2685:0.3068525738):0.5892158072,t2494:0.1190970417):0.9669882841,t960:0.01768629532):0.7562129954):0.2649265544):0.04158595973):0.5476686584,((((((t4885:0.6036086108,((t163:0.05341434479,t3678:0.3839624913):0.8366795059,t4646:0.7459367712):0.9285725402):0.2042326485,(t800:0.1949443868,t1529:0.637674721):0.7487082528):0.9519212856,(t4298:0.940584904,(t3112:0.3193803735,t421:0.9848027977):0.2660647412):0.3867670198):0.8447402944,(((t4611:0.2840887408,t697:0.5459590608):0.1260031154,(((t817:0.168413579,t1005:0.04619488539):0.5606830614,(t2461:0.5133481955,t719:0.5807964071):0.2418366654):0.3707122523,(((t4015:0.2809359985,t3232:0.9727459485):0.4669691774,t1691:0.1167021689):0.1773620148,((t3960:0.7819925714,(t1270:0.8251114942,t1657:0.8424768401):0.9290333618):0.8754190612,(t2385:0.726571664,((t136:0.9743728598,(t1037:0.9002091601,t1693:0.1854209953):0.1567238006):0.2268195439,t3695:0.168481156):0.9425161555):0.7382732432):0.1674307517):0.9388361066):0.9433112922):0.07135570259,(((((t698:0.09476080979,t3612:0.5202466031):0.3063060495,t4308:0.7797084867):0.4945989328,(t3656:0.6734930673,(t1504:0.9186055744,t522:0.4559930016):0.733440347):0.6986998825):0.6505058671,t2188:0.5812294136):0.2950857924,t1511:0.1919472597):0.9852753803):0.9860354227):0.8168691553,((((t2346:0.8252629736,t4999:0.7314522774):0.932968115,(t1906:0.6576297546,t1801:0.3423777614):0.3427604758):0.9780303473,(((((t1872:0.4545243341,t3976:0.407866965):0.8170823257,t2046:0.6592412295):0.605292287,t2616:0.8327596944):0.3932966609,(t865:0.8652938486,(((t1100:0.4206713825,t1367:0.7636579236):0.8426675398,(t3316:0.1339187864,t724:0.9703182587):0.06509670545):0.7531080809,((t352:0.6948613795,t1059:0.3993287652):0.111172345,(t26:0.37449057,((t4619:0.6784601388,t3400:0.6304920518):0.777129425,(t821:0.06690427661,t2921:0.3082080726):0.1424122038):0.9643930101):0.009625436272):0.6136510167):0.2339827868):0.1596097811):0.2322916591,((t3073:0.7737157289,(t3171:0.993981977,t3459:0.4856949998):0.995985952):0.3542228341,((t1348:0.9890636127,t659:0.4483737007):0.1820985358,(((t1715:0.814271773,t282:0.2210659839):0.6863403202,(t2811:0.3204415792,t1881:0.001117120963):0.5955292173):0.2411594845,(t181:0.3470769129,(t3530:0.1804235061,(t268:0.6668869823,t1493:0.8584465648):0.4320758998):0.9514574884):0.6850090863):0.1220752629):0.03290794301):0.9641948475):0.29137076):0.6051525972,((t4498:0.5664488261,t184:0.07591620577):0.06855167286,((t2809:0.06503812806,t989:0.1750362192):0.6576453603,t3851:0.6571312097):0.6393228138):0.0267996327):0.3974038174):0.4680626276,(((((t4337:0.8608171451,(t536:0.6687977931,t841:0.4141964123):0.8912750718):0.7674989346,t2659:0.7293822088):0.8846026615,t1803:0.2793462821):0.8416803593,(((t3272:0.7777515596,t2080:0.7941319598):0.02631839667,t3214:0.5942487724):0.5160064469,(t4013:0.4476619852,t1807:0.2488512525):0.729859788):0.9063261263):0.2504933844,((((((t2695:0.6049396207,t2760:0.2017731613):0.6053460324,t942:0.2537483457):0.1888931596,t2422:0.8775926859):0.6970652381,(((t3573:0.932032407,t3795:0.3131393332):0.6880443385,t4037:0.9025869428):0.7243335943,(t4817:0.6195163475,t461:0.03366296994):0.1930365085):0.4740079097):0.9647499376,(t2355:0.3847696183,t157:0.1842326594):0.1302771189):0.7576121869,t2038:0.6688211027):0.4653305307):0.1350538032):0.7489073463):0.7532517379,t3963:0.5097164386):0.6461684881):0.2250971582,(((t1941:0.9922585045,t1781:0.8412561384):0.5502698335,(((t2396:0.4528068693,t867:0.7831091029):0.1561398564,(((((t731:0.676843561,t2501:0.4730992818):0.957833352,t993:0.8961748842):0.08554596337,t970:0.005601833574):0.4731912559,(((t3126:0.8893651974,t3673:0.2956292017):0.711548175,t784:0.02041305206):0.1260020446,t870:0.04179394385):0.8649424119):0.7283189215,(t2814:0.7600165009,t3604:0.8496893423):0.2718608612):0.6049078095):0.4245714592,((t3842:0.6401632235,(t864:0.6753688771,t1446:0.6583346301):0.8580043251):0.2429852469,((t1436:0.5414854873,(t4713:0.4202378707,(t4067:0.5034863369,t2629:0.8074325623):0.5543840749):0.9349443933):0.02426961903,((t3522:0.9907591171,t704:0.4116551063):0.6100854974,(((((t2236:0.9821746647,t2856:0.5976271427):0.4596348302,t474:0.7539694754):0.7961019615,(t1670:0.844424784,t1914:0.4355164059):0.4819602959):0.1552385129,(t1217:0.09929499333,(t4313:0.2940169228,t4195:0.4057241958):0.3305140531):0.1464614107):0.9916022678,((t1402:0.7184557286,((t2830:0.8429572626,t4338:0.8593967271):0.1068637548,(t2198:0.6595575139,t3900:0.5030493571):0.3447983882):0.5798652845):0.3265116732,(((t2459:0.7050274473,(t807:0.02875242569,t1116:0.4342548172):0.5984207008):0.4081060758,t3743:0.4978628801):0.8867255775,((t3:0.4002111179,t2529:0.7841629293):0.9892691621,t4563:0.2127349749):0.7635733793):0.9813009251):0.4457525031):0.7050674709):0.01797982049):0.1475887189):0.4840322225):0.952921378):0.6017595802,((((t371:0.2917505798,((t2445:0.5741511493,(t4053:0.5087266094,t4199:0.9622469801):0.8829812761):0.3686938186,((t1017:0.1899550389,(t1675:0.1847110714,t3321:0.1000435338):0.8136302102):0.05761236348,(t1187:0.9236007365,(t1297:0.643090605,t2478:0.4971563988):0.01230134862):0.3631388545):0.9496942644):0.5330755601):0.3273670757,(t4147:0.7434184744,(t4209:0.006608442636,t3547:0.3864654989):0.5036692368):0.7791287575):0.5481267711,(t4655:0.2857632346,(((((t4503:0.491377546,(t4446:0.6829332202,t4068:0.8872857341):0.101997524):0.4042064734,(t3063:0.3124079481,t456:0.7027420413):0.2071606708):0.213365573,t3648:0.4300736585):0.4465680362,t4234:0.6146167591):0.7072238619,(t738:0.9487351964,(t708:0.03216825891,(t3058:0.02041206858,t939:0.8475363867):0.3119547649):0.9873044582):0.7471097999):0.5353735681):0.7399762019):0.3138862944,(((t3577:0.4774673784,t1079:0.8010679984):0.7665190892,(((((t956:0.8557815284,(t1536:0.01053266483,t4947:0.2072744241):0.1867820737):0.9786464318,t2366:0.2062665191):0.9265410195,(t833:0.7665833295,t2913:0.6900583762):0.6524839972):0.7507895418,(t2266:0.9176654874,t810:0.2819611947):0.6825832187):0.5155524861,t1850:0.01931662858):0.5042956853):0.5280335112,(((t3954:0.03723145206,t4026:0.2671931747):0.621120679,t3663:0.9844085721):0.3474534666,(t2610:0.9074339687,t3252:0.1153040451):0.6771469368):0.2899941714):0.7533737174):0.9421177851):0.214113242):0.4109271232,((((((t3738:0.4897033526,t4061:0.5179669438):0.5344402904,t1478:0.733168399):0.81612151,t4765:0.7530770674):0.4004520692,t3334:0.7293370827):0.1946834333,t517:0.9455822895):0.509756065,(((t3715:0.411435524,t301:0.2152178786):0.9039262128,(t379:0.5406909853,t4158:0.5671118884):0.2476651608):0.2585469089,((((t2019:0.1609158688,t4672:0.9744254511):0.5824158809,((t3818:0.7242818398,t279:0.6975175098):0.06363215949,t1705:0.577070741):0.6929105257):0.8594568998,(((t2223:0.1445876227,(((t1167:0.3875568907,t407:0.4401276307):0.02390055754,(t3156:0.8969328478,(((t4515:0.6290904046,t246:0.8875405404):0.9459705176,((t1821:0.1054594966,(t4860:0.8164967902,t2463:0.3606067856):0.4598261092):0.8025708992,t3732:0.4307876192):0.9053041022):0.9632835123,((t2758:0.6740479255,t4363:0.9797021735):0.369993452,t251:0.1658985827):0.8324960438):0.923767681):0.7272378544):0.7545190814,((((t2451:0.8340366441,t4225:0.4726320782):0.2216190945,((t744:0.06684687221,(t985:0.8464032472,(t3764:0.5871600183,t1034:0.200227855):0.1730330288):0.3641210548):0.03705981933,t4690:0.1805881504):0.001105027972):0.9149940698,t3664:0.1224591259):0.047328969,(((t1871:0.4440514371,t1382:0.5654030533):0.2993641077,t398:0.733409388):0.5125037802,t2999:0.7959689612):0.7150404458):0.4553740651):0.1601598465):0.2918342238,(t1411:0.6928606085,t2449:0.9594068481):0.9958454215):0.3353172482,((((t3679:0.1677752833,t1376:0.04711970338):0.6202824602,t1756:0.4104493335):0.8587978831,(t3170:0.3176038689,t2864:0.957967134):0.02636153693):0.6062387773,(t4610:0.3108427317,(t1362:0.9030945443,t1068:0.5604104076):0.8757567252):0.7632295315):0.04064894933):0.2232549784):0.628997812,(((t3768:0.1566263596,(t3607:0.4206105024,(t3703:0.3426952295,t123:0.3864531559):0.2454953203):0.9778551538):0.1311787055,(((t4818:0.05779135739,t2380:0.8055878512):0.7620784689,t1679:0.6460430915):0.6275952449,(((t530:0.7700445983,t253:0.4565344534):0.9460171629,t4632:0.761197855):0.4460401647,t4680:0.3564638582):0.5242779846):0.4834057768):0.8769592731,(t761:0.3393602732,t4731:0.9288442452):0.8950331125):0.9656578552):0.5733375484):0.8089238452):0.371760186):0.4571701568):0.5839023788,((((((t1405:0.3410794481,t4093:0.8740192691):0.4900524048,t4447:0.5486000609):0.4527245886,t894:0.9202072534):0.5744145846,t4544:0.4655123905):0.2877174865,(((((t3714:0.09355640807,t687:0.9913308492):0.1579199715,t4246:0.05942391441):0.1237697485,(((t3978:0.6360942677,(t4241:0.5603810607,t1976:0.3958477092):0.8848404167):0.1439993505,t365:0.06173945032):0.04628659366,(t1652:0.6518087266,t2217:0.6493082752):0.5947727524):0.1308768615):0.8713698816,t3525:0.3978211852):0.1043918738,(((((t2237:0.4411648072,t4785:0.7887479833):0.6400109541,(((t579:0.1236431387,t4608:0.3669953754):0.8551357121,(t2883:0.2008376981,(t654:0.2831728391,t549:0.04207564401):0.1124596118):0.2878458707):0.1101977334,(t1522:0.7643014907,((((t3637:0.7181239906,t2839:0.5545086074):0.1233856338,t2377:0.5307070061):0.2645297893,(t2112:0.1060882825,t674:0.1496263011):0.4675709056):0.9756635111,t1398:0.7076241844):0.7406989874):0.5116205458):0.4633492469):0.4129221223,((((t1263:0.4424055975,((t804:0.7907508074,t4250:0.3605149454):0.5134488565,(t1473:0.4992189119,t618:0.2603028438):0.1634404908):0.7843309953):0.3743015395,((t2320:0.3843308464,t3465:0.9278697739):0.8083463043,t4440:0.628224422):0.05299037904):0.9401315134,(t3583:0.3739482611,(t1076:0.3160902497,((t1215:0.7531720635,t1735:0.2548443023):0.7895536548,(t543:0.6813795976,t4959:0.212971952):0.7440906379):0.4517899104):0.502027449):0.2298337934):0.533468059,((t4006:0.4790110339,t2209:0.6109415914):0.6524037013,((t2347:0.9133991173,(t1822:0.9322780124,t2870:0.3376011704):0.1167531998):0.7109583481,((((t2415:0.2207468068,t2330:0.7700544712):0.3702927576,t3153:0.7907847287):0.9436856124,t1864:0.1997117186):0.9736551445,(t3916:0.4504204586,t4894:0.2011600961):0.04128022632):0.2135370239):0.8806718746):0.8986412424):0.5699410723):0.000889308285,(((t558:0.3986893112,t3550:0.5465479738):0.9157213969,t2801:0.6400023517):0.1303906906,((t3017:0.4600611243,(t925:0.07483185246,(t726:0.08860425605,t712:0.6382322453):0.9106433736):0.6903121625):0.1526340102,((t2769:0.07937857555,t2484:0.00636099698):0.327379134,t3840:0.06049633096):0.1150006331):0.1386192669):0.4229549451):0.14104647,((((t2797:0.6915456241,t3845:0.2853532957):0.002746100072,((t4299:0.04495831905,t2547:0.832870461):0.6874656754,(t1714:0.8282494994,t3336:0.6876233588):0.5157194266):0.7740377728):0.4490776972,((t2162:0.3726100142,t4163:0.5217220357):0.8599568645,t4798:0.4332366856):0.003032601671):0.599205577,((t3793:0.2167206712,t1602:0.4828743548):0.6336604476,t4942:0.129051452):0.8173094194):0.6487480053):0.426485128):0.9773376414):0.2332423734,(((((t3675:0.2571347454,((t648:0.5368512599,t4614:0.1861010774):0.6708834157,((((t1019:0.3640183373,(t4222:0.5973960282,t3504:0.7826897602):0.9706250103):0.03938376624,t4913:0.3402702992):0.1179172271,(t3062:0.2907878899,t14:0.04600134864):0.5087479879):0.5838079664,t4339:0.1632072914):0.2161066621):0.8087973462):0.9553436746,(t2568:0.7699490909,((t211:0.6279546837,t2371:0.7174582616):0.9557109701,t2180:0.7659825566):0.5124245761):0.8390805058):0.9175898388,((((t685:0.3090886951,t3386:0.1632468766):0.6776911383,(t1431:0.8405884341,t2966:0.1350890822):0.3237425582):0.6846183382,((t1350:0.8948231146,((((((t3346:0.4652242505,t3233:0.8187016025):0.6190449626,(t2927:0.1817500934,(t4766:0.5061134007,t1243:0.2611983921):0.874855252):0.191382831):0.240372947,(t2127:0.4083701479,t2561:0.9648618139):0.3547373989):0.1857547446,(t2465:0.3640749089,t4242:0.8528760753):0.1369490041):0.1869288546,(t3474:0.255303846,((t1520:0.3506630973,(t658:0.1827699845,t4455:0.6013996941):0.003884866601):0.2177621098,t3014:0.4282708431):0.4948792197):0.5679433024):0.7563491242,((((t3285:0.5464606709,t3654:0.8065130122):0.485662417,t4369:0.4019155286):0.421482132,t2944:0.852736376):0.4287808735,(((t2938:0.7252988992,t2226:0.5725544426):0.2285876956,t2565:0.941062941):0.09651523549,((((t5:0.5622136258,t2550:0.7672292592):0.9598785795,t1619:0.9202110937):0.6087566302,t2033:0.1718263414):0.3696553046,(t736:0.5931533442,t2964:0.1665791923):0.08873515227):0.7938032232):0.604756119):0.3674918958):0.9211683061):0.977715356,((((t4321:0.5483830238,(t60:0.8771601054,((t3086:0.4951712782,t2713:0.7673781179):0.9599792035,(t3483:0.559349718,(t2397:0.6046585266,t3755:0.1617642222):0.3125891963):0.3613814288):0.6659193118):0.2526101363):0.2362044044,t87:0.8992475935):0.6564086722,((t3093:0.5009844925,((t179:0.7124912015,t199:0.6642799305):0.7274399356,((t4025:0.662153438,t788:0.3679777402):0.4186581359,t2487:0.9260152183):0.01705316757):0.6735027055):0.06042522332,((t3984:0.578337745,(t2993:0.5466957504,((t2351:0.3863696747,t922:0.003211963223):0.1377389745,t3134:0.0002677245066):0.1861927486):0.2443795658):0.2822314708,(((t2022:0.9661625989,t4965:0.3808704342):0.410354868,t1503:0.9690009682):0.04416323476,((t115:0.6886158257,t623:0.1603065783):0.1316250502,t3613:0.9137728077):0.2649233781):0.4119740911):0.1699066835):0.9302723659):0.09755894146,((((t820:0.41237715,t4577:0.3932309777):0.0177700331,(t3863:0.3919039837,(t1843:0.8328757617,t502:0.9966381101):0.9595946865):0.8455722586):0.7603968547,t2270:0.1011411694):0.2674119461,(((t3251:0.75886738,t4456:0.02969338908):0.1968087065,(t1113:0.1841469354,t2077:0.9948621711):0.1941053602):0.9457943118,(t4224:0.9676091748,t4400:0.7844602531):0.6271663648):0.8098717353):0.3605214506):0.3990343572):0.72140503):0.7477272335,(t3816:0.237037628,(t3189:0.3180443179,(t3844:0.9766364673,(t1663:0.5793221609,t2612:0.1566872641):0.2382052164):0.8207444798):0.7014923515):0.000950576039):0.6527773989):0.08068528376,((t1913:0.2927209353,(((t564:0.9854166233,t3166:0.724259912):0.06748726103,(t3964:0.9388971915,t2878:0.9828156806):0.4245496443):0.8142656272,(((t4570:0.669150352,t4475:0.1197499835):0.04868988553,t2146:0.05920276046):0.8770124083,((t304:0.8131135686,t3264:0.3294975276):0.02364071808,((t3830:0.9030749612,t2390:0.4097846355):0.1162557362,(t4207:0.4562103222,t2588:0.3052081454):0.7898879626):0.3730835761):0.5011584221):0.3482263223):0.2859137726):0.7367207818,(((((t3275:0.2965475882,t3938:0.6318004408):0.6899144626,t1075:0.9548938728):0.4058938681,(((t2401:0.6032664403,(t91:0.1916772807,t1633:0.2797593242):0.8171005715):0.9557044648,t4886:0.1509225138):0.7267728702,(t667:0.69198466,((t2939:0.204459033,t3796:0.5214365677):0.7705864243,(t3147:0.6041777888,t2026:0.5567409468):0.1945171847):0.5293050292):0.5256341663):0.2590727985):0.3291487223,(((t3717:0.3102658635,t2212:0.6570851081):0.1496878567,((t3075:0.570383966,(t3509:0.596302384,t2200:0.7389818386):0.07671168726):0.07556008059,t570:0.7031274596):0.9816620047):0.7960608648,(((t1182:0.9967376189,t1626:0.4292174319):0.5938372426,(t3797:0.2202536878,(t2650:0.9118863153,t458:0.7497696455):0.7708439888):0.2235526047):0.1152040223,(t3753:0.1393688803,(t4582:0.6319757288,t4034:0.1301449435):0.4675617015):0.1175752387):0.2792746995):0.1754937284):0.8086005675,t1569:0.9257724534):0.9076594345):0.6875401447):0.5104553818,(((((((((t2563:0.1363624658,t3765:0.9202692467):0.9257132113,(t2520:0.6759444729,t1298:0.5676707628):0.8963690957):0.3723585384,t1573:0.31263947):0.9224747794,t3731:0.903335416):0.9975980555,t889:0.8668577496):0.232557948,(((((t1546:0.2543979045,t2409:0.1255876366):0.836668466,(t3180:0.9936766224,t19:0.8435047844):0.08403993864):0.4758515987,t2540:0.870227223):0.4236848291,t4777:0.7962760825):0.03359326138,(((t2098:0.3824362066,t1631:0.7811347474):0.687681691,(t4484:0.9383989652,t2930:0.1308226418):0.3607058111):0.09610536019,((t3206:0.04067969532,(t863:0.6647462756,(t1751:0.7216929472,(t4560:0.6345807051,(t4581:0.5007709875,t1001:0.6118313314):0.7747018184):0.4160925555):0.435028417):0.1332621083):0.1909463892,t2352:0.2718817648):0.7102355091):0.8143547003):0.36480806):0.7283255935,t2960:0.3068709415):0.4948668045,((((((t2753:0.764312085,t716:0.218243862):0.9250942331,t1228:0.5936011197):0.4565174433,t3488:0.4740776198):0.5358699113,(t1550:0.8656544944,t1072:0.05063128681):0.1541706251):0.51893936,((t183:0.6190196164,t1381:0.07202614751):0.5134890084,t4217:0.4622565708):0.4610308979):0.2344676564,t540:0.6314234834):0.8761790851):0.7457153781,((t4724:0.777753806,t4653:0.9002032038):0.3986418915,t418:0.9348111798):0.8059940161):0.5312227998):0.6387782258):0.8071524436):0.3774198121):0.5995206884,(((((((((t2072:0.3388171913,(t170:0.5656538592,t3967:0.4063566003):0.5896465038):0.9441501922,(t627:0.4855824807,((t1189:0.5794946223,t3197:0.1915091269):0.2378133114,t3481:0.5996205928):0.7407016151):0.01266718842):0.2284986938,((t806:0.08717884682,(t1507:0.02452146541,t2341:0.720454541):0.7639606923):0.4820979536,((t3977:0.4992395511,t3030:0.2948540405):0.9467841212,t314:0.2200207585):0.16470029):0.1974597035):0.2590556734,(t4906:0.7817889815,t871:0.9511088107):0.6128140548):0.425111552,((t1977:0.7853290171,((t2194:0.3953421833,t4284:0.1290105192):0.4117005398,t4579:0.914105603):0.9291999149):0.2821669155,(t1326:0.1465432816,(((t4375:0.2596870905,t4541:0.2010122901):0.3766071773,t1269:0.3674016863):0.1208044465,t4204:0.3698100182):0.8646778397):0.4921671802):0.8154761821):0.5094137727,((t802:0.5341457941,t1937:0.7998197789):0.05112488824,t1492:0.6897424383):0.8094769791):0.8457810935,((t1057:0.4046775179,((t1923:0.6462825201,t4200:0.2072024788):0.3298604705,t488:0.6419402526):0.003014594782):0.7265125066,(((t3155:0.5750570076,t3676:0.9892197067):0.906002905,t4085:0.9781129526):0.996173508,t4069:0.06360293226):0.3592883812):0.5619509818):0.6953451682,(((((t1415:0.09918350447,(t2442:0.155588863,t4412:0.3040755172):0.492990284):0.8465760103,t978:0.3687013045):0.02367241774,((t855:0.4608079048,t2912:0.09758407623):0.3701218856,((t3694:0.07802248141,t3138:0.8096938841):0.4384925135,t3606:0.6476927546):0.9487582464):0.4470769409):0.7942296606,(t1123:0.6678380442,t1625:0.1613316855):0.8397362826):0.3529371219,(t305:0.1103538726,((((t428:0.4417167683,((t4772:0.8882771546,t4091:0.4981870821):0.4756792276,t4229:0.38463756):0.864977079):0.8223084074,(t3363:0.5329570151,t2744:0.4839793602):0.7691739288):0.3956305948,(t15:0.4813148049,((t609:0.3889157842,t1603:0.4138532944):0.1289779663,t2136:0.6060533351):0.0904724244):0.03975326265):0.1583935879,t3523:0.08873104909):0.9958280958):0.7265755415):0.2345846083):0.5260387631,(t1624:0.6208119127,(t3032:0.3652262872,t4186:0.4689125395):0.9585901925):0.6533458913):0.8967711527):0.1973156768):0.5217929904):0.1052631885):0.2312479161):0.1278742545,(((t2806:0.7516691501,(t1505:0.5894373418,t4170:0.8753572488):0.5608480333):0.3014607499,(t3070:0.6524684269,t3124:0.2174219687):0.9468783992):0.8326292518,t490:0.4388881766):0.9946793513); diff --git a/R/unifrac_cpp/api_s.cpp b/R/unifrac_cpp/api_s.cpp index 4768a6913..d73c39ab3 100644 --- a/R/unifrac_cpp/api_s.cpp +++ b/R/unifrac_cpp/api_s.cpp @@ -16,13 +16,13 @@ using namespace su; using namespace std; -std::vector faith_pd_one_off(const Rcpp::S4 & treeSE){ +std::vector faith_pd_one_off(const Rcpp::S4 & treeSE, bool isRooted){ // Check that tree and table are non-empty and match before calling the c++ code // shear the tree (to contain only the obs in the table?) - Also should be done before the call? std::cout << "Start\n"; - su::BPTree tree = su::BPTree(treeSE); + su::BPTree tree = su::BPTree(treeSE, isRooted); std::cout << "Tree ok\n"; su::tse table = su::tse(treeSE); std::cout << "Table ok\n"; diff --git a/R/unifrac_cpp/api_s.hpp b/R/unifrac_cpp/api_s.hpp index 5d0afc48e..efd2951f3 100644 --- a/R/unifrac_cpp/api_s.hpp +++ b/R/unifrac_cpp/api_s.hpp @@ -22,4 +22,4 @@ * tree_missing : the filename for the tree does not exist * table_empty : the table does not have any entries */ -std::vector faith_pd_one_off(const Rcpp::S4 & treeSE); \ No newline at end of file +std::vector faith_pd_one_off(const Rcpp::S4 & treeSE, bool rooted); \ No newline at end of file diff --git a/R/unifrac_cpp/biom_s.cpp b/R/unifrac_cpp/biom_s.cpp index 156383dcd..7a627c903 100644 --- a/R/unifrac_cpp/biom_s.cpp +++ b/R/unifrac_cpp/biom_s.cpp @@ -29,6 +29,11 @@ tse::tse(const Rcpp::S4 & treeSE) { Rcpp::List phylo = rowTree["phylo"]; Rcpp::StringVector tip_label = phylo["tip.label"]; obs_ids = Rcpp::as>(tip_label); + + Rcpp::S4 assays = treeSE.slot("assays"); + Rcpp::S4 data = assays.slot("data"); + Rcpp::List listData = data.slot("listData"); + assay = Rcpp::as(listData["counts"]); n_samples = sample_ids.size(); n_obs = obs_ids.size(); diff --git a/R/unifrac_cpp/su_R_s.cpp b/R/unifrac_cpp/su_R_s.cpp index cccae83a9..afdeea2de 100644 --- a/R/unifrac_cpp/su_R_s.cpp +++ b/R/unifrac_cpp/su_R_s.cpp @@ -6,63 +6,140 @@ #include -using namespace std; -using namespace Rcpp; - - -/* // [[Rcpp::export]] -Rcpp::List faith_pd(const char* table, const char* tree){ - r_vec* result = NULL; - ComputeStatus status; - status = faith_pd_one_off(table, tree, &result); - vector values; - for(int i = 0; i < result->n_samples; i++){ - values.push_back(result->values[i]); - } +Rcpp::NumericVector faith_pd(const Rcpp::S4 & treeSE, bool isRooted){ - return Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, - Rcpp::Named("faith_pd") = values); + std::vector results = faith_pd_one_off(treeSE, isRooted); + + Rcpp::NumericVector faith = Rcpp::NumericVector(results.size()); + + for(unsigned int i = 0; i < results.size(); i++){ + faith[i] = results[i]; + } + return(faith); } -*/ + // [[Rcpp::export]] -void faith_pd_new(const Rcpp::S4 & treeSE){ +Rcpp::LogicalVector rowTree_to_bp(const Rcpp::List & rowTree) { + Rcpp::NumericMatrix edge = rowTree["edge"]; + Rcpp::StringVector tips = rowTree["tip.label"]; + std::vector structure = std::vector(); + + uint32_t ntips = tips.size(); // phylo tips are always numbered from 1 to number of tips; + + std::stack nodes; // Keeps track of the branch's internal nodes - std::vector results = faith_pd_one_off(treeSE); + int currentNode = 0; + int nextNode = 0; - std::cout << results.size() << "\n"; + // Goal: Insert true when a branch starts, a false when it closes, and a true-false for each tip. - if(results.size() >= 20){ - for(unsigned int i = 0; i < 20; i++){ - std::cout << results[i] << "\n"; + for (unsigned int i = 0; i < edge.nrow(); i++){ + currentNode = edge(i, 0); + nextNode = edge(i, 1); + + if(nodes.size() > 0 && currentNode < nodes.top()) { + // We've exhausted the branch and moved backwards in the tree + do { + nodes.pop(); + structure.push_back(false); + } while(currentNode != nodes.top()); + } + + if(nodes.size() == 0 || currentNode > nodes.top() ) { + // We are either at the root, or entering a new node + // What if the tree is unrooted? + nodes.push(currentNode); + structure.push_back(true); + + } + + if(nextNode <= ntips) { + // We've found a tip + structure.push_back(true); + structure.push_back(false); + } + + if(i == edge.nrow() - 1) { + // We've reached the end of the tree + do { + nodes.pop(); + structure.push_back(false); + } while(nodes.size() > 0); } } - //get_sample_counts(treeSE); - /* - r_vec* result = NULL; - ComputeStatus status; - status = faith_pd_one_off(treeSE, &result); - vector values; - for(int i = 0; i < result->n_samples; i++){ - values.push_back(result->values[i]); - } + Rcpp::LogicalVector bp = Rcpp::LogicalVector(structure.size()); - Rcpp::List rlist = Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, - Rcpp::Named("faith_pd") = values); + for(unsigned int i = 0; i < structure.size(); i++){ + bp[i] = structure[i]; + } - destroy_results_vec(&result); + return(bp); +} + + +// [[Rcpp::export]] +Rcpp::LogicalVector newick_to_bp(std::string newick) { + char last_structure; + bool potential_single_descendent = false; + int count = 0; + bool in_quote = false; + std::vector structure; + for(auto c = newick.begin(); c != newick.end(); c++) { + if(*c == '\'') + in_quote = !in_quote; + + if(in_quote) + continue; + + switch(*c) { + case '(': + // opening of a node + count++; + structure.push_back(true); + last_structure = *c; + potential_single_descendent = true; + break; + case ')': + // closing of a node + if(potential_single_descendent || (last_structure == ',')) { + // we have a single descendent or a last child (i.e. ",)" scenario) + count += 3; + structure.push_back(true); + structure.push_back(false); + structure.push_back(false); + potential_single_descendent = false; + } else { + // it is possible still to have a single descendent in the case of + // multiple single descendents (e.g., (...()...) ) + count += 1; + structure.push_back(false); + } + last_structure = *c; + break; + case ',': + if(last_structure != ')') { + // we have a new tip + count += 2; + structure.push_back(true); + structure.push_back(false); + } + potential_single_descendent = false; + last_structure = *c; + break; + default: + break; + } + } - return rlist; - */ + Rcpp::LogicalVector bp = Rcpp::LogicalVector(structure.size()); - //Rcpp::List rowTree = treeSE.slot("rowTree"); - //const List & phylo = rowTree["phylo"]; - //const Rcpp::NumericMatrix & edge = phylo["edge"]; + for(unsigned int i = 0; i < structure.size(); i++){ + bp[i] = structure[i]; + } - //return rowTree; + return(bp); } - - diff --git a/R/unifrac_cpp/tree_s.cpp b/R/unifrac_cpp/tree_s.cpp index ba996d8da..a8e700856 100644 --- a/R/unifrac_cpp/tree_s.cpp +++ b/R/unifrac_cpp/tree_s.cpp @@ -6,7 +6,9 @@ using namespace su; -BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names) { +BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names, bool rooted) { + isRooted = rooted; + structure = input_structure; lengths = input_lengths; names = input_names; @@ -25,7 +27,9 @@ BPTree::BPTree(std::vector input_structure, std::vector input_leng index_and_cache(); } -BPTree::BPTree(const Rcpp::S4 & treeSE) { +BPTree::BPTree(const Rcpp::S4 & treeSE, bool rooted) { + + isRooted = rooted; //Initialize vectors openclose = std::vector(); @@ -96,7 +100,7 @@ BPTree BPTree::mask(std::vector topology_mask, std::vector in_leng } } - return BPTree(new_structure, new_lengths, new_names); + return BPTree(new_structure, new_lengths, new_names, isRooted); } std::unordered_set BPTree::get_tip_names() { @@ -284,6 +288,9 @@ int32_t BPTree::bwd(uint32_t i, int d) const { // The algorithms that this class uses need the tree to be stored in a binary format // In terms of the Newick format, an opening bracket corresponds to a TRUE, a closing bracket to a FALSE, and a tip to a TRUE FALSE // This functions assumes that the tree representation is in cladewise order - Ensure this with ape's reorder.phylo() function +// Need to check whether tree being rooted or not affects construction +// If rooted, root is by definition ntips+1 +// If unrooted, root is chosen arbitrarily? void BPTree::rowTree_to_bp(const Rcpp::List & rowTree) { Rcpp::List phylo = rowTree["phylo"]; Rcpp::NumericMatrix edge = phylo["edge"]; @@ -296,7 +303,6 @@ void BPTree::rowTree_to_bp(const Rcpp::List & rowTree) { int currentNode = 0; int nextNode = 0; - // Goal: Insert true when a branch starts, a false when it closes, and a true-false for each tip. for (unsigned int i = 0; i < edge.nrow(); i++){ @@ -313,8 +319,10 @@ void BPTree::rowTree_to_bp(const Rcpp::List & rowTree) { if(nodes.size() == 0 || currentNode > nodes.top() ) { // We are either at the root, or entering a new node + // What if the tree is unrooted? nodes.push(currentNode); structure.push_back(true); + } if(nextNode <= ntips) { diff --git a/R/unifrac_cpp/tree_s.hpp b/R/unifrac_cpp/tree_s.hpp index f691f1d47..189036244 100644 --- a/R/unifrac_cpp/tree_s.hpp +++ b/R/unifrac_cpp/tree_s.hpp @@ -34,13 +34,13 @@ namespace su { * @param input_lengths A vector of double of the branch lengths * @param input_names A vector of str of the vertex names */ - BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names); + BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names, bool rooted); /* constructor from a TreeSummarizedExperiment * * @param treeSE An R treeSE object */ - BPTree(const Rcpp::S4 & treeSE); + BPTree(const Rcpp::S4 & treeSE, bool rooted); ~BPTree(); @@ -122,6 +122,7 @@ namespace su { std::vector select_0_index; // cache of select 0 std::vector select_1_index; // cache of select 1 std::vector excess; + bool isRooted; // Is the tree rooted or not? void index_and_cache(); // construct the select caches void rowTree_to_bp(const Rcpp::List & rowTree); // convert ape tree structure to boolean structure diff --git a/R/unifrac_cpp/unifrac_internal_s.cpp b/R/unifrac_cpp/unifrac_internal_s.cpp index c87082e1b..8e1f18a0e 100644 --- a/R/unifrac_cpp/unifrac_internal_s.cpp +++ b/R/unifrac_cpp/unifrac_internal_s.cpp @@ -71,6 +71,7 @@ std::vector su::set_proportions(const BPTree &tree, if(tree.isleaf(node)) { std::string leaf = tree.names[node]; props = table.get_obs_data(leaf); // Here we basically just need the row for the specified node + //std::cout << "l " << props[0] << " " << props[1] << " " << props[2] << "\n"; if (normalize) { //#pragma omp parallel for schedule(static) for(unsigned int i = 0; i < table.n_samples; i++) { @@ -97,6 +98,8 @@ std::vector su::set_proportions(const BPTree &tree, current = tree.rightsibling(current); } + //std::cout << "n " << props[0] << " " << props[1] << " " << props[2] << "\n"; + } ps.update(node, props); return(props); diff --git a/R/unifrac_cpp/unifrac_s.cpp b/R/unifrac_cpp/unifrac_s.cpp index 2d0f298c3..effc4ec0c 100644 --- a/R/unifrac_cpp/unifrac_s.cpp +++ b/R/unifrac_cpp/unifrac_s.cpp @@ -37,7 +37,7 @@ std::vector su::faith_pd(tse_interface &table, std::vector node_proportions; double length; - std::vector results = std::vector(); + std::vector results = std::vector(table.n_samples, 0.0); // for node in postorderselect for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { @@ -52,7 +52,8 @@ std::vector su::faith_pd(tse_interface &table, for (unsigned int sample = 0; sample < table.n_samples; sample++){ // calculate contribution of node to score - results.push_back((node_proportions[sample] > 0) * length); + results[sample] += (node_proportions[sample] > 0) * length; + //std::cout << node_proportions[sample] << " " << (node_proportions[sample] > 0) * length << " " << results[sample] << " - "; } } return results; From 9f3bb8832f3a0cbfc4b92e2ec6f3615d2aa80302 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Wed, 15 Jan 2025 15:23:34 +0200 Subject: [PATCH 05/48] Clean up unnecessary files --- R/unifrac_cpp/affinity.hpp | 125 -- R/unifrac_cpp/api.cpp | 1489 +------------------- R/unifrac_cpp/api.hpp | 558 +------- R/unifrac_cpp/api_s.cpp | 36 - R/unifrac_cpp/api_s.hpp | 25 - R/unifrac_cpp/benchtest.sh | 17 - R/unifrac_cpp/biom.cpp | 315 +---- R/unifrac_cpp/biom.hpp | 51 +- R/unifrac_cpp/biom_interface.hpp | 38 +- R/unifrac_cpp/biom_interface_s.hpp | 60 - R/unifrac_cpp/biom_s.cpp | 100 -- R/unifrac_cpp/biom_s.hpp | 91 -- R/unifrac_cpp/capi_test.c | 71 - R/unifrac_cpp/cmd.cpp | 1 - R/unifrac_cpp/cmd.hpp | 32 - R/unifrac_cpp/faithpd.cpp | 84 -- R/unifrac_cpp/skbio_alt.cpp | 617 --------- R/unifrac_cpp/skbio_alt.hpp | 44 - R/unifrac_cpp/su.cpp | 492 ------- R/unifrac_cpp/su_R.cpp | 49 +- R/unifrac_cpp/su_R_s.cpp | 145 -- R/unifrac_cpp/task_parameters.hpp | 35 - R/unifrac_cpp/test_api.cpp | 744 ---------- R/unifrac_cpp/test_ska.cpp | 516 ------- R/unifrac_cpp/test_su.cpp | 1872 -------------------------- R/unifrac_cpp/tree.cpp | 337 +++-- R/unifrac_cpp/tree.hpp | 26 +- R/unifrac_cpp/tree_s.cpp | 438 ------ R/unifrac_cpp/tree_s.hpp | 142 -- R/unifrac_cpp/unifrac.cpp | 471 +------ R/unifrac_cpp/unifrac.hpp | 94 +- R/unifrac_cpp/unifrac_cmp.cpp | 395 ------ R/unifrac_cpp/unifrac_cmp.hpp | 38 - R/unifrac_cpp/unifrac_internal.cpp | 262 +--- R/unifrac_cpp/unifrac_internal.hpp | 68 +- R/unifrac_cpp/unifrac_internal_s.cpp | 113 -- R/unifrac_cpp/unifrac_internal_s.hpp | 43 - R/unifrac_cpp/unifrac_s.cpp | 61 - R/unifrac_cpp/unifrac_s.hpp | 29 - R/unifrac_cpp/unifrac_task.cpp | 785 ----------- R/unifrac_cpp/unifrac_task.hpp | 577 -------- 41 files changed, 350 insertions(+), 11136 deletions(-) delete mode 100644 R/unifrac_cpp/affinity.hpp delete mode 100644 R/unifrac_cpp/api_s.cpp delete mode 100644 R/unifrac_cpp/api_s.hpp delete mode 100644 R/unifrac_cpp/benchtest.sh delete mode 100644 R/unifrac_cpp/biom_interface_s.hpp delete mode 100644 R/unifrac_cpp/biom_s.cpp delete mode 100644 R/unifrac_cpp/biom_s.hpp delete mode 100644 R/unifrac_cpp/capi_test.c delete mode 100644 R/unifrac_cpp/cmd.cpp delete mode 100644 R/unifrac_cpp/cmd.hpp delete mode 100644 R/unifrac_cpp/faithpd.cpp delete mode 100644 R/unifrac_cpp/skbio_alt.cpp delete mode 100644 R/unifrac_cpp/skbio_alt.hpp delete mode 100644 R/unifrac_cpp/su.cpp delete mode 100644 R/unifrac_cpp/su_R_s.cpp delete mode 100644 R/unifrac_cpp/task_parameters.hpp delete mode 100644 R/unifrac_cpp/test_api.cpp delete mode 100644 R/unifrac_cpp/test_ska.cpp delete mode 100644 R/unifrac_cpp/test_su.cpp delete mode 100644 R/unifrac_cpp/tree_s.cpp delete mode 100644 R/unifrac_cpp/tree_s.hpp delete mode 100644 R/unifrac_cpp/unifrac_cmp.cpp delete mode 100644 R/unifrac_cpp/unifrac_cmp.hpp delete mode 100644 R/unifrac_cpp/unifrac_internal_s.cpp delete mode 100644 R/unifrac_cpp/unifrac_internal_s.hpp delete mode 100644 R/unifrac_cpp/unifrac_s.cpp delete mode 100644 R/unifrac_cpp/unifrac_s.hpp delete mode 100644 R/unifrac_cpp/unifrac_task.cpp delete mode 100644 R/unifrac_cpp/unifrac_task.hpp diff --git a/R/unifrac_cpp/affinity.hpp b/R/unifrac_cpp/affinity.hpp deleted file mode 100644 index 55d62c983..000000000 --- a/R/unifrac_cpp/affinity.hpp +++ /dev/null @@ -1,125 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include - -#ifdef __LINUX__ -#include -#endif - - -#ifdef __APPLE__ -#include -#include -#include -#include -#include - -// OSX code adapted from -// http://yyshen.github.io/2015/01/18/binding_threads_to_cores_osx.html -// these macros and methods don't exist on OSX - -#define SYSCTL_CORE_COUNT "machdep.cpu.core_count" - -typedef struct cpu_set { - uint32_t count; -} cpu_set_t; - -static inline void -CPU_ZERO(cpu_set_t *cs) { cs->count = 0; } - -static inline void -CPU_SET(int num, cpu_set_t *cs) { cs->count |= (1 << num); } - -static inline int -CPU_ISSET(int num, cpu_set_t *cs) { return (cs->count & (1 << num)); } - -static inline int -CPU_COUNT(cpu_set_t *cs) { return __builtin_popcount(cs->count); } - -#define CPU_SETSIZE 32 - -static int sched_getaffinity(pid_t pid, size_t cpu_size, cpu_set_t *cpu_set) -{ - int32_t core_count = 0; - size_t len = sizeof(core_count); - int ret = sysctlbyname(SYSCTL_CORE_COUNT, &core_count, &len, 0, 0); - if (ret) { - return -1; - } - cpu_set->count = 0; - for (int i = 0; i < core_count; i++) { - cpu_set->count |= (1 << i); - } - - return 0; -} - -static int pthread_setaffinity_np(pthread_t thread, size_t cpu_size, - cpu_set_t *cpu_set) -{ - thread_port_t mach_thread; - int core = 0; - - for (core = 0; core < 8 * cpu_size; core++) { - if (CPU_ISSET(core, cpu_set)) break; - } - thread_affinity_policy_data_t policy = { core }; - mach_thread = pthread_mach_thread_np(thread); - thread_policy_set(mach_thread, THREAD_AFFINITY_POLICY, - (thread_policy_t)&policy, 1); - return 0; -} - -#endif - -#define handle_error_en(en, msg) \ - do { errno = en; perror(msg); exit(EXIT_FAILURE); } while (0) - -static int bind_to_core(int core) { - /* bind the calling thread to the requested core - * - * The use of this method is for better NUMA utilization. The - * default NUMA policy is local, where memory is allocated on the NUMA node - * relative to the core if possible. The intention with this method is to - * bind to a core first, and then allocate memory. A beneficial side effect - * is that threads should not hop between cores either. - * - * This method is cgroup safe. - */ - // https://stackoverflow.com/a/11583550/19741 - // http://blog.saliya.org/2015/07/get-and-set-process-affinity-in-c.html - pthread_t thread = pthread_self(); - pid_t pid = getpid(); - - cpu_set_t current_set, new_set; - int j, ret; - - CPU_ZERO(¤t_set); - CPU_ZERO(&new_set); - - ret = sched_getaffinity(pid, sizeof(current_set), ¤t_set); - - // find which core in our cpu_set corresponds to the callers - // request - int target = -1; - for(j = 0; j < CPU_SETSIZE; j++) { - if(CPU_ISSET(j, ¤t_set)) { - target++; - } - if(target == core) - break; - } - - if(target != core) { - fprintf(stderr, "Unable to bind this thread to core %d. Are sufficient processors available?", thread); - return -1; - } - - CPU_SET(j, &new_set); - int serr = pthread_setaffinity_np(thread, sizeof(new_set), &new_set); - return serr; -} diff --git a/R/unifrac_cpp/api.cpp b/R/unifrac_cpp/api.cpp index fec69a268..21f550f6f 100644 --- a/R/unifrac_cpp/api.cpp +++ b/R/unifrac_cpp/api.cpp @@ -2,7 +2,7 @@ #include "biom.hpp" #include "tree.hpp" #include "unifrac.hpp" -#include "skbio_alt.hpp" + #include #include #include @@ -12,1483 +12,26 @@ #include #include -/* Platform-specific memory management headers - for windows, we need to add #if defined(_WIN32) and rewrite the mmap portions (possibly with memoryapi.h?)*/ -#ifdef __linux__ -#include -#elif _WIN32 -#include -#endif - -/* Fast compression algorithm */ -#include - -#define MMAP_FD_MASK 0x0fff -#define MMAP_FLAG 0x1000 - -/* O_NOATIME is defined at fcntl.h when supported */ -#ifndef O_NOATIME -#define O_NOATIME 0 -#endif - - -#define CHECK_FILE(filename, err) if(!is_file_exists(filename)) { \ - return err; \ - } - -#define SET_METHOD(requested_method, err) Method method; \ - if(std::strcmp(requested_method, "unweighted") == 0) \ - method = unweighted; \ - else if(std::strcmp(requested_method, "weighted_normalized") == 0) \ - method = weighted_normalized; \ - else if(std::strcmp(requested_method, "weighted_unnormalized") == 0) \ - method = weighted_unnormalized; \ - else if(std::strcmp(requested_method, "generalized") == 0) \ - method = generalized; \ - else if(std::strcmp(requested_method, "unweighted_fp32") == 0) \ - method = unweighted_fp32; \ - else if(std::strcmp(requested_method, "weighted_normalized_fp32") == 0) \ - method = weighted_normalized_fp32; \ - else if(std::strcmp(requested_method, "weighted_unnormalized_fp32") == 0) \ - method = weighted_unnormalized_fp32; \ - else if(std::strcmp(requested_method, "generalized_fp32") == 0) \ - method = generalized_fp32; \ - else { \ - return err; \ - } - -#define PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) std::ifstream ifs(tree_filename); \ - std::string content = std::string(std::istreambuf_iterator(ifs), \ - std::istreambuf_iterator()); \ - su::BPTree tree = su::BPTree(content); \ - su::biom table = su::biom(biom_filename); \ - if(table.n_samples <= 0 | table.n_obs <= 0) { \ - return table_empty; \ - } \ - std::string bad_id = su::test_table_ids_are_subset_of_tree(table, tree); \ - if(bad_id != "") { \ - return table_and_tree_do_not_overlap; \ - } \ - std::unordered_set to_keep(table.obs_ids.begin(), \ - table.obs_ids.end()); \ - su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - +#include using namespace su; using namespace std; -// https://stackoverflow.com/a/19841704/19741 -bool is_file_exists(const char *fileName) { - std::ifstream infile(fileName); - return infile.good(); -} - - -void destroy_stripes(vector &dm_stripes, vector &dm_stripes_total, unsigned int n_samples, - unsigned int stripe_start, unsigned int stripe_stop) { - unsigned int n_rotations = (n_samples + 1) / 2; - - if(stripe_stop == 0) { - for(unsigned int i = 0; i < n_rotations; i++) { - free(dm_stripes[i]); - if(dm_stripes_total[i] != NULL) - free(dm_stripes_total[i]); - } - } else { - // if a stripe_stop is specified, and if we're in the stripe window, do not free - // dm_stripes. this is done as the pointers in dm_stripes are assigned to the partial_mat_t - // and subsequently freed in destroy_partial_mat. but, we do need to free dm_stripes_total - // if appropriate - for(unsigned int i = stripe_start; i < stripe_stop; i++) { - if(dm_stripes_total[i] != NULL) - free(dm_stripes_total[i]); - } - } -} - - -void initialize_mat(mat_t* &result, biom &table, bool is_upper_triangle) { - result = (mat_t*)malloc(sizeof(mat)); - result->n_samples = table.n_samples; - - result->cf_size = su::comb_2(table.n_samples); - result->is_upper_triangle = is_upper_triangle; - result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); - result->condensed_form = (double*)malloc(sizeof(double) * su::comb_2(table.n_samples)); - - for(unsigned int i = 0; i < result->n_samples; i++) { - size_t len = table.sample_ids[i].length(); - result->sample_ids[i] = (char*)malloc(sizeof(char) * len + 1); - table.sample_ids[i].copy(result->sample_ids[i], len); - result->sample_ids[i][len] = '\0'; - } -} - -void initialize_results_vec(r_vec* &result, biom& table){ - // Stores results for Faith PD - result = (r_vec*)malloc(sizeof(results_vec)); - result->n_samples = table.n_samples; - result->values = (double*)malloc(sizeof(double) * result->n_samples); - result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); - - for(unsigned int i = 0; i < result->n_samples; i++) { - size_t len = table.sample_ids[i].length(); - result->sample_ids[i] = (char*)malloc(sizeof(char) * len + 1); - table.sample_ids[i].copy(result->sample_ids[i], len); - result->sample_ids[i][len] = '\0'; - result->values[i] = 0; - } - -} - -void initialize_mat_no_biom(mat_t* &result, char** sample_ids, unsigned int n_samples, bool is_upper_triangle) { - result = (mat_t*)malloc(sizeof(mat)); - result->n_samples = n_samples; - - result->cf_size = su::comb_2(n_samples); - result->is_upper_triangle = is_upper_triangle; - result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); - result->condensed_form = (double*)malloc(sizeof(double) * su::comb_2(n_samples)); - - for(unsigned int i = 0; i < n_samples; i++) { - result->sample_ids[i] = strdup(sample_ids[i]); - } -} - -template -void initialize_mat_full_no_biom_T(TMat* &result, const char* const * sample_ids, unsigned int n_samples, - const char *mmap_dir /* if NULL or "", use malloc */) { - result = (TMat*)malloc(sizeof(mat)); - result->n_samples = n_samples; - - uint64_t n_samples_64 = result->n_samples; // force 64bit to avoit overflow problems - - result->sample_ids = (char**)malloc(sizeof(char*) * n_samples_64); - result->flags=0; - - if (mmap_dir!=NULL) { - if (mmap_dir[0]==0) mmap_dir = NULL; // easier to have a simple test going on - } - - uint64_t msize = sizeof(TReal) * n_samples_64 * n_samples_64; - if (mmap_dir==NULL) { - result->matrix = (TReal*)malloc(msize); - } else { - std::string mmap_template(mmap_dir); - mmap_template+="/su_mmap_XXXXXX"; - - #ifdef __linux__ // Linux-specific memory handling - - // note: mkostemp will update mmap_template in place - int fd=mkostemp((char *) mmap_template.c_str(), O_NOATIME ); // replace for windows - - if (fd<0) { - result->matrix = NULL; - // leave error handling to the caller - } else { - // remove the file name, so it will be destroyed on close - unlink(mmap_template.c_str()); - // make it big enough - ftruncate(fd,msize); - // now can be used, just like a malloc-ed buffer - result->matrix = (TReal*)mmap(NULL, msize,PROT_READ|PROT_WRITE, MAP_SHARED|MAP_NORESERVE, fd, 0); // replace for windows - result->flags=(uint32_t(fd) & MMAP_FD_MASK) | MMAP_FLAG; - } - - #elif _WIN32 // Windows-specific memory handling - - - - #endif - } - - for(unsigned int i = 0; i < n_samples; i++) { - result->sample_ids[i] = strdup(sample_ids[i]); - } -} - -void initialize_partial_mat(partial_mat_t* &result, biom &table, std::vector &dm_stripes, - unsigned int stripe_start, unsigned int stripe_stop, bool is_upper_triangle) { - result = (partial_mat_t*)malloc(sizeof(partial_mat)); - result->n_samples = table.n_samples; - - result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); - for(unsigned int i = 0; i < result->n_samples; i++) { - size_t len = table.sample_ids[i].length(); - result->sample_ids[i] = (char*)malloc(sizeof(char) * len + 1); - table.sample_ids[i].copy(result->sample_ids[i], len); - result->sample_ids[i][len] = '\0'; - } - - result->stripes = (double**)malloc(sizeof(double*) * (stripe_stop - stripe_start)); - result->stripe_start = stripe_start; - result->stripe_stop = stripe_stop; - result->is_upper_triangle = is_upper_triangle; - result->stripe_total = dm_stripes.size(); - - for(unsigned int i = stripe_start; i < stripe_stop; i++) { - result->stripes[i - stripe_start] = dm_stripes[i]; - } -} - -void destroy_results_vec(r_vec** result) { - // for Faith PD - for(unsigned int i = 0; i < (*result)->n_samples; i++) { - free((*result)->sample_ids[i]); - }; - free((*result)->sample_ids); - free((*result)->values); - free(*result); -} - -void destroy_mat(mat_t** result) { - for(unsigned int i = 0; i < (*result)->n_samples; i++) { - free((*result)->sample_ids[i]); - }; - free((*result)->sample_ids); - if (((*result)->condensed_form)!=NULL) { - free((*result)->condensed_form); - } - free(*result); -} - -template -inline void destroy_mat_full_T(TMat** result) { - for(uint32_t i = 0; i < (*result)->n_samples; i++) { - free((*result)->sample_ids[i]); - }; - free((*result)->sample_ids); - if (((*result)->matrix)!=NULL) { - if (((*result)->flags & MMAP_FLAG) == 0) { - free((*result)->matrix); - } else { - uint64_t n_samples = (*result)->n_samples; - munmap((*result)->matrix, sizeof(TReal)*n_samples*n_samples); // replace for windows - - int fd = (*result)->flags & MMAP_FD_MASK; - close(fd); - } - (*result)->matrix=NULL; - } - free(*result); -} - - -void destroy_mat_full_fp64(mat_full_fp64_t** result) { - destroy_mat_full_T(result); -} - -void destroy_mat_full_fp32(mat_full_fp32_t** result) { - destroy_mat_full_T(result); -} - -void destroy_partial_mat(partial_mat_t** result) { - for(unsigned int i = 0; i < (*result)->n_samples; i++) { - if((*result)->sample_ids[i] != NULL) - free((*result)->sample_ids[i]); - }; - if((*result)->sample_ids != NULL) - free((*result)->sample_ids); - - unsigned int n_stripes = (*result)->stripe_stop - (*result)->stripe_start; - for(unsigned int i = 0; i < n_stripes; i++) - if((*result)->stripes[i] != NULL) - free((*result)->stripes[i]); - if((*result)->stripes != NULL) - free((*result)->stripes); - - free(*result); -} - -void destroy_partial_dyn_mat(partial_dyn_mat_t** result) { - for(unsigned int i = 0; i < (*result)->n_samples; i++) { - if((*result)->sample_ids[i] != NULL) - free((*result)->sample_ids[i]); - }; - if((*result)->sample_ids != NULL) - free((*result)->sample_ids); - - unsigned int n_stripes = (*result)->stripe_stop - (*result)->stripe_start; - for(unsigned int i = 0; i < n_stripes; i++) - if((*result)->stripes[i] != NULL) - free((*result)->stripes[i]); - if((*result)->stripes != NULL) - free((*result)->stripes); - if((*result)->offsets != NULL) - free((*result)->offsets); - if((*result)->filename != NULL) - free((*result)->filename); - - free(*result); -} - - -void set_tasks(std::vector &tasks, - double alpha, - unsigned int n_samples, - unsigned int stripe_start, - unsigned int stripe_stop, - bool bypass_tips, - unsigned int nthreads) { - - // compute from start to the max possible stripe if stop doesn't make sense - if(stripe_stop <= stripe_start) - stripe_stop = (n_samples + 1) / 2; - - /* chunking strategy is to balance as much as possible. eg if there are 15 stripes - * and 4 threads, our goal is to assign 4 stripes to 3 threads, and 3 stripes to one thread. - * - * we use the remaining the chunksize for bins which cannot be full maximally - */ - unsigned int fullchunk = ((stripe_stop - stripe_start) + nthreads - 1) / nthreads; // this computes the ceiling - unsigned int smallchunk = (stripe_stop - stripe_start) / nthreads; - - unsigned int n_fullbins = (stripe_stop - stripe_start) % nthreads; - if(n_fullbins == 0) - n_fullbins = nthreads; - - unsigned int start = stripe_start; - - for(unsigned int tid = 0; tid < nthreads; tid++) { - tasks[tid].tid = tid; - tasks[tid].start = start; // stripe start - tasks[tid].bypass_tips = bypass_tips; - - if(tid < n_fullbins) { - tasks[tid].stop = start + fullchunk; // stripe end - start = start + fullchunk; - } else { - tasks[tid].stop = start + smallchunk; // stripe end - start = start + smallchunk; - } - - tasks[tid].n_samples = n_samples; - tasks[tid].g_unifrac_alpha = alpha; - } -} - -compute_status partial(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, bool bypass_tips, - unsigned int nthreads, unsigned int stripe_start, unsigned int stripe_stop, - partial_mat_t** result) { - - CHECK_FILE(biom_filename, table_missing) - CHECK_FILE(tree_filename, tree_missing) - SET_METHOD(unifrac_method, unknown_method) - PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) - - // we resize to the largest number of possible stripes even if only computing - // partial, however we do not allocate arrays for non-computed stripes so - // there is a little memory waste here but should be on the order of - // 8 bytes * N samples per vector. - std::vector dm_stripes((table.n_samples + 1) / 2); - std::vector dm_stripes_total((table.n_samples + 1) / 2); - - if(nthreads > dm_stripes.size()) { - fprintf(stderr, "More threads were requested than stripes. Using %d threads.\n", dm_stripes.size()); - nthreads = dm_stripes.size(); - } - - std::vector tasks(nthreads); - std::vector threads(nthreads); - - if(((table.n_samples + 1) / 2) < stripe_stop) { - fprintf(stderr, "Stopping stripe is out-of-bounds, max %d\n", (table.n_samples + 1) / 2); - exit(EXIT_FAILURE); - } - - set_tasks(tasks, alpha, table.n_samples, stripe_start, stripe_stop, bypass_tips, nthreads); - su::process_stripes(table, tree_sheared, method, variance_adjust, dm_stripes, dm_stripes_total, threads, tasks); - - initialize_partial_mat(*result, table, dm_stripes, stripe_start, stripe_stop, true); // true -> is_upper_triangle - destroy_stripes(dm_stripes, dm_stripes_total, table.n_samples, stripe_start, stripe_stop); - - return okay; -} - -compute_status faith_pd_one_off(const char* biom_filename, const char* tree_filename, - r_vec** result){ - CHECK_FILE(biom_filename, table_missing) - CHECK_FILE(tree_filename, tree_missing) - PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) - - initialize_results_vec(*result, table); - - // compute faithpd - su::faith_pd(table, tree_sheared, std::ref((*result)->values)); - - return okay; -} - -compute_status one_off(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, mat_t** result) { - - CHECK_FILE(biom_filename, table_missing) - CHECK_FILE(tree_filename, tree_missing) - SET_METHOD(unifrac_method, unknown_method) - PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) - - const unsigned int stripe_stop = (table.n_samples + 1) / 2; - std::vector dm_stripes(stripe_stop); - std::vector dm_stripes_total(stripe_stop); - - if(nthreads > dm_stripes.size()) { - fprintf(stderr, "More threads were requested than stripes. Using %d threads.\n", dm_stripes.size()); - nthreads = dm_stripes.size(); - } - - std::vector tasks(nthreads); - std::vector threads(nthreads); - - set_tasks(tasks, alpha, table.n_samples, 0, stripe_stop, bypass_tips, nthreads); - su::process_stripes(table, tree_sheared, method, variance_adjust, dm_stripes, dm_stripes_total, threads, tasks); - - initialize_mat(*result, table, true); // true -> is_upper_triangle - for(unsigned int tid = 0; tid < threads.size(); tid++) { - threads[tid] = std::thread(su::stripes_to_condensed_form, - std::ref(dm_stripes), - table.n_samples, - std::ref((*result)->condensed_form), - tasks[tid].start, - tasks[tid].stop); - } - for(unsigned int tid = 0; tid < threads.size(); tid++) { - threads[tid].join(); - } - - destroy_stripes(dm_stripes, dm_stripes_total, table.n_samples, 0, 0); - - return okay; -} - -// TMat mat_full_fp32_t -template -compute_status one_off_matrix_T(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, - const char *mmap_dir, - TMat** result) { - if (mmap_dir!=NULL) { - if (mmap_dir[0]==0) mmap_dir = NULL; // easier to have a simple test going on - } - - CHECK_FILE(biom_filename, table_missing) - CHECK_FILE(tree_filename, tree_missing) - SET_METHOD(unifrac_method, unknown_method) - PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) - - const unsigned int stripe_stop = (table.n_samples + 1) / 2; - partial_mat_t *partial_mat = NULL; - - { - std::vector dm_stripes(stripe_stop); - std::vector dm_stripes_total(stripe_stop); - - std::vector tasks(nthreads); - std::vector threads(nthreads); - - set_tasks(tasks, alpha, table.n_samples, 0, stripe_stop, bypass_tips, nthreads); - su::process_stripes(table, tree_sheared, method, variance_adjust, dm_stripes, dm_stripes_total, threads, tasks); - - initialize_partial_mat(partial_mat, table, dm_stripes, 0, stripe_stop, true); // true -> is_upper_triangle - if ((partial_mat==NULL) || (partial_mat->stripes==NULL) || (partial_mat->sample_ids==NULL) ) { - fprintf(stderr, "Memory allocation error! (initialize_partial_mat)\n"); - exit(EXIT_FAILURE); - } - destroy_stripes(dm_stripes, dm_stripes_total, table.n_samples, 0, stripe_stop); - } - - initialize_mat_full_no_biom_T(*result, partial_mat->sample_ids, partial_mat->n_samples,mmap_dir); - - if (((*result)==NULL) || ((*result)->matrix==NULL) || ((*result)->sample_ids==NULL) ) { - fprintf(stderr, "Memory allocation error! (initialize_mat)\n"); - exit(EXIT_FAILURE); - } - - - { - MemoryStripes ps(partial_mat->stripes); - const uint32_t tile_size = (mmap_dir==NULL) ? \ - (128/sizeof(TReal)) : /* keep it small for memory access, to fit in chip cache */ \ - (4096/sizeof(TReal)); /* make it larger for mmap, as the limiting factor is swapping */ - su::stripes_to_matrix_T(ps, partial_mat->n_samples, partial_mat->stripe_total, (*result)->matrix, tile_size); - } - destroy_partial_mat(&partial_mat); - - return okay; -} - - -compute_status one_off_matrix(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, - const char *mmap_dir, - mat_full_fp64_t** result) { - return one_off_matrix_T(biom_filename,tree_filename,unifrac_method,variance_adjust,alpha,bypass_tips,nthreads,mmap_dir,result); -} - -compute_status one_off_matrix_fp32(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, - const char *mmap_dir, - mat_full_fp32_t** result) { - return one_off_matrix_T(biom_filename,tree_filename,unifrac_method,variance_adjust,alpha,bypass_tips,nthreads,mmap_dir,result); -} - -inline compute_status is_fp64(const std::string &method_string, const std::string &format_string, bool &fp64) { - if (format_string == "hdf5_fp32") { - fp64 = false; - } else if (format_string == "hdf5_fp64") { - fp64 = true; - } else if (format_string == "hdf5") { - if ((method_string=="unweighted_fp32") || (method_string=="weighted_normalized_fp32") || (method_string=="weighted_unnormalized_fp32") || (method_string=="generalized_fp32")) { - fp64 = false; - } else if ((method_string=="unweighted") || (method_string=="weighted_normalized") || (method_string=="weighted_unnormalized") || (method_string=="generalized")) { - fp64 = true; - } else { - return unknown_method; - } - } else { - return unknown_method; - } - - return okay; -} - - -compute_status unifrac_to_file(const char* biom_filename, const char* tree_filename, const char* out_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int threads, const char* format, - unsigned int pcoa_dims, const char *mmap_dir) -{ - bool fp64; - compute_status rc = is_fp64(unifrac_method, format, fp64); - - if (rc==okay) { - if (fp64) { - mat_full_fp64_t* result; - rc = one_off_matrix(biom_filename, tree_filename, - unifrac_method, variance_adjust, alpha, - bypass_tips, threads, mmap_dir, - &result); - - if (rc==okay) { - // we have no alternative to hdf5 right now - IOStatus iostatus = write_mat_from_matrix_hdf5(out_filename, result, pcoa_dims); - destroy_mat_full_fp64(&result); - - if (iostatus!=write_okay) rc=output_error; - } - } else { - mat_full_fp32_t* result; - rc = one_off_matrix_fp32(biom_filename, tree_filename, - unifrac_method, variance_adjust, alpha, - bypass_tips, threads, mmap_dir, - &result); - - if (rc==okay) { - // we have no alternative to hdf5 right now - IOStatus iostatus = write_mat_from_matrix_hdf5_fp32(out_filename, result, pcoa_dims); - destroy_mat_full_fp32(&result); - - if (iostatus!=write_okay) rc=output_error; - } - } - } - - return rc; -} - -IOStatus write_mat(const char* output_filename, mat_t* result) { - std::ofstream output; - output.open(output_filename); - - uint64_t comb_N = su::comb_2(result->n_samples); - uint64_t comb_N_minus = 0; - double v; - - for(unsigned int i = 0; i < result->n_samples; i++) - output << "\t" << result->sample_ids[i]; - output << std::endl; - - for(unsigned int i = 0; i < result->n_samples; i++) { - output << result->sample_ids[i]; - for(unsigned int j = 0; j < result->n_samples; j++) { - if(i < j) { // upper triangle - comb_N_minus = su::comb_2(result->n_samples - i); - v = result->condensed_form[comb_N - comb_N_minus + (j - i - 1)]; - } else if (i > j) { // lower triangle - comb_N_minus = su::comb_2(result->n_samples - j); - v = result->condensed_form[comb_N - comb_N_minus + (i - j - 1)]; - } else { - v = 0.0; - } - output << std::setprecision(16) << "\t" << v; - } - output << std::endl; - } - output.close(); - - return write_okay; -} - -IOStatus write_mat_from_matrix(const char* output_filename, mat_full_fp64_t* result) { - const double *buf2d = result->matrix; - - std::ofstream output; - output.open(output_filename); - - double v; - const uint64_t n_samples_64 = result->n_samples; // 64-bit to avoid overflow - - for(unsigned int i = 0; i < result->n_samples; i++) - output << "\t" << result->sample_ids[i]; - output << std::endl; - - for(unsigned int i = 0; i < result->n_samples; i++) { - output << result->sample_ids[i]; - for(unsigned int j = 0; j < result->n_samples; j++) { - v = buf2d[i*n_samples_64+j]; - output << std::setprecision(16) << "\t" << v; - } - output << std::endl; - } - output.close(); - - return write_okay; -} - -herr_t write_hdf5_string(hid_t output_file_id,const char *dname, const char *str) -{ - // this is the convoluted way to store a string - // Will use the FORTRAN forma, so we do not depend on null termination - hid_t filetype_id = H5Tcopy (H5T_FORTRAN_S1); - H5Tset_size(filetype_id, strlen(str)); - hid_t memtype_id = H5Tcopy (H5T_C_S1); - H5Tset_size(memtype_id, strlen(str)+1); - - hsize_t dims[1] = {1}; - hid_t dataspace_id = H5Screate_simple (1, dims, NULL); - - hid_t dataset_id = H5Dcreate(output_file_id,dname, filetype_id, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, - H5P_DEFAULT); - herr_t status = H5Dwrite(dataset_id, memtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, str); - - H5Dclose(dataset_id); - H5Sclose(dataspace_id); - H5Tclose(memtype_id); - H5Tclose(filetype_id); - - return status; -} - -// Internal: Make sure TReal and real_id match -template -IOStatus write_mat_from_matrix_hdf5_T(const char* output_filename, TMat * result, hid_t real_id, unsigned int pcoa_dims) { - /* Create a new file using default properties. */ - hid_t output_file_id = H5Fcreate(output_filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - if (output_file_id<0) return write_error; - - // simple header - if (write_hdf5_string(output_file_id,"format","BDSM")<0) { - H5Fclose (output_file_id); - return write_error; - } - if (write_hdf5_string(output_file_id,"version","2020.12")<0) { - H5Fclose (output_file_id); - return write_error; - } - - // save the ids - { - hsize_t dims[1]; - dims[0] = result->n_samples; - hid_t dataspace_id = H5Screate_simple(1, dims, NULL); - - // this is the convoluted way to store an array of strings - hid_t datatype_id = H5Tcopy(H5T_C_S1); - H5Tset_size(datatype_id,H5T_VARIABLE); - - hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); - - hid_t dataset_id = H5Dcreate1(output_file_id, "order", datatype_id, dataspace_id, dcpl_id); - - herr_t status = H5Dwrite(dataset_id, datatype_id, H5S_ALL, H5S_ALL, - H5P_DEFAULT, result->sample_ids); - - H5Dclose(dataset_id); - H5Tclose(datatype_id); - H5Sclose(dataspace_id); - H5Pclose(dcpl_id); - - // check status after cleanup, for simplicity - if (status<0) { - H5Fclose (output_file_id); - return write_error; - } - } - - // save the matrix - { - hsize_t dims[2]; - dims[0] = result->n_samples; - dims[1] = result->n_samples; - hid_t dataspace_id = H5Screate_simple(2, dims, NULL); - - hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); - - hid_t dataset_id = H5Dcreate2(output_file_id, "matrix",real_id, dataspace_id, - H5P_DEFAULT, dcpl_id, H5P_DEFAULT); - herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, - result->matrix); - - - H5Pclose(dcpl_id); - H5Dclose(dataset_id); - H5Sclose(dataspace_id); - - // check status after cleanup, for simplicity - if (status<0) { - H5Fclose (output_file_id); - return write_error; - } - } - - if (pcoa_dims>0) { - // compute pcoa and save it in the file - // use inplace variant to keep memory use in check; we don't need matrix anymore - TReal * eigenvalues; - TReal * samples; - TReal * proportion_explained; - - su::pcoa_inplace(result->matrix, result->n_samples, pcoa_dims, eigenvalues, samples, proportion_explained); - - - if (write_hdf5_string(output_file_id,"pcoa_method","FSVD")<0) { - H5Fclose (output_file_id); - return write_error; - } - - // save the eigenvalues - { - hsize_t dims[1]; - dims[0] = pcoa_dims; - hid_t dataspace_id = H5Screate_simple(1, dims, NULL); - - hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); - - hid_t dataset_id = H5Dcreate2(output_file_id, "pcoa_eigvals",real_id, dataspace_id, - H5P_DEFAULT, dcpl_id, H5P_DEFAULT); - herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, - eigenvalues); - - - H5Pclose(dcpl_id); - H5Dclose(dataset_id); - H5Sclose(dataspace_id); - - // check status after cleanup, for simplicity - if (status<0) { - H5Fclose (output_file_id); - free(samples); - free(proportion_explained); - free(eigenvalues); - return write_error; - } - } - - // save the proportion_explained - { - hsize_t dims[1]; - dims[0] = pcoa_dims; - hid_t dataspace_id = H5Screate_simple(1, dims, NULL); - - hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); - - hid_t dataset_id = H5Dcreate2(output_file_id, "pcoa_proportion_explained",real_id, dataspace_id, - H5P_DEFAULT, dcpl_id, H5P_DEFAULT); - herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, - proportion_explained); - - - H5Pclose(dcpl_id); - H5Dclose(dataset_id); - H5Sclose(dataspace_id); - - // check status after cleanup, for simplicity - if (status<0) { - H5Fclose (output_file_id); - free(samples); - free(proportion_explained); - free(eigenvalues); - return write_error; - } - } - - // save the samples - { - hsize_t dims[2]; - dims[0] = result->n_samples; - dims[1] = pcoa_dims; - hid_t dataspace_id = H5Screate_simple(2, dims, NULL); - - hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); - - hid_t dataset_id = H5Dcreate2(output_file_id, "pcoa_samples",real_id, dataspace_id, - H5P_DEFAULT, dcpl_id, H5P_DEFAULT); - herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, - samples); - - - H5Pclose(dcpl_id); - H5Dclose(dataset_id); - H5Sclose(dataspace_id); - - // check status after cleanup, for simplicity - if (status<0) { - H5Fclose (output_file_id); - free(samples); - free(proportion_explained); - free(eigenvalues); - return write_error; - } - } - - free(samples); - free(proportion_explained); - free(eigenvalues); - - } - - H5Fclose (output_file_id); - return write_okay; -} - -// Internal: Make sure TReal and real_id match -template -IOStatus write_mat_hdf5_T(const char* output_filename, mat_t* result,hid_t real_id, unsigned int pcoa_dims) { - // compute the matrix - TMat mat_full; - mat_full.n_samples = result->n_samples; - - const uint64_t n_samples = result->n_samples; - mat_full.flags = 0; - mat_full.matrix = (TReal*) malloc(n_samples*n_samples*sizeof(TReal)); - if (mat_full.matrix==NULL) { - return open_error; // we don't have a better error code - } - - mat_full.sample_ids = result->sample_ids; // just link - - condensed_form_to_matrix_T(result->condensed_form, n_samples, mat_full.matrix); - IOStatus err = write_mat_from_matrix_hdf5_T(output_filename, &mat_full, real_id, pcoa_dims); - - free(mat_full.matrix); - return err; -} - -IOStatus write_mat_hdf5(const char* output_filename, mat_t* result, unsigned int pcoa_dims) { - return write_mat_hdf5_T(output_filename,result,H5T_IEEE_F64LE,pcoa_dims); -} - -IOStatus write_mat_hdf5_fp32(const char* output_filename, mat_t* result, unsigned int pcoa_dims) { - return write_mat_hdf5_T(output_filename,result,H5T_IEEE_F32LE,pcoa_dims); -} - -IOStatus write_mat_from_matrix_hdf5(const char* output_filename, mat_full_fp64_t* result, unsigned int pcoa_dims) { - return write_mat_from_matrix_hdf5_T(output_filename,result,H5T_IEEE_F64LE,pcoa_dims); -} - -IOStatus write_mat_from_matrix_hdf5_fp32(const char* output_filename, mat_full_fp32_t* result, unsigned int pcoa_dims) { - return write_mat_from_matrix_hdf5_T(output_filename,result,H5T_IEEE_F32LE,pcoa_dims); -} - -IOStatus write_vec(const char* output_filename, r_vec* result) { - std::ofstream output; - output.open(output_filename); - - // write sample ids in first column of file and faith's pd in second column - output << "#SampleID\tfaith_pd" << std::endl; - for(unsigned int i = 0; i < result->n_samples; i++) { - output << result->sample_ids[i]; - output << std::setprecision(16) << "\t" << result->values[i]; - output << std::endl; - } - output.close(); - - return write_okay; -} - -IOStatus write_partial(const char* output_filename, const partial_mat_t* result) { - int fd = open(output_filename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR ); // replace for windows - if (fd==-1) return write_error; - - int cnt = -1; - - uint32_t n_stripes = result->stripe_stop - result->stripe_start; - - uint32_t sample_id_length = 0; - for(unsigned int i = 0; i < result->n_samples; i++) { - sample_id_length += strlen(result->sample_ids[i])+1; - } - - { - char * const samples_buf = (char *)malloc(sample_id_length); - - char *samples_ptr = samples_buf; - - /* sample IDs */ - for(unsigned int i = 0; i < result->n_samples; i++) { - uint32_t length = strlen(result->sample_ids[i])+1; - memcpy(samples_ptr,result->sample_ids[i],length); - samples_ptr+= length; - } - - int max_compressed = LZ4_compressBound(sample_id_length); - char * const cmp_buf = (char *)malloc(max_compressed); - - int sample_id_length_compressed = LZ4_compress_default(samples_buf,cmp_buf,sample_id_length,max_compressed); - if (sample_id_length_compressed<1) {close(fd); return open_error;} - - uint32_t header[8]; - header[0] = PARTIAL_MAGIC_V2; - header[1] = result->n_samples; - header[2] = n_stripes; - header[3] = result->stripe_start; - header[4] = result->stripe_total; - header[5] = result->is_upper_triangle; - header[6] = sample_id_length; - header[7] = sample_id_length_compressed; - - cnt=write(fd,header, 8 * sizeof(uint32_t)); - if (cnt<1) {close(fd); return write_error;} - - cnt=write(fd,cmp_buf, sample_id_length_compressed); - if (cnt<1) {close(fd); return write_error;} - - free(cmp_buf); - free(samples_buf); - } - - { - int max_compressed = LZ4_compressBound(sizeof(double) * result->n_samples); - char * const cmp_buf_raw = (char *)malloc(max_compressed+sizeof(uint32_t)); - char * const cmp_buf = cmp_buf_raw + sizeof(uint32_t); - - /* stripe information */ - for(unsigned int i = 0; i < n_stripes; i++) { - int cmp_size = LZ4_compress_default((const char *) result->stripes[i],cmp_buf,sizeof(double) * result->n_samples,max_compressed); - if (cmp_size<1) {close(fd); return open_error;} - - uint32_t *cmp_buf_size_p = (uint32_t *)cmp_buf_raw; - *cmp_buf_size_p = cmp_size; - - cnt=write(fd, cmp_buf_raw, cmp_size+sizeof(uint32_t)); - if (cnt<1) {return write_error;} - } - - free(cmp_buf_raw); - } - - /* footer */ - { - uint32_t header[1]; - header[0] = PARTIAL_MAGIC_V2; - - cnt=write(fd,header, 1 * sizeof(uint32_t)); - if (cnt<1) {close(fd); return open_error;} - } - - close(fd); - - return write_okay; -} - -IOStatus _is_partial_file(const char* input_filename) { - int fd = open(input_filename, O_RDONLY ); - if (fd==-1) return open_error; - - uint32_t header[1]; - int cnt = read(fd,header,sizeof(uint32_t)); - close(fd); - - if (cnt!=sizeof(uint32_t)) return magic_incompatible; - if ( header[0] != PARTIAL_MAGIC_V2) return magic_incompatible; - - return read_okay; -} - -template -inline IOStatus read_partial_header_fd(int fd, TPMat &result) { - int cnt=-1; - - uint32_t header[8]; - cnt = read(fd,header,8*sizeof(uint32_t)); - if (cnt != (8*sizeof(uint32_t))) {return magic_incompatible;} - - if ( header[0] != PARTIAL_MAGIC_V2) {return magic_incompatible;} - - const uint32_t n_samples = header[1]; - const uint32_t n_stripes = header[2]; - const uint32_t stripe_start = header[3]; - const uint32_t stripe_total = header[4]; - const bool is_upper_triangle = header[5]; - - /* sanity check header */ - if(n_samples <= 0 || n_stripes <= 0 || stripe_total <= 0 || is_upper_triangle < 0) - {return bad_header;} - if(stripe_total >= n_samples || n_stripes > stripe_total || stripe_start >= stripe_total || stripe_start + n_stripes > stripe_total) - {return bad_header;} - - /* initialize the partial result structure */ - result.n_samples = n_samples; - result.sample_ids = (char**)malloc(sizeof(char*) * n_samples); - result.stripes = (double**)malloc(sizeof(double*) * (n_stripes)); - result.stripe_start = stripe_start; - result.stripe_stop = stripe_start + n_stripes; - result.is_upper_triangle = is_upper_triangle; - result.stripe_total = stripe_total; - - /* load samples */ - { - const uint32_t sample_id_length = header[6]; - const uint32_t sample_id_length_compressed = header[7]; - - /* sanity check header */ - if (sample_id_length<=0 || sample_id_length_compressed <=0) - { return bad_header;} - - char * const cmp_buf = (char *)malloc(sample_id_length_compressed); - if (cmp_buf==NULL) { return bad_header;} // no better error code - cnt = read(fd,cmp_buf,sample_id_length_compressed); - if (cnt != sample_id_length_compressed) {free(cmp_buf); return magic_incompatible;} - - char *samples_buf = (char *)malloc(sample_id_length); - if (samples_buf==NULL) { free(cmp_buf); return bad_header;} // no better error code - - cnt = LZ4_decompress_safe(cmp_buf,samples_buf,sample_id_length_compressed,sample_id_length); - if (cnt!=sample_id_length) {free(samples_buf); free(cmp_buf); return magic_incompatible;} - - const char *samples_ptr = samples_buf; - - for(int i = 0; i < n_samples; i++) { - uint32_t sample_length = strlen(samples_ptr); - if ((samples_ptr+sample_length+1)>(samples_buf+sample_id_length)) {free(samples_buf); free(cmp_buf); return magic_incompatible;} - - result.sample_ids[i] = (char*)malloc(sample_length + 1); - memcpy(result.sample_ids[i],samples_ptr,sample_length + 1); - samples_ptr += sample_length + 1; - } - free(samples_buf); - free(cmp_buf); - } - - return read_okay; -} - -template -inline IOStatus read_partial_data_fd(int fd, TPMat &result) { - int cnt=-1; - - const uint32_t n_samples = result.n_samples; - const uint32_t n_stripes = result.stripe_stop-result.stripe_start; - - /* load stripes */ - { - int max_compressed = LZ4_compressBound(sizeof(double) * n_samples); - char * const cmp_buf = (char *)malloc(max_compressed+sizeof(uint32_t)); - if (cmp_buf==NULL) { return bad_header;} // no better error code - - uint32_t *cmp_buf_size_p = (uint32_t *)cmp_buf; - - cnt = read(fd,cmp_buf_size_p , sizeof(uint32_t) ); - if (cnt != sizeof(uint32_t) ) {free(cmp_buf); return magic_incompatible;} - - for(int i = 0; i < n_stripes; i++) { - uint32_t cmp_size = *cmp_buf_size_p; - - uint32_t read_size = cmp_size; - if ( (i+1) -inline IOStatus read_partial_one_stripe_fd(int fd, TPMat &result, uint32_t stripe_idx) { - int cnt=-1; - - const uint32_t n_samples = result.n_samples; - - /* load stripes */ - { - int max_compressed = LZ4_compressBound(sizeof(double) * n_samples); - char * const cmp_buf = (char *)malloc(max_compressed+sizeof(uint32_t)); - if (cmp_buf==NULL) { return bad_header;} // no better error code - - uint32_t *cmp_buf_size_p = (uint32_t *)cmp_buf; - - uint32_t curr_idx = stripe_idx; - while (result.offsets[curr_idx]==0) --curr_idx; // must start reading from the first known offset - - for (;curr_idx(fd, *result); - if (sts==read_okay) - sts = read_partial_data_fd(fd, *result); - - if (sts==read_okay) { - IOStatus sts = read_okay; - /* sanity check the footer */ - uint32_t header[1]; - header[0] = 0; - int cnt = read(fd,header,sizeof(uint32_t)); - if (cnt != (sizeof(uint32_t))) {sts= magic_incompatible;} +std::vector faith_pd_one_off(const Rcpp::S4 & treeSE, bool isRooted){ - if (sts==read_okay) { - if ( header[0] != PARTIAL_MAGIC_V2) {sts= magic_incompatible;} - } - } - - close(fd); - - if (sts==read_okay) { - (*result_out) = result; - } else { - free(result); - (*result_out) = NULL; - } - return sts; -} - -IOStatus read_partial_header(const char* input_filename, partial_dyn_mat_t** result_out) { - int fd = open(input_filename, O_RDONLY ); - if (fd==-1) return open_error; - - /* initialize the partial result structure */ - partial_dyn_mat_t* result = (partial_dyn_mat_t*)malloc(sizeof(partial_dyn_mat)); - { - IOStatus sts = read_partial_header_fd(fd, *result); - if (sts!=read_okay) {free(result); close(fd); return sts;} - } - - // save the offset of the first stripe - const uint32_t n_stripes = result->stripe_stop-result->stripe_start; - result->stripes = (double**) calloc(n_stripes,sizeof(double*)); - result->offsets = (uint64_t*) calloc(n_stripes,sizeof(uint64_t)); - result->offsets[0] = lseek(fd,0,SEEK_CUR); + // Check that tree and table are non-empty and match before calling the c++ code + // shear the tree (to contain only the obs in the table?) - Also should be done before the call? - close(fd); - - result->filename= strdup(input_filename); - - (*result_out) = result; - return read_okay; -} - -IOStatus read_partial_one_stripe(partial_dyn_mat_t* result, uint32_t stripe_idx) { - if (result->stripes[stripe_idx]!=0) return read_okay; // will not re-read - - int fd = open(result->filename, O_RDONLY ); - if (fd==-1) return open_error; - - IOStatus sts = read_partial_one_stripe_fd(fd, *result, stripe_idx); - - close(fd); - return sts; -} - - -template -MergeStatus check_partial(const TPMat* const * partial_mats, int n_partials) { - if(n_partials <= 0) { - fprintf(stderr, "Zero or less partials.\n"); - exit(EXIT_FAILURE); - } - - // sanity check - int n_samples = partial_mats[0]->n_samples; - bool *stripe_map = (bool*)calloc(sizeof(bool), partial_mats[0]->stripe_total); - int stripe_count = 0; - - for(int i = 0; i < n_partials; i++) { - if(partial_mats[i]->n_samples != n_samples) { - free(stripe_map); - return partials_mismatch; - } - - if(partial_mats[0]->stripe_total != partial_mats[i]->stripe_total) { - free(stripe_map); - return partials_mismatch; - } - if(partial_mats[0]->is_upper_triangle != partial_mats[i]->is_upper_triangle) { - free(stripe_map); - return square_mismatch; - } - for(int j = 0; j < n_samples; j++) { - if(strcmp(partial_mats[0]->sample_ids[j], partial_mats[i]->sample_ids[j]) != 0) { - free(stripe_map); - return sample_id_consistency; - } - } - for(int j = partial_mats[i]->stripe_start; j < partial_mats[i]->stripe_stop; j++) { - if(stripe_map[j]) { - free(stripe_map); - return stripes_overlap; - } - stripe_map[j] = true; - stripe_count += 1; - } - } - free(stripe_map); - - if(stripe_count != partial_mats[0]->stripe_total) { - return incomplete_stripe_set; - } - - return merge_okay; -} + std::cout << "Start\n"; + su::BPTree tree = su::BPTree(treeSE, isRooted); + std::cout << "Tree ok\n"; + su::tse table = su::tse(treeSE); + std::cout << "Table ok\n"; -MergeStatus merge_partial(partial_mat_t** partial_mats, int n_partials, unsigned int nthreads, mat_t** result) { - MergeStatus err = check_partial(partial_mats, n_partials); - if (err!=merge_okay) return err; - - int n_samples = partial_mats[0]->n_samples; - std::vector stripes(partial_mats[0]->stripe_total); - std::vector stripes_totals(partial_mats[0]->stripe_total); // not actually used but destroy_stripes needs this to "exist" - for(int i = 0; i < n_partials; i++) { - int n_stripes = partial_mats[i]->stripe_stop - partial_mats[i]->stripe_start; - for(int j = 0; j < n_stripes; j++) { - // as this is potentially a large amount of memory, don't copy, just adopt - *&(stripes[j + partial_mats[i]->stripe_start]) = partial_mats[i]->stripes[j]; - } - } - - initialize_mat_no_biom(*result, partial_mats[0]->sample_ids, n_samples, partial_mats[0]->is_upper_triangle); - if ((*result)==NULL) return incomplete_stripe_set; - if ((*result)->condensed_form==NULL) return incomplete_stripe_set; - if ((*result)->sample_ids==NULL) return incomplete_stripe_set; - - su::stripes_to_condensed_form(stripes, n_samples, (*result)->condensed_form, 0, partial_mats[0]->stripe_total); - - destroy_stripes(stripes, stripes_totals, n_samples, 0, n_partials); - - return merge_okay; -} - -// Will keep only the strictly necessary stripes in memory... reading just in time -class PartialStripes : public su::ManagedStripes { - private: - const uint32_t n_partials; - mutable partial_dyn_mat_t* * partial_mats; // link only, not owned - - static bool in_range(const partial_dyn_mat_t &partial_mat, uint32_t stripe) { - return (stripe>=partial_mat.stripe_start) && (stripestripe_start; - - if (partial_mat->stripes[sidx]==NULL) { - read_partial_one_stripe(partial_mat,sidx); - // ignore any errors, not clear what to do - // will just return NULL - } - - return partial_mat->stripes[sidx]; - } - virtual void release_stripe(uint32_t stripe) const { - uint32_t pidx = find_partial_idx(stripe); - partial_dyn_mat_t * const partial_mat = partial_mats[pidx]; - uint32_t sidx = stripe-partial_mat->stripe_start; - - if (partial_mat->stripes[sidx]!=NULL) { - free(partial_mat->stripes[sidx]); - partial_mat->stripes[sidx]=NULL; - } - } -}; - -template -MergeStatus merge_partial_to_matrix_T(partial_dyn_mat_t* * partial_mats, int n_partials, - const char *mmap_dir, /* if NULL or "", use malloc */ - TMat** result /* out */ ) { - if (mmap_dir!=NULL) { - if (mmap_dir[0]==0) mmap_dir = NULL; // easier to have a simple test going on - } - - MergeStatus err = check_partial(partial_mats, n_partials); - if (err!=merge_okay) return err; - - initialize_mat_full_no_biom_T(*result, partial_mats[0]->sample_ids, partial_mats[0]->n_samples,mmap_dir); - - if ((*result)==NULL) return incomplete_stripe_set; - if ((*result)->matrix==NULL) return incomplete_stripe_set; - if ((*result)->sample_ids==NULL) return incomplete_stripe_set; - - PartialStripes ps(n_partials,partial_mats); - const uint32_t tile_size = (mmap_dir==NULL) ? \ - (128/sizeof(TReal)) : /* keep it small for memory access, to fit in chip cache */ \ - (4096/sizeof(TReal)); /* make it larger for mmap, as the limiting factor is swapping */ - su::stripes_to_matrix_T(ps, partial_mats[0]->n_samples, partial_mats[0]->stripe_total, (*result)->matrix, tile_size); - - return merge_okay; -} - -MergeStatus merge_partial_to_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp64_t** result) { - return merge_partial_to_matrix_T(partial_mats, n_partials, NULL, result); -} - -MergeStatus merge_partial_to_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp32_t** result) { - return merge_partial_to_matrix_T(partial_mats, n_partials, NULL, result); -} - -MergeStatus merge_partial_to_mmap_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp64_t** result) { - return merge_partial_to_matrix_T(partial_mats, n_partials, mmap_dir, result); -} - -MergeStatus merge_partial_to_mmap_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp32_t** result) { - return merge_partial_to_matrix_T(partial_mats, n_partials, mmap_dir, result); -} - - -// skbio_alt pass-thoughs - - -// Find eigen values and vectors -// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. -// Original Paper: https://arxiv.org/abs/1007.5510 -// centered == n x n, must be symmetric, Note: will be used in-place as temp buffer - -void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double **eigenvalues, double **eigenvectors) { - su::find_eigens_fast(n_samples, n_dims, centered, *eigenvalues, *eigenvectors); -} - -void find_eigens_fast_fp32(const uint32_t n_samples, const uint32_t n_dims, float * centered, float **eigenvalues, float **eigenvectors) { - su::find_eigens_fast(n_samples, n_dims, centered, *eigenvalues, *eigenvectors); -} - -/* - Perform Principal Coordinate Analysis. - - Principal Coordinate Analysis (PCoA) is a method similar - to Principal Components Analysis (PCA) with the difference that PCoA - operates on distance matrices, typically with non-euclidian and thus - ecologically meaningful distances like UniFrac in microbiome research. - - In ecology, the euclidean distance preserved by Principal - Component Analysis (PCA) is often not a good choice because it - deals poorly with double zeros (Species have unimodal - distributions along environmental gradients, so if a species is - absent from two sites at the same site, it can't be known if an - environmental variable is too high in one of them and too low in - the other, or too low in both, etc. On the other hand, if an - species is present in two sites, that means that the sites are - similar.). - - Note that the returned eigenvectors are not normalized to unit length. -*/ - -// mat - in, result of unifrac compute -// n_samples - in, size of the matrix (n x n) -// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. -// eigenvalues - out, alocated buffer of size n_dims -// samples - out, alocated buffer of size n_dims x n_samples -// proportion_explained - out, allocated buffer of size n_dims - -void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, double * *eigenvalues, double * *samples, double * *proportion_explained) { - su::pcoa(mat, n_samples, n_dims, *eigenvalues, *samples, *proportion_explained); -} - -void pcoa_fp32(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained) { - su::pcoa(mat, n_samples, n_dims, *eigenvalues, *samples, *proportion_explained); -} - -void pcoa_mixed(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained) { - su::pcoa(mat, n_samples, n_dims, *eigenvalues, *samples, *proportion_explained); -} + std::vector results = su::faith_pd(table, tree); + std::cout << "Results ok\n"; + // compute faithpd + return results; + //return std::vector(); +} \ No newline at end of file diff --git a/R/unifrac_cpp/api.hpp b/R/unifrac_cpp/api.hpp index 7b856b6bb..528d52109 100644 --- a/R/unifrac_cpp/api.hpp +++ b/R/unifrac_cpp/api.hpp @@ -1,212 +1,13 @@ -#include "task_parameters.hpp" - #ifdef __cplusplus #include +#include #define EXTERN extern "C" - #else #include #define EXTERN #endif -#define PARTIAL_MAGIC "SSU-PARTIAL-01" -#define PARTIAL_MAGIC_V2 0x088ABA02 - - -typedef enum compute_status {okay=0, tree_missing, table_missing, table_empty, unknown_method, table_and_tree_do_not_overlap, output_error} ComputeStatus; -typedef enum io_status {read_okay=0, write_okay, open_error, read_error, magic_incompatible, bad_header, unexpected_end, write_error} IOStatus; -typedef enum merge_status {merge_okay=0, incomplete_stripe_set, sample_id_consistency, square_mismatch, partials_mismatch, stripes_overlap} MergeStatus; - -/* a result matrix - * - * n_samples the number of samples. - * cf_size the size of the condensed form. - * is_upper_triangle if true, indicates condensed_form represents a square - * matrix, and only the upper triangle is contained. if false, - * condensed_form represents the lower triangle of a matrix. - * condensed_form the matrix values of length cf_size. - * sample_ids the sample IDs of length n_samples. - */ -typedef struct mat { - unsigned int n_samples; - unsigned int cf_size; - bool is_upper_triangle; - double* condensed_form; - char** sample_ids; -} mat_t; - -/* a result matrix, full, fp64 - * - * n_samples the number of samples. - * matrix the matrix values, n_sample**2 size - * sample_ids the sample IDs of length n_samples. - */ -typedef struct mat_full_fp64 { - uint32_t n_samples; - uint32_t flags; //opaque, 0 for default behavior - double* matrix; - char** sample_ids; -} mat_full_fp64_t; - -/* a result matrix, full, fp32 - * - * n_samples the number of samples. - * matrix the matrix values, n_sample**2 size - * sample_ids the sample IDs of length n_samples. - */ -typedef struct mat_full_fp32 { - uint32_t n_samples; - uint32_t flags; //opaque, 0 for default behavior - float* matrix; - char** sample_ids; -} mat_full_fp32_t; - - - -/* a result vector - * - * n_samples the number of samples. - * values the score values of length n_samples. - * sample_ids the sample IDs of length n_samples. - */ -typedef struct results_vec{ - unsigned int n_samples; - double* values; - char** sample_ids; -} r_vec; - -/* a partial result containing stripe data - * - * n_samples the number of samples. - * sample_ids the sample IDs of length n_samples. - * stripes the stripe data of dimension (stripe_stop - stripe_start, n_samples) - * stripe_start the logical starting stripe in the final matrix. - * stripe_stop the logical stopping stripe in the final matrix. - * stripe_total the total number of stripes present in the final matrix. - * is_upper_triangle whether the stripes correspond to the upper triangle of the resulting matrix. - * This is useful for asymmetric unifrac metrics. - */ -typedef struct partial_mat { - uint32_t n_samples; - char** sample_ids; - double** stripes; - uint32_t stripe_start; - uint32_t stripe_stop; - uint32_t stripe_total; - bool is_upper_triangle; -} partial_mat_t; - -/* a partial resuly, can be populated dynamically - * - * n_samples the number of samples. - * sample_ids the sample IDs of length n_samples. - * offsets offsets to the stripes in the file; 0 means unknown - * stripes the stripe data of dimension (stripe_stop - stripe_start, n_samples) - * stripe_start the logical starting stripe in the final matrix. - * stripe_stop the logical stopping stripe in the final matrix. - * stripe_total the total number of stripes present in the final matrix. - * is_upper_triangle whether the stripes correspond to the upper triangle of the resulting matrix. - * This is useful for asymmetric unifrac metrics. - * filename Name of the file from which to read - */ -typedef struct partial_dyn_mat { - uint32_t n_samples; - char** sample_ids; - uint64_t* offsets; - double** stripes; - uint32_t stripe_start; - uint32_t stripe_stop; - uint32_t stripe_total; - bool is_upper_triangle; - char* filename; -} partial_dyn_mat_t; - - - -void destroy_mat(mat_t** result); -void destroy_mat_full_fp64(mat_full_fp64_t** result); -void destroy_mat_full_fp32(mat_full_fp32_t** result); -void destroy_partial_mat(partial_mat_t** result); -void destroy_partial_dyn_mat(partial_dyn_mat_t** result); -void destroy_results_vec(r_vec** result); - -/* Compute UniFrac - condensed form - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads to use. - * result the resulting distance matrix in condensed form, this is initialized within the method so using ** - * - * one_off returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - * table_empty : the table does not have any entries - */ -EXTERN ComputeStatus one_off(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int threads, mat_t** result); - -/* Compute UniFrac - matrix form - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads/blocks to use. - * mmap_dir If not NULL, area to use for temp memory storage - * result the resulting distance matrix in matrix form, this is initialized within the method so using ** - * - * one_off_matrix returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - * table_empty : the table does not have any entries - */ -EXTERN ComputeStatus one_off_matrix(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, - const char *mmap_dir, - mat_full_fp64_t** result); - -/* Compute UniFrac - matrix form, fp32 variant - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads/blocks to use. - * mmap_dir If not NULL, area to use for temp memory storage - * result the resulting distance matrix in matrix form, this is initialized within the method so using ** - * - * one_off_matrix_fp32 returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - * table_empty : the table does not have any entries - */ -EXTERN ComputeStatus one_off_matrix_fp32(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, - const char *mmap_dir, - mat_full_fp32_t** result); - - /* compute Faith PD * biom_filename the filename to the biom table. * tree_filename the filename to the correspodning tree. @@ -219,359 +20,4 @@ EXTERN ComputeStatus one_off_matrix_fp32(const char* biom_filename, const char* * tree_missing : the filename for the tree does not exist * table_empty : the table does not have any entries */ -EXTERN ComputeStatus faith_pd_one_off(const char* biom_filename, const char* tree_filename, - r_vec** result); - -/* Compute UniFrac and save to file - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * out_filename the filename of the output file. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads to use. - * format output format to use. - * pcoa_dims if not 0, number of dimensions to use or PCoA - * mmap_dir if not empty, temp dir to use for disk-based memory - * - * unifrac_to_file returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - * table_empty : the table does not have any entries - * output_error : failed to properly write the output file - */ -EXTERN ComputeStatus unifrac_to_file(const char* biom_filename, const char* tree_filename, const char* out_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int threads, const char* format, - unsigned int pcoa_dims, const char *mmap_dir); - -/* Write a matrix object - * - * filename the file to write into - * result the results object - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat(const char* filename, mat_t* result); - -/* Write a matrix object using hdf5 format - * - * filename the file to write into - * result the results object - * pcoa_dims PCoAdimensions to compute, if >0 - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat_hdf5(const char* filename, mat_t* result, unsigned int pcoa_dims); - -/* Write a matrix object using hdf5 format, using fp32 precision - * - * filename the file to write into - * result the results object - * pcoa_dims PCoAdimensions to compute, if >0 - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat_hdf5_fp32(const char* filename, mat_t* result, unsigned int pcoa_dims); - -/* Write a matrix object - * - * filename the file to write into - * result the results object - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat_from_matrix(const char* filename, mat_full_fp64_t* result); - - -/* Write a matrix object from buffer using hdf5 format - * - * filename the file to write into - * result the results object - * pcoa_dims PCoAdimensions to compute, if >0 - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat_from_matrix_hdf5(const char* filename, mat_full_fp64_t* result, unsigned int pcoa_dims); - -/* Write a matrix object from buffer using hdf5 format, using fp32 precision - * - * filename the file to write into - * result the results object - * pcoa_dims PCoAdimensions to compute, if >0 - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat_from_matrix_hdf5_fp32(const char* filename, mat_full_fp32_t* result, unsigned int pcoa_dims); - -/* Write a series - * - * filename the file to write into - * result the results object - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_vec(const char* filename, r_vec* result); - -/* Read a matrix object - * - * filename the file to write into - * result the results object - * - * The following error codes are returned: - * - * read_okay : no problems - * open_error : could not open the file - * magic_incompatible : format magic not found or incompatible - * unexpected_end : format end not found in expected location - */ -//EXTERN IOStatus read_mat(const char* filename, mat_t** result); - -/* Compute a subset of a UniFrac distance matrix - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads to use. - * stripe_start the starting stripe to compute - * stripe_stop the last stripe to compute - * dm_stripes the unique branch length stripes. This is expected to be - * uninitialized, and is an output parameter. - * dm_stripes_total the total branch length stripes. This is expected to be - * uninitialized, and is an output parameter. - * result the resulting distance matrix in condensed form, this is initialized within the method so using ** - * - * partial returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - */ - -EXTERN ComputeStatus partial(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int threads, unsigned int stripe_start, - unsigned int stripe_stop, partial_mat_t** result); - -/* Write a partial matrix object - * - * filename the file to write into - * result the partial results object - * - * The following error codes are returned: - * - * write_okay : no problems - * open_error : could not open the file - * - * The structure of the binary output file is as follows. Newlines added for clarity, but are not stored. - * The file has logical blocks, but are not explicitly denoted in the format. These logical blocks are - * just used to improve readability here, and are denoted by ### marks. - * - * ### HEADER ### - * : uint16_t, the length of the magic - * : char, e.g., SSU-PARTIAL-01 - * : uint32_t, the number of samples - * : uint32_t, the number of stripes represented in this file - * : uint32_t, the starting stripe number - * : uint32_t, the total number of stripes in the full matrix - * : uint8_t, zero is false, nonzero is true - * - * ### SAMPLE IDS ### - * : uint16_t, the length of the next sample ID - * : LEN bytes, char - * ... : ... repeated - * : uint16_t, the length of the next sample ID - * : LEN bytes, char - * - * ### STRIPE VALUES; SS -> STRIPE_START, NS -> N_STRIPES - * : double, the first value in the 0th stripe - * ... : ... repeated for N_SAMPLES values - * : double, the last value in the 0th stripe - * : double, the first value in the Kth stripe - * ... : ... repeated for N_SAMPLES values - * : double, the last value in the Kth stripe - * - * ### FOOTER ### - * : char, e.g., SSU-PARTIAL-01, same as starting magic - */ -EXTERN IOStatus write_partial(const char* filename, const partial_mat_t* result); - -/* Read a partial matrix object - * - * filename the file to write into - * result the partial results object, output parameter - * - * The following error codes are returned: - * - * read_okay : no problems - * open_error : could not open the file - * magic_incompatible : format magic not found or incompatible - * bad_header : header seems malformed - * unexpected_end : format end not found in expected location - */ -EXTERN IOStatus read_partial(const char* filename, partial_mat_t** result); - -/* Read a partial matrix object header - * - * filename the file to write into - * result the partial results object, output parameter - * - * The following error codes are returned: - * - * read_okay : no problems - * open_error : could not open the file - * magic_incompatible : format magic not found or incompatible - * bad_header : header seems malformed - * unexpected_end : format end not found in expected location - */ -EXTERN IOStatus read_partial_header(const char* input_filename, partial_dyn_mat_t** result_out); - -/* Read a stripe of a partial matrix - * - * filename the file to write into - * result the partial results object - * stripe_idx relative stripe number - * - * The following error codes are returned: - * - * read_okay : no problems - * open_error : could not open the file - * magic_incompatible : format magic not found or incompatible - * bad_header : header seems malformed - * unexpected_end : format end not found in expected location - */ -EXTERN IOStatus read_partial_one_stripe(partial_dyn_mat_t* result, uint32_t stripe_idx); - - -/* Merge partial results - * - * results an array of partial_mat_t*, the buffers will be destroyed in the process - * n_partials number of partial mats - * merged the full matrix, output parameters, this is initialized in the method so using ** - * - * The following error codes are returned: - * - * merge_okay : no problems - * incomplete_stripe_set : not all stripes needed to create a full matrix were foun - * sample_id_consistency : samples described by stripes are inconsistent - * square_mismatch : inconsistency on denotation of square matrix - */ -EXTERN MergeStatus merge_partial(partial_mat_t** partial_mats, int n_partials, unsigned int nthreads, mat_t** result); - -/* Merge partial results - * - * partial_mats an array of partial_dyn_mat_t* - * n_partials number of partial mats - * result the full matrix, output parameters, this is initialized in the method so using ** - * - * The following error codes are returned: - * - * merge_okay : no problems - * incomplete_stripe_set : not all stripes needed to create a full matrix were foun - * sample_id_consistency : samples described by stripes are inconsistent - * square_mismatch : inconsistency on denotation of square matrix - */ -MergeStatus merge_partial_to_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp64_t** result); - -/* Merge partial results - * - * partial_mats an array of partial_dyn_mat_t* - * n_partials number of partial mats - * result the full matrix, output parameters, this is initialized in the method so using ** - * - * The following error codes are returned: - * - * merge_okay : no problems - * incomplete_stripe_set : not all stripes needed to create a full matrix were foun - * sample_id_consistency : samples described by stripes are inconsistent - * square_mismatch : inconsistency on denotation of square matrix - */ -MergeStatus merge_partial_to_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp32_t** result); - - -/* Merge partial results - * - * partial_mats an array of partial_dyn_mat_t* - * n_partials number of partial mats - * mmap_dir Where to host the mmap file - * result the full matrix, output parameters, this is initialized in the method so using ** - * - * The following error codes are returned: - * - * merge_okay : no problems - * incomplete_stripe_set : not all stripes needed to create a full matrix were foun - * sample_id_consistency : samples described by stripes are inconsistent - * square_mismatch : inconsistency on denotation of square matrix - */ -MergeStatus merge_partial_to_mmap_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp64_t** result); - -/* Merge partial results - * - * partial_mats an array of partial_dyn_mat_t* - * n_partials number of partial mats - * mmap_dir Where to host the mmap file - * result the full matrix, output parameters, this is initialized in the method so using ** - * - * The following error codes are returned: - * - * merge_okay : no problems - * incomplete_stripe_set : not all stripes needed to create a full matrix were foun - * sample_id_consistency : samples described by stripes are inconsistent - * square_mismatch : inconsistency on denotation of square matrix - */ -MergeStatus merge_partial_to_mmap_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp32_t** result); - - -// Find eigen values and vectors -// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. -// Original Paper: https://arxiv.org/abs/1007.5510 -// centered == n x n, must be symmetric, Note: will be used in-place as temp buffer -void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double **eigenvalues, double **eigenvectors); -void find_eigens_fast_p32(const uint32_t n_samples, const uint32_t n_dims,float * centered, float **eigenvalues, float **eigenvectors); - -// Perform Principal Coordinate Analysis -// mat - in, result of unifrac compute -// n_samples - in, size of the matrix (n x n) -// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. -// eigenvalues - out, alocated buffer of size n_dims -// samples - out, alocated buffer of size n_dims x n_samples -// proportion_explained - out, allocated buffer of size n_dims -void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, double **eigenvalues, double **samples, double **proportion_explained); -void pcoa_fp32(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained); -void pcoa_mixed(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained); - - -#ifdef __cplusplus -// TODO: only needed for testing, should be encased in a macro -void set_tasks(std::vector &tasks, - double alpha, - unsigned int n_samples, - unsigned int stripe_start, - unsigned int stripe_stop, - bool bypass_tips, - unsigned int nthreads); - -#endif +std::vector faith_pd_one_off(const Rcpp::S4 & treeSE, bool rooted); \ No newline at end of file diff --git a/R/unifrac_cpp/api_s.cpp b/R/unifrac_cpp/api_s.cpp deleted file mode 100644 index d73c39ab3..000000000 --- a/R/unifrac_cpp/api_s.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "api_s.hpp" -#include "biom_s.hpp" -#include "tree_s.hpp" -#include "unifrac_s.hpp" -#include -#include -#include -#include -#include - -#include -#include - -#include - -using namespace su; -using namespace std; - -std::vector faith_pd_one_off(const Rcpp::S4 & treeSE, bool isRooted){ - - // Check that tree and table are non-empty and match before calling the c++ code - // shear the tree (to contain only the obs in the table?) - Also should be done before the call? - - std::cout << "Start\n"; - su::BPTree tree = su::BPTree(treeSE, isRooted); - std::cout << "Tree ok\n"; - su::tse table = su::tse(treeSE); - std::cout << "Table ok\n"; - - std::vector results = su::faith_pd(table, tree); - std::cout << "Results ok\n"; - - // compute faithpd - return results; - //return std::vector(); -} \ No newline at end of file diff --git a/R/unifrac_cpp/api_s.hpp b/R/unifrac_cpp/api_s.hpp deleted file mode 100644 index efd2951f3..000000000 --- a/R/unifrac_cpp/api_s.hpp +++ /dev/null @@ -1,25 +0,0 @@ -#include "task_parameters.hpp" - -#ifdef __cplusplus -#include -#include -#define EXTERN extern "C" - -#else -#include -#define EXTERN -#endif - -/* compute Faith PD - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * result the resulting vector of computed Faith PD values - * - * faith_pd_one_off returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * table_empty : the table does not have any entries - */ -std::vector faith_pd_one_off(const Rcpp::S4 & treeSE, bool rooted); \ No newline at end of file diff --git a/R/unifrac_cpp/benchtest.sh b/R/unifrac_cpp/benchtest.sh deleted file mode 100644 index ab9280b1d..000000000 --- a/R/unifrac_cpp/benchtest.sh +++ /dev/null @@ -1,17 +0,0 @@ -set -e -set -x - -basedir=bench_tables_trees -resdir=$basedir/results -mkdir -p $resdir -for f in $basedir/*.biom -do - bench=${basedir}/$(basename $f .biom) - res=${resdir}/$(basename $f .biom) - for method in {unweighted,weighted_normalized,weighted_unnormalized} - do - /usr/bin/time -l ./su ${bench}.tre ${bench}.biom $method > ${res}.${method}.su.dm 2> ${res}.${method}.su.stats - /usr/bin/time -l ./sk ${bench}.tre ${bench}.biom $method > ${res}.${method}.sk.dm 2> ${res}.${method}.sk.stats - python compare_dms.py ${res}.${method}.sk.dm ${res}.${method}.su.dm - done -done diff --git a/R/unifrac_cpp/biom.cpp b/R/unifrac_cpp/biom.cpp index a62a9e27f..38e8fd978 100644 --- a/R/unifrac_cpp/biom.cpp +++ b/R/unifrac_cpp/biom.cpp @@ -10,46 +10,34 @@ #include #include #include -#include "biom.hpp" - -using namespace H5; -using namespace su; +#include -/* datasets defined by the BIOM 2.x spec */ -const std::string OBS_INDPTR = std::string("/observation/matrix/indptr"); -const std::string OBS_INDICES = std::string("/observation/matrix/indices"); -const std::string OBS_DATA = std::string("/observation/matrix/data"); -const std::string OBS_IDS = std::string("/observation/ids"); +#include "biom.hpp" -const std::string SAMPLE_INDPTR = std::string("/sample/matrix/indptr"); -const std::string SAMPLE_INDICES = std::string("/sample/matrix/indices"); -const std::string SAMPLE_DATA = std::string("/sample/matrix/data"); -const std::string SAMPLE_IDS = std::string("/sample/ids"); +#include -biom::biom(std::string filename) { - file = H5File(filename.c_str(), H5F_ACC_RDONLY); +using namespace su; - /* establish the datasets */ - obs_indices = file.openDataSet(OBS_INDICES.c_str()); - obs_data = file.openDataSet(OBS_DATA.c_str()); - sample_indices = file.openDataSet(SAMPLE_INDICES.c_str()); - sample_data = file.openDataSet(SAMPLE_DATA.c_str()); - - /* cache IDs and indptr */ - sample_ids = std::vector(); +tse::tse(const Rcpp::S4 & treeSE) { + sample_ids = std::vector(); obs_ids = std::vector(); - sample_indptr = std::vector(); - obs_indptr = std::vector(); - - load_ids(OBS_IDS.c_str(), obs_ids); - load_ids(SAMPLE_IDS.c_str(), sample_ids); - load_indptr(OBS_INDPTR.c_str(), obs_indptr); - load_indptr(SAMPLE_INDPTR.c_str(), sample_indptr); + + Rcpp::S4 colData = treeSE.slot("colData"); + Rcpp::StringVector rownames = colData.slot("rownames"); + sample_ids = Rcpp::as>(rownames); + + Rcpp::List rowTree = treeSE.slot("rowTree"); + Rcpp::List phylo = rowTree["phylo"]; + Rcpp::StringVector tip_label = phylo["tip.label"]; + obs_ids = Rcpp::as>(tip_label); + + Rcpp::S4 assays = treeSE.slot("assays"); + Rcpp::S4 data = assays.slot("data"); + Rcpp::List listData = data.slot("listData"); + assay = Rcpp::as(listData["counts"]); - /* cache shape and nnz info */ n_samples = sample_ids.size(); n_obs = obs_ids.size(); - set_nnz(); /* define a mapping between an ID and its corresponding offset */ obs_id_index = std::unordered_map(); @@ -57,109 +45,16 @@ biom::biom(std::string filename) { create_id_index(obs_ids, obs_id_index); create_id_index(sample_ids, sample_id_index); - - /* load obs sparse data */ - obs_indices_resident = (uint32_t**)malloc(sizeof(uint32_t**) * n_obs); - if(obs_indices_resident == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(uint32_t**) * n_obs, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - obs_data_resident = (double**)malloc(sizeof(double**) * n_obs); - if(obs_data_resident == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double**) * n_obs, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - obs_counts_resident = (unsigned int*)malloc(sizeof(unsigned int) * n_obs); - if(obs_counts_resident == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(unsigned int) * n_obs, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - - uint32_t *current_indices = NULL; - double *current_data = NULL; - for(unsigned int i = 0; i < obs_ids.size(); i++) { - std::string id_ = obs_ids[i]; - unsigned int n = get_obs_data_direct(id_, current_indices, current_data); - obs_counts_resident[i] = n; - obs_indices_resident[i] = current_indices; - obs_data_resident[i] = current_data; - } + sample_counts = get_sample_counts(); -} - -biom::~biom() { - for(unsigned int i = 0; i < n_obs; i++) { - free(obs_indices_resident[i]); - free(obs_data_resident[i]); - } - free(obs_indices_resident); - free(obs_data_resident); - free(obs_counts_resident); -} - -void biom::set_nnz() { - // should these be cached? - DataType dtype = obs_data.getDataType(); - DataSpace dataspace = obs_data.getSpace(); - - hsize_t dims[1]; - dataspace.getSimpleExtentDims(dims, NULL); - nnz = dims[0]; -} - -void biom::load_ids(const char *path, std::vector &ids) { - DataSet ds_ids = file.openDataSet(path); - DataType dtype = ds_ids.getDataType(); - DataSpace dataspace = ds_ids.getSpace(); - - hsize_t dims[1]; - dataspace.getSimpleExtentDims(dims, NULL); - - /* the IDs are a dataset of variable length strings */ - char **dataout = (char**)malloc(sizeof(char*) * dims[0]); - if(dataout == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(char*) * dims[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - ds_ids.read((void*)dataout, dtype); - - ids.reserve(dims[0]); - for(unsigned int i = 0; i < dims[0]; i++) { - ids.push_back(dataout[i]); - } - for(unsigned int i = 0; i < dims[0]; i++) - free(dataout[i]); - free(dataout); } -void biom::load_indptr(const char *path, std::vector &indptr) { - DataSet ds = file.openDataSet(path); - DataType dtype = ds.getDataType(); - DataSpace dataspace = ds.getSpace(); - - hsize_t dims[1]; - dataspace.getSimpleExtentDims(dims, NULL); +tse::~tse() { - uint32_t *dataout = (uint32_t*)malloc(sizeof(uint32_t) * dims[0]); - if(dataout == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(uint32_t) * dims[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - ds.read((void*)dataout, dtype); - - indptr.reserve(dims[0]); - for(unsigned int i = 0; i < dims[0]; i++) - indptr.push_back(dataout[i]); - free(dataout); } -void biom::create_id_index(std::vector &ids, +void tse::create_id_index(std::vector &ids, std::unordered_map &map) { uint32_t count = 0; map.reserve(ids.size()); @@ -168,157 +63,39 @@ void biom::create_id_index(std::vector &ids, } } -unsigned int biom::get_obs_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out) { - uint32_t idx = obs_id_index.at(id); - uint32_t start = obs_indptr[idx]; - uint32_t end = obs_indptr[idx + 1]; - - hsize_t count[1] = {end - start}; - hsize_t offset[1] = {start}; - - DataType indices_dtype = obs_indices.getDataType(); - DataType data_dtype = obs_data.getDataType(); - - DataSpace indices_dataspace = obs_indices.getSpace(); - DataSpace data_dataspace = obs_data.getSpace(); - - DataSpace indices_memspace(1, count, NULL); - DataSpace data_memspace(1, count, NULL); - - indices_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); - data_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); - - current_indices_out = (uint32_t*)malloc(sizeof(uint32_t) * count[0]); - if(current_indices_out == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(uint32_t) * count[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - current_data_out = (double*)malloc(sizeof(double) * count[0]); - if(current_data_out == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double) * count[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - - obs_indices.read((void*)current_indices_out, indices_dtype, indices_memspace, indices_dataspace); - obs_data.read((void*)current_data_out, data_dtype, data_memspace, data_dataspace); - - return count[0]; -} - -template -void biom::get_obs_data_TT(const std::string &id, TFloat* out) const { - uint32_t idx = obs_id_index.at(id); - unsigned int count = obs_counts_resident[idx]; - const uint32_t * const indices = obs_indices_resident[idx]; - const double * const data = obs_data_resident[idx]; - - // reset our output buffer - for(unsigned int i = 0; i < n_samples; i++) - out[i] = 0.0; - - for(unsigned int i = 0; i < count; i++) { - out[indices[i]] = data[i]; - } -} - -void biom::get_obs_data(const std::string &id, double* out) const { - biom::get_obs_data_TT(id,out); -} - -void biom::get_obs_data(const std::string &id, float* out) const { - biom::get_obs_data_TT(id,out); -} - - -// note: out is supposed to be fully filled, i.e. out[start:end] +//Basically just gets the row for the specified id template -void biom::get_obs_data_range_TT(const std::string &id, unsigned int start, unsigned int end, bool normalize, TFloat* out) const { +std::vector tse::get_obs_data_TT(const std::string &id, TFloat t) const { + std::vector out = std::vector(); uint32_t idx = obs_id_index.at(id); - unsigned int count = obs_counts_resident[idx]; - const uint32_t * const indices = obs_indices_resident[idx]; - const double * const data = obs_data_resident[idx]; - - // reset our output buffer - for(unsigned int i = start; i < end; i++) - out[i-start] = 0.0; - - if (normalize) { - for(unsigned int i = 0; i < count; i++) { - const int32_t j = indices[i]; - if ((j>=start)&&(j=start)&&(j tse::get_obs_data(const std::string &id) const { + double t = 0.0; + return(tse::get_obs_data_TT(id, t)); } -unsigned int biom::get_sample_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out) { - uint32_t idx = sample_id_index.at(id); - uint32_t start = sample_indptr[idx]; - uint32_t end = sample_indptr[idx + 1]; - hsize_t count[1] = {end - start}; - hsize_t offset[1] = {start}; +//Returns a pointer-based array - can perhaps be changed to simply referring to the R object's internal storage? +//What exactly does this array contain? It contains n_samples elements which are doubles. +//I'm fairly sure that it just sums the counts over samples. Basically just get a column sum. +//std::vector uses move semantics so it shouldn't affect memory usage too much +//the R representation is inherently 'dense' so we can just iterate over the columns +//Might be useful to store? - DataType indices_dtype = sample_indices.getDataType(); - DataType data_dtype = sample_data.getDataType(); - - DataSpace indices_dataspace = sample_indices.getSpace(); - DataSpace data_dataspace = sample_data.getSpace(); +std::vector tse::get_sample_counts() { + std::vector sample_counts = std::vector(); - DataSpace indices_memspace(1, count, NULL); - DataSpace data_memspace(1, count, NULL); - - indices_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); - data_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); - - current_indices_out = (uint32_t*)malloc(sizeof(uint32_t) * count[0]); - if(current_indices_out == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(uint32_t) * count[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - current_data_out = (double*)malloc(sizeof(double) * count[0]); - if(current_data_out == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double) * count[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - - sample_indices.read((void*)current_indices_out, indices_dtype, indices_memspace, indices_dataspace); - sample_data.read((void*)current_data_out, data_dtype, data_memspace, data_dataspace); - - return count[0]; -} - -double* biom::get_sample_counts() { - double *sample_counts = (double*)calloc(sizeof(double), n_samples); - for(unsigned int i = 0; i < n_obs; i++) { - unsigned int count = obs_counts_resident[i]; - uint32_t *indices = obs_indices_resident[i]; - double *data = obs_data_resident[i]; - for(unsigned int j = 0; j < count; j++) { - uint32_t index = indices[j]; - double datum = data[j]; - sample_counts[index] += datum; + for(unsigned int i = 0; i < n_samples; i++) { + unsigned int sum = 0; + for(unsigned int j = 0; j < n_obs; j++){ + sum += assay(j, i); } + sample_counts.push_back(sum); } - return sample_counts; + return(sample_counts); } diff --git a/R/unifrac_cpp/biom.hpp b/R/unifrac_cpp/biom.hpp index 7e3fdb1fc..1e257f2d1 100644 --- a/R/unifrac_cpp/biom.hpp +++ b/R/unifrac_cpp/biom.hpp @@ -11,27 +11,27 @@ #ifndef _UNIFRAC_BIOM_H #define _UNIFRAC_BIOM_H -#include -#include #include #include #include "biom_interface.hpp" +#include + namespace su { - class biom : public biom_interface { + class tse : public tse_interface { public: /* default constructor * - * @param filename The path to the BIOM table to read + * @param treeSE An R TreeSummarizedExperiment object */ - biom(std::string filename); + tse(const Rcpp::S4 & treeSE); /* default destructor * * Temporary arrays are freed */ - virtual ~biom(); + virtual ~tse(); /* get a dense vector of observation data * @@ -40,36 +40,12 @@ namespace su { * Values of an index position [0, n_samples) which do not * have data will be zero'd. */ - void get_obs_data(const std::string &id, double* out) const; - void get_obs_data(const std::string &id, float* out) const; - - /* get a dense vector of a range of observation data - * - * @param id The observation ID to fetc - * @param start Initial index - * @param end First index past the end - * @param normalize If set, divide by sample_counts - * @param out An allocated array of at least size (end-start). First element will corrrectpoint to index start. - * Values of an index position [0, (end-start)) which do not - * have data will be zero'd. - */ - void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, double* out) const; - void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, float* out) const; + std::vector get_obs_data(const std::string &id) const; private: - /* retain DataSet handles within the HDF5 file */ - H5::DataSet obs_indices; - H5::DataSet sample_indices; - H5::DataSet obs_data; - H5::DataSet sample_data; - H5::H5File file; - uint32_t **obs_indices_resident; - double **obs_data_resident; - unsigned int *obs_counts_resident; - - unsigned int get_obs_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out); - unsigned int get_sample_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out); - double* get_sample_counts(); + Rcpp::NumericMatrix assay; // Access to the raw sample counts in R's memory + + std::vector get_sample_counts(); /* At construction, lookups mapping IDs -> index position within an * axis are defined @@ -103,11 +79,10 @@ namespace su { void create_id_index(std::vector &ids, std::unordered_map &map); - // templatized version - template void get_obs_data_TT(const std::string &id, TFloat* out) const; - template void get_obs_data_range_TT(const std::string &id, unsigned int start, unsigned int end, bool normalize, TFloat* out) const; - }; + template std::vector get_obs_data_TT(const std::string &id, TFloat t) const; + + }; } #endif /* _UNIFRAC_BIOM_H */ diff --git a/R/unifrac_cpp/biom_interface.hpp b/R/unifrac_cpp/biom_interface.hpp index bbfc80e4d..73e09e0bd 100644 --- a/R/unifrac_cpp/biom_interface.hpp +++ b/R/unifrac_cpp/biom_interface.hpp @@ -14,35 +14,37 @@ #include #include +#include + +//Faith calculations mainly need n_samples, get_obs_data and sample_counts +//sample_counts - OK +//n_samples - OK +//get_obs_data + namespace su { - class biom_interface { + class tse_interface { public: // cache the IDs contained within the table std::vector sample_ids; std::vector obs_ids; - // cache both index pointers into both CSC and CSR representations - std::vector sample_indptr; - std::vector obs_indptr; - uint32_t n_samples; // the number of samples uint32_t n_obs; // the number of observations - uint32_t nnz; // the total number of nonzero entries - double *sample_counts; + std::vector sample_counts; // Counts summed per sample /* default constructor * * Automatically create the needed objects. * All other initialization happens in children constructors. */ - biom_interface() {} + tse_interface() {} /* default destructor * * Automatically destroy the objects. * All other cleanup must have been performed by the children constructors. */ - virtual ~biom_interface() {} + virtual ~tse_interface() {} /* get a dense vector of observation data * @@ -51,22 +53,8 @@ namespace su { * Values of an index position [0, n_samples) which do not * have data will be zero'd. */ - virtual void get_obs_data(const std::string &id, double* out) const = 0; - virtual void get_obs_data(const std::string &id, float* out) const = 0; - - /* get a dense vector of a range of observation data - * - * @param id The observation ID to fetc - * @param start Initial index - * @param end First index past the end - * @param normalize If set, divide by sample_counts - * @param out An allocated array of at least size (end-start). First element will corrrectpoint to index start. - * Values of an index position [0, (end-start)) which do not - * have data will be zero'd. - */ - virtual void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, double* out) const = 0; - virtual void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, float* out) const = 0; - }; + virtual std::vector get_obs_data(const std::string &id) const = 0; + }; } #endif /* _UNIFRAC_BIOOM_INTERFACE_H */ diff --git a/R/unifrac_cpp/biom_interface_s.hpp b/R/unifrac_cpp/biom_interface_s.hpp deleted file mode 100644 index 73e09e0bd..000000000 --- a/R/unifrac_cpp/biom_interface_s.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2021-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - - -#ifndef _UNIFRAC_BIOM_INTERFACE_H -#define _UNIFRAC_BIOM_INTERFACE_H - -#include -#include - -#include - -//Faith calculations mainly need n_samples, get_obs_data and sample_counts -//sample_counts - OK -//n_samples - OK -//get_obs_data - -namespace su { - class tse_interface { - public: - // cache the IDs contained within the table - std::vector sample_ids; - std::vector obs_ids; - - uint32_t n_samples; // the number of samples - uint32_t n_obs; // the number of observations - std::vector sample_counts; // Counts summed per sample - - /* default constructor - * - * Automatically create the needed objects. - * All other initialization happens in children constructors. - */ - tse_interface() {} - - /* default destructor - * - * Automatically destroy the objects. - * All other cleanup must have been performed by the children constructors. - */ - virtual ~tse_interface() {} - - /* get a dense vector of observation data - * - * @param id The observation ID to fetch - * @param out An allocated array of at least size n_samples. - * Values of an index position [0, n_samples) which do not - * have data will be zero'd. - */ - virtual std::vector get_obs_data(const std::string &id) const = 0; - }; -} - -#endif /* _UNIFRAC_BIOOM_INTERFACE_H */ diff --git a/R/unifrac_cpp/biom_s.cpp b/R/unifrac_cpp/biom_s.cpp deleted file mode 100644 index 7a627c903..000000000 --- a/R/unifrac_cpp/biom_s.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include -#include -#include -#include -#include "biom_s.hpp" - -#include - -using namespace su; - -tse::tse(const Rcpp::S4 & treeSE) { - sample_ids = std::vector(); - obs_ids = std::vector(); - - Rcpp::S4 colData = treeSE.slot("colData"); - Rcpp::StringVector rownames = colData.slot("rownames"); - sample_ids = Rcpp::as>(rownames); - - Rcpp::List rowTree = treeSE.slot("rowTree"); - Rcpp::List phylo = rowTree["phylo"]; - Rcpp::StringVector tip_label = phylo["tip.label"]; - obs_ids = Rcpp::as>(tip_label); - - Rcpp::S4 assays = treeSE.slot("assays"); - Rcpp::S4 data = assays.slot("data"); - Rcpp::List listData = data.slot("listData"); - assay = Rcpp::as(listData["counts"]); - - n_samples = sample_ids.size(); - n_obs = obs_ids.size(); - - /* define a mapping between an ID and its corresponding offset */ - obs_id_index = std::unordered_map(); - sample_id_index = std::unordered_map(); - - create_id_index(obs_ids, obs_id_index); - create_id_index(sample_ids, sample_id_index); - - sample_counts = get_sample_counts(); - -} - -tse::~tse() { - -} - -void tse::create_id_index(std::vector &ids, - std::unordered_map &map) { - uint32_t count = 0; - map.reserve(ids.size()); - for(auto i = ids.begin(); i != ids.end(); i++, count++) { - map[*i] = count; - } -} - -//Basically just gets the row for the specified id -template -std::vector tse::get_obs_data_TT(const std::string &id, TFloat t) const { - std::vector out = std::vector(); - uint32_t idx = obs_id_index.at(id); - for(unsigned int i = 0; i < n_samples; i++) { - out.push_back(assay(idx, i)); - } - return out; -} - -std::vector tse::get_obs_data(const std::string &id) const { - double t = 0.0; - return(tse::get_obs_data_TT(id, t)); -} - - -//Returns a pointer-based array - can perhaps be changed to simply referring to the R object's internal storage? -//What exactly does this array contain? It contains n_samples elements which are doubles. -//I'm fairly sure that it just sums the counts over samples. Basically just get a column sum. -//std::vector uses move semantics so it shouldn't affect memory usage too much -//the R representation is inherently 'dense' so we can just iterate over the columns -//Might be useful to store? - -std::vector tse::get_sample_counts() { - std::vector sample_counts = std::vector(); - - for(unsigned int i = 0; i < n_samples; i++) { - unsigned int sum = 0; - for(unsigned int j = 0; j < n_obs; j++){ - sum += assay(j, i); - } - sample_counts.push_back(sum); - } - return(sample_counts); -} diff --git a/R/unifrac_cpp/biom_s.hpp b/R/unifrac_cpp/biom_s.hpp deleted file mode 100644 index 78f5d8d49..000000000 --- a/R/unifrac_cpp/biom_s.hpp +++ /dev/null @@ -1,91 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - - -#ifndef _UNIFRAC_BIOM_H -#define _UNIFRAC_BIOM_H - -#include -#include -#include -#include - -#include "biom_interface_s.hpp" - -#include - -namespace su { - class tse : public tse_interface { - public: - /* default constructor - * - * @param treeSE An R TreeSummarizedExperiment object - */ - tse(const Rcpp::S4 & treeSE); - - /* default destructor - * - * Temporary arrays are freed - */ - virtual ~tse(); - - /* get a dense vector of observation data - * - * @param id The observation ID to fetch - * @param out An allocated array of at least size n_samples. - * Values of an index position [0, n_samples) which do not - * have data will be zero'd. - */ - std::vector get_obs_data(const std::string &id) const; - - private: - Rcpp::NumericMatrix assay; // Access to the raw sample counts in R's memory - - std::vector get_sample_counts(); - - /* At construction, lookups mapping IDs -> index position within an - * axis are defined - */ - std::unordered_map obs_id_index; - std::unordered_map sample_id_index; - - /* load ids from an axis - * - * @param path The dataset path to the ID dataset to load - * @param ids The variable representing the IDs to load into - */ - void load_ids(const char *path, std::vector &ids); - - /* load the index pointer for an axis - * - * @param path The dataset path to the index pointer to load - * @param indptr The vector to load the data into - */ - void load_indptr(const char *path, std::vector &indptr); - - /* count the number of nonzero values and set nnz */ - void set_nnz(); - - /* create an index mapping an ID to its corresponding index - * position. - * - * @param ids A vector of IDs to index - * @param map A hash table to populate - */ - void create_id_index(std::vector &ids, - std::unordered_map &map); - - // templatized version - template std::vector get_obs_data_TT(const std::string &id, TFloat t) const; - - }; -} - -#endif /* _UNIFRAC_BIOM_H */ - diff --git a/R/unifrac_cpp/capi_test.c b/R/unifrac_cpp/capi_test.c deleted file mode 100644 index c1981f048..000000000 --- a/R/unifrac_cpp/capi_test.c +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include -#include -#include "api.hpp" - -#ifndef bool -#define bool char -#define true 1 -#define false 0 -#endif - -void err(bool condition, const char* msg) { - if(condition) { - fprintf(stderr, "%s\n", msg); - exit(1); - } -} - -void test_su(int num_cores){ - mat_t* result = NULL; - const char* table = "test.biom"; - const char* tree = "test.tre"; - const char* method = "unweighted"; - double exp[] = {0.2, 0.57142857, 0.6, 0.5, 0.2, 0.42857143, 0.66666667, 0.6, 0.33333333, 0.71428571, 0.85714286, 0.42857143, 0.33333333, 0.4, 0.6}; - - ComputeStatus status; - status = one_off(table, tree, method, - false, 1.0, false, num_cores, &result); - - err(status != okay, "Compute failed"); - err(result == NULL, "Empty result"); - err(result->n_samples != 6, "Wrong number of samples"); - err(result->cf_size != 15, "Wrong condensed form size"); - err(!result->is_upper_triangle, "Result is not squaure"); - - for(unsigned int i = 0; i < result->cf_size; i++) - err(fabs(exp[i] - result->condensed_form[i]) > 0.00001, "Result is wrong"); - -} - -void test_faith_pd(){ - r_vec* result = NULL; - const char* table = "test.biom"; - const char* tree = "test.tre"; - double exp[] = {4, 5, 6, 3, 2, 5}; - - ComputeStatus status; - status = faith_pd_one_off(table, tree, &result); - - err(status != okay, "Compute failed"); - err(result == NULL, "Empty result"); - err(result->n_samples != 6, "Wrong number of samples"); - - for(unsigned int i = 0; i < result->n_samples; i++) - err(fabs(exp[i] - result->values[i]) > 0.00001, "Result is wrong"); - -} - -int main(int argc, char** argv) { - int num_cores = strtol(argv[1], NULL, 10); - - printf("Testing Striped UniFrac...\n"); - test_su(num_cores); - printf("Tests passed.\n"); - printf("Testing Faith's PD...\n"); - test_faith_pd(); - printf("Tests passed.\n"); - return 0; -} - diff --git a/R/unifrac_cpp/cmd.cpp b/R/unifrac_cpp/cmd.cpp deleted file mode 100644 index 4cfd31083..000000000 --- a/R/unifrac_cpp/cmd.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "cmd.hpp" diff --git a/R/unifrac_cpp/cmd.hpp b/R/unifrac_cpp/cmd.hpp deleted file mode 100644 index 408a169e3..000000000 --- a/R/unifrac_cpp/cmd.hpp +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include -#include -#include - -class InputParser{ - /* this object was shamelessly adapted from - http://stackoverflow.com/a/868894 - */ - public: - InputParser (int &argc, char **argv){ - for (int i=1; i < argc; ++i) - this->tokens.push_back(std::string(argv[i])); - } - /// @author iain - const std::string& getCmdOption(const std::string &option) const{ - std::vector::const_iterator itr; - itr = std::find(this->tokens.begin(), this->tokens.end(), option); - if (itr != this->tokens.end() && ++itr != this->tokens.end()){ - return *itr; - } - return empty; - } - /// @author iain - bool cmdOptionExists(const std::string &option) const{ - return std::find(this->tokens.begin(), this->tokens.end(), option) - != this->tokens.end(); - } - private: - std::vector tokens; - const std::string empty; -}; diff --git a/R/unifrac_cpp/faithpd.cpp b/R/unifrac_cpp/faithpd.cpp deleted file mode 100644 index 206749653..000000000 --- a/R/unifrac_cpp/faithpd.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include -#include -#include -#include -#include "api.hpp" -#include "cmd.hpp" -#include "tree.hpp" -#include "biom.hpp" -#include "unifrac.hpp" - - -void usage() { - std::cout << "usage: faithpd -i -t -o " << std::endl; - std::cout << std::endl; - std::cout << " -i\t\tThe input BIOM table." << std::endl; - std::cout << " -t\t\tThe input phylogeny in newick." << std::endl; - std::cout << " -o\t\tThe output series." << std::endl; - std::cout << std::endl; - std::cout << "Citations: " << std::endl; - std::cout << " For Faith's PD, please see:" << std::endl; - std::cout << " Faith Biological Conservation 1992; DOI: 10.1016/0006-3207(92)91201-3" << std::endl; - std::cout << std::endl; - -} - -const char* compute_status_messages[7] = {"No error.", - "The tree file cannot be found.", - "The table file cannot be found.", - "The table file contains an empty table.", - "An unknown method was requested.", - "Table observation IDs are not a subset of the tree tips. This error can also be triggered if a node name contains a single quote (this is unlikely).", - "Error creating the output."}; - -void err(std::string msg) { - std::cerr << "ERROR: " << msg << std::endl << std::endl; - usage(); -} - -int faith_cli_one_off(std::string table_filename, std::string tree_filename, - std::string output_filename) { - if(output_filename.empty()) { - err("output filename missing"); - return EXIT_FAILURE; - } - - if(table_filename.empty()) { - err("table filename missing"); - return EXIT_FAILURE; - } - - if(tree_filename.empty()) { - err("tree filename missing"); - return EXIT_FAILURE; - } - - r_vec *result = NULL; - compute_status status; - status = faith_pd_one_off(table_filename.c_str(), tree_filename.c_str(), &result); - if(status != okay || result == NULL) { - fprintf(stderr, "Compute failed in faith_pd_one_off: %s\n", compute_status_messages[status]); - exit(EXIT_FAILURE); - } - - write_vec(output_filename.c_str(), result); - destroy_results_vec(&result); - - return EXIT_SUCCESS; -} - -int main(int argc, char **argv){ - InputParser input(argc, argv); - if(input.cmdOptionExists("-h") || input.cmdOptionExists("--help") || argc == 1) { - usage(); - return EXIT_SUCCESS; - } - - const std::string &table_filename = input.getCmdOption("-i"); - const std::string &tree_filename = input.getCmdOption("-t"); - const std::string &output_filename = input.getCmdOption("-o"); - - faith_cli_one_off(table_filename, tree_filename, output_filename); - - return EXIT_SUCCESS; -} diff --git a/R/unifrac_cpp/skbio_alt.cpp b/R/unifrac_cpp/skbio_alt.cpp deleted file mode 100644 index 464f6f78b..000000000 --- a/R/unifrac_cpp/skbio_alt.cpp +++ /dev/null @@ -1,617 +0,0 @@ -/* - * Classes, methods and unction that provide skbio-like unctionality - */ - -#include "skbio_alt.hpp" -#include - -#include - -// Not using anything mkl specific, but this is what we get from Conda -#include -#include - -// Compute the E_matrix with means -// centered must be pre-allocated and same size as mat (n_samples*n_samples)...will work even if centered==mat -// row_means must be pre-allocated and n_samples in size -template -inline void E_matrix_means(const TRealIn * mat, const uint32_t n_samples, // IN - TReal * centered, TReal * row_means, TReal &global_mean) { // OUT - /* - Compute E matrix from a distance matrix and store in temp centered matrix. - - Squares and divides by -2 the input elementwise. Eq. 9.20 in - Legendre & Legendre 1998. - - Compute sum of the rows at the same time. - */ - - TReal global_sum = 0.0; - -#pragma omp parallel for shared(mat,centered,row_means) reduction(+: global_sum) - for (uint32_t row=0; row -inline void F_matrix_inplace(const TReal * __restrict__ row_means, const TReal global_mean, TReal * __restrict__ centered, const uint32_t n_samples) { - /* - Compute F matrix from E matrix. - - Centring step: for each element, the mean of the corresponding - row and column are substracted, and the mean of the whole - matrix is added. Eq. 9.21 in Legendre & Legendre 1998. - Pseudo-code: - row_means = E_matrix.mean(axis=1, keepdims=True) - col_means = Transpose(row_means) - matrix_mean = E_matrix.mean() - return E_matrix - row_means - col_means + matrix_mean - */ - - // use a tiled pattern to maximize locality of row_means -#pragma omp parallel for shared(centered,row_means) - for (uint32_t trow=0; trow -inline void mat_to_centered_T(const TRealIn * mat, const uint32_t n_samples, TReal * centered) { - - TReal global_mean; - TReal *row_means = (TReal *) malloc(uint64_t(n_samples)*sizeof(TReal)); - E_matrix_means(mat, n_samples, centered, row_means, global_mean); - F_matrix_inplace(row_means, global_mean, centered, n_samples); - free(row_means); -} - -void su::mat_to_centered(const double * mat, const uint32_t n_samples, double * centered) { - mat_to_centered_T(mat,n_samples,centered); -} - -void su::mat_to_centered(const float * mat, const uint32_t n_samples, float * centered) { - mat_to_centered_T(mat,n_samples,centered); -} - -void su::mat_to_centered(const double * mat, const uint32_t n_samples, float * centered) { - mat_to_centered_T(mat,n_samples,centered); -} - -// Matrix dot multiplication -// Expects FORTRAN-style ColOrder -// mat must be cols x rows -// other must be cols x rows (ColOrder... rows elements together) -template -inline void mat_dot_T(const TReal *mat, const TReal *other, const uint32_t rows, const uint32_t cols, TReal *out); - -template<> -inline void mat_dot_T(const double *mat, const double *other, const uint32_t rows, const uint32_t cols, double *out) -{ - cblas_dgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, other, rows, 0.0, out, rows); -} - -template<> -inline void mat_dot_T(const float *mat, const float *other, const uint32_t rows, const uint32_t cols, float *out) -{ - cblas_sgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, other, rows, 0.0, out, rows); -} - -// Expects FORTRAN-style ColOrder -// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. -// Original Paper: https://arxiv.org/abs/1007.5510 -// Step 1 -// centered == n x n -// randomized = k*2 x n (ColOrder... n elements together) -template -inline void centered_randomize_T(const TReal * centered, const uint32_t n_samples, const uint32_t k, TReal * randomized) { - uint64_t matrix_els = uint64_t(n_samples)*uint64_t(k); - TReal * tmp = (TReal *) malloc(matrix_els*sizeof(TReal)); - - // Form a real n x k matrix whose entries are independent, identically - // distributed Gaussian random variables of zero mean and unit variance - TReal *G = tmp; - { - std::default_random_engine generator; - std::normal_distribution distribution; - for (uint64_t i=0; i(centered,G,n_samples,k,randomized); - - // power method... single iteration.. store in 2nd part of output - // Reusing tmp buffer for intermediate storage - mat_dot_T(centered,randomized,n_samples,k,tmp); - mat_dot_T(centered,tmp,n_samples,k,randomized+matrix_els); - - free(tmp); -} - -// templated LAPACKE wrapper - -// Compute QR -// H is in,overwritten by Q on out -// H is (r x c), Q is (r x qc), with rc<=c -template -inline int qr_i_T(const uint32_t rows, const uint32_t cols, TReal *H, uint32_t &qcols); - -template<> -inline int qr_i_T(const uint32_t rows, const uint32_t cols, double *H, uint32_t &qcols) { - qcols= std::min(rows,cols); - double *tau= new double[qcols]; - int rc = LAPACKE_dgeqrf(LAPACK_COL_MAJOR, rows, cols, H, rows, tau); - if (rc==0) { - qcols= std::min(rows,cols); - rc = LAPACKE_dorgqr(LAPACK_COL_MAJOR, rows, qcols, qcols, H, rows, tau); - } - delete[] tau; - return rc; -} - -template<> -inline int qr_i_T(const uint32_t rows, const uint32_t cols, float *H, uint32_t &qcols) { - qcols= std::min(rows,cols); - float *tau= new float[qcols]; - int rc = LAPACKE_sgeqrf(LAPACK_COL_MAJOR, rows, cols, H, rows, tau); - if (rc==0) { - qcols= std::min(rows,cols); - rc = LAPACKE_sorgqr(LAPACK_COL_MAJOR, rows, qcols, qcols, H, rows, tau); - } - delete[] tau; - return rc; -} - -namespace su { - -// helper class, since QR ops are multi function -template -class QR { - public: - uint32_t rows; - uint32_t cols; - - TReal *Q; - - // will take ownership of _H - QR(const uint32_t _rows, const uint32_t _cols, TReal *_H) - : rows(_rows) - , Q(_H) - { - int rc = qr_i_T(_rows, _cols, Q, cols); - if (rc!=0) { - fprintf(stderr, "qr_i_T(_rows,_cols, H, cols) failed with %i\n", rc); - exit(1); // should never fail - } - } - - ~QR() { - free(Q); - } - - // res = mat * Q - // mat must be rows x rows - // res will be rows * cols - void qdot_r_sq(const TReal *mat, TReal *res); - - // res = Q * mat - // mat must be cols * cols - // res will be rows * cols - void qdot_l_sq(const TReal *mat, TReal *res); - -}; - -} - -template<> -inline void su::QR::qdot_r_sq(const double *mat, double *res) { - cblas_dgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, Q, rows, 0.0, res, rows); -} - -template<> -inline void su::QR::qdot_r_sq(const float *mat, float *res) { - cblas_sgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, Q, rows, 0.0, res, rows); -} - -template<> -inline void su::QR::qdot_l_sq(const double *mat, double *res) { - cblas_dgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, cols, 1.0, Q, rows, mat, cols, 0.0, res, rows); -} - -template<> -inline void su::QR::qdot_l_sq(const float *mat, float *res) { - cblas_sgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, cols, 1.0, Q, rows, mat, cols, 0.0, res, rows); -} - -// compute svd, and return S and V -// T = input -// S output -// T is Vt on output -template -inline int svd_it_T(const uint32_t rows, const uint32_t cols, TReal *T, TReal *S); - -template<> -inline int svd_it_T(const uint32_t rows, const uint32_t cols, double *T, double *S) { - double *superb = (double *) malloc(sizeof(double)*rows); - int res =LAPACKE_dgesvd(LAPACK_COL_MAJOR, 'N', 'O', rows, cols, T, rows, S, NULL, rows, NULL, cols, superb); - free(superb); - - return res; -} - -template<> -inline int svd_it_T(const uint32_t rows, const uint32_t cols, float *T, float *S) { - float *superb = (float *) malloc(sizeof(float)*rows); - int res =LAPACKE_sgesvd(LAPACK_COL_MAJOR, 'N', 'O', rows, cols, T, rows, S, NULL, rows, NULL, cols, superb); - free(superb); - - return res; -} - -// square matrix transpose, with org not alingned -template -inline void transpose_sq_st_T(const uint64_t n, const uint64_t stride, const TReal *in, TReal *out) { - // n expected to be small, so simple single-thread perfect - // org_n>=n guaranteed - for (uint64_t i=0; i -inline void transpose_T(const uint64_t rows, const uint64_t cols, const TReal *in, TReal *out) { - // To be optimizedc - for (uint64_t i=0; i -inline void find_eigens_fast_T(const uint32_t n_samples, const uint32_t n_dims, TReal * centered, TReal * &eigenvalues, TReal * &eigenvectors) { - const uint32_t k = n_dims+2; - - int rc; - - TReal *S = (TReal *) malloc(uint64_t(n_samples)*sizeof(TReal)); // take worst case size as a start - TReal *Ut = NULL; - - { - TReal *H = (TReal *) malloc(sizeof(TReal)*uint64_t(n_samples)*uint64_t(k)*2); - - // step 1 - centered_randomize_T(centered, n_samples, k, H); - - // step 2 - // QR decomposition of H - - su::QR qr_obj(n_samples, k*2, H); // H is now owned by qr_obj, as Q - - // step 3 - // T = centered * Q (since centered^T == centered, due to being symmetric) - // centered = n x n - // T = n x ref - - TReal *T = (TReal *) malloc(sizeof(TReal)*uint64_t(qr_obj.rows)*uint64_t(qr_obj.cols)); - qr_obj.qdot_r_sq(centered,T); - - // step 4 - // compute svd - // update T in-place, Wt on output (Vt according to the LAPACK nomenclature) - rc=svd_it_T(qr_obj.rows,qr_obj.cols, T, S); - if (rc!=0) { - fprintf(stderr, "svd_it_T(n_samples, T, S) failed with %i\n",rc); - exit(1); // should never fail - } - - // step 5 - // Compute U = Q*Wt^t - { - // transpose Wt -> W, Wt uses n_samples strides - TReal * W = (TReal *) malloc(sizeof(TReal)*uint64_t(qr_obj.cols)*uint64_t(qr_obj.cols)); - transpose_sq_st_T(qr_obj.cols, qr_obj.rows, T, W); // Wt == T on input - - Ut = T; // Ut takes ownership of the T buffer - qr_obj.qdot_l_sq(W, Ut); - - free(W); - } - - } // we don't need qr_obj anymore, release memory - - // step 6 - // get the interesting subset, and return - - // simply truncate the values, since it is a vector - eigenvalues = (TReal *) realloc(S, sizeof(TReal)*n_dims); - - // *eigenvectors = U = Vt - // use only the truncated part of W, then transpose - TReal *U = (TReal *) malloc(uint64_t(n_samples)*uint64_t(n_dims)*sizeof(TReal)); - - transpose_T(n_samples, n_dims, Ut, U); - eigenvectors = U; - - free(Ut); -} - -void su::find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double * &eigenvalues, double * &eigenvectors) { - find_eigens_fast_T(n_samples, n_dims, centered, eigenvalues, eigenvectors); -} - -void su::find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, float * centered, float * &eigenvalues, float * &eigenvectors) { - find_eigens_fast_T(n_samples, n_dims, centered, eigenvalues, eigenvectors); -} - -// helper class - -namespace su { - -template -class NewCentered { -private: - const uint32_t n_samples; - const uint32_t n_dims; - TReal * centered_buf; -public: - NewCentered(const uint32_t _n_samples, const uint32_t _n_dims) - : n_samples(_n_samples) - , n_dims(_n_dims) - , centered_buf(NULL) - {} - - TReal * get_buf() { - if (centered_buf==NULL) centered_buf = (TReal *) malloc(sizeof(TReal)*uint64_t(n_samples)*uint64_t(n_samples)); - return centered_buf; - } - - void release_buf() { - if (centered_buf!=NULL) free(centered_buf); - centered_buf=NULL; - } - - ~NewCentered() { - if (centered_buf!=NULL) release_buf(); - } - -private: - NewCentered(const NewCentered &other) = delete; - NewCentered& operator=(const NewCentered &other) = delete; -}; - -template -class InPlaceCentered { -private: - TReal * mat; -public: - InPlaceCentered(TReal * _mat) - : mat(_mat) - {} - - TReal * get_buf() { return mat; } - - void release_buf() {} - - ~InPlaceCentered() {} -}; - -} - -/* - Perform Principal Coordinate Analysis. - - Principal Coordinate Analysis (PCoA) is a method similar - to Principal Components Analysis (PCA) with the difference that PCoA - operates on distance matrices, typically with non-euclidian and thus - ecologically meaningful distances like UniFrac in microbiome research. - - In ecology, the euclidean distance preserved by Principal - Component Analysis (PCA) is often not a good choice because it - deals poorly with double zeros (Species have unimodal - distributions along environmental gradients, so if a species is - absent from two sites at the same site, it can't be known if an - environmental variable is too high in one of them and too low in - the other, or too low in both, etc. On the other hand, if an - species is present in two sites, that means that the sites are - similar.). - - Note that the returned eigenvectors are not normalized to unit length. -*/ - -// mat - in, result of unifrac compute -// inplace - in, if true, use mat as a work buffer -// n_samples - in, size of the matrix (n x n) -// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. -// eigenvalues - out, alocated buffer of size n_dims -// samples - out, alocated buffer of size n_dims x n_samples -// proportion_explained - out, allocated buffer of size n_dims - -template -inline void pcoa_T(TRealIn * mat, TCenter ¢er_obj, const uint32_t n_samples, const uint32_t n_dims, TReal * &eigenvalues, TReal * &samples,TReal * &proportion_explained) { - proportion_explained = (TReal *) malloc(sizeof(TReal)*n_dims); - - TReal diag_sum = 0.0; - TReal *eigenvectors = NULL; - - { - TReal *centered = center_obj.get_buf(); - - // First must center the matrix - mat_to_centered_T(mat,n_samples,centered); - - // get the sum of the diagonal, needed later - // and centered will be updated in-place in find_eigen - for (uint32_t i=0; i(n_samples,n_dims,centered,eigenvalues,eigenvectors); - - center_obj.release_buf(); - } - - // expects eigenvalues to be ordered and non-negative - // The above unction guarantees that - - - // Scale eigenvalues to have length = sqrt(eigenvalue). This - // works because np.linalg.eigh returns normalized - // eigenvectors. Each row contains the coordinates of the - // objects in the space of principal coordinates. Note that at - // least one eigenvalue is zero because only n-1 axes are - // needed to represent n points in a euclidean space. - // samples = eigvecs * np.sqrt(eigvals) - // we will just update in place and pass out - samples = eigenvectors; - - // use proportion_explained as tmp buffer here - { - TReal *sqvals = proportion_explained; - for (uint32_t i=0; i cobj(n_samples, n_dims); - pcoa_T(mat, cobj , n_samples, n_dims, eigenvalues, samples, proportion_explained); -} - -void su::pcoa(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained) { - su::NewCentered cobj(n_samples, n_dims); - pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); -} - -void su::pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained) { - su::NewCentered cobj(n_samples, n_dims); - pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); -} - -void su::pcoa_inplace(double * mat, const uint32_t n_samples, const uint32_t n_dims, double * &eigenvalues, double * &samples, double * &proportion_explained) { - su::InPlaceCentered cobj(mat); - pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); -} - -void su::pcoa_inplace(float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained) { - su::InPlaceCentered cobj(mat); - pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); -} - diff --git a/R/unifrac_cpp/skbio_alt.hpp b/R/unifrac_cpp/skbio_alt.hpp deleted file mode 100644 index 284f0ea6e..000000000 --- a/R/unifrac_cpp/skbio_alt.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Classes, methods and unction that provide skbio-like unctionality - */ - -#ifndef UNIFRAC_SKBIO_ALT_H -#define UNIFRAC_SKBIO_ALT_H - -#include - -namespace su { - -// Center the matrix -// mat and center must be nxn and symmetric -// centered must be pre-allocated and same size as mat...will work even if centered==mat -void mat_to_centered(const double * mat, const uint32_t n_samples, double * centered); -void mat_to_centered(const float * mat, const uint32_t n_samples, float * centered); -void mat_to_centered(const double * mat, const uint32_t n_samples, float * centered); - -// Find eigen values and vectors -// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. -// Original Paper: https://arxiv.org/abs/1007.5510 -// centered == n x n, must be symmetric, Note: will be used in-place as temp buffer -void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double * &eigenvalues, double * &eigenvectors); -void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, float * centered, float * &eigenvalues, float * &eigenvectors); - -// Perform Principal Coordinate Analysis -// mat - in, result of unifrac compute -// n_samples - in, size of the matrix (n x n) -// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. -// eigenvalues - out, alocated buffer of size n_dims -// samples - out, alocated buffer of size n_dims x n_samples -// proportion_explained - out, allocated buffer of size n_dims -void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, double * &eigenvalues, double * &samples, double * &proportion_explained); -void pcoa(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained); -void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained); - -// in-place version, will use mat as temp buffer internally -void pcoa_inplace(double * mat, const uint32_t n_samples, const uint32_t n_dims, double * &eigenvalues, double * &samples, double * &proportion_explained); -void pcoa_inplace(float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained); - - -} - -#endif diff --git a/R/unifrac_cpp/su.cpp b/R/unifrac_cpp/su.cpp deleted file mode 100644 index 25c238738..000000000 --- a/R/unifrac_cpp/su.cpp +++ /dev/null @@ -1,492 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include "api.hpp" -#include "cmd.hpp" -#include "tree.hpp" -#include "biom.hpp" -#include "unifrac.hpp" - -enum Format {format_invalid,format_ascii, format_hdf5_fp32, format_hdf5_fp64}; - -void usage() { - std::cout << "usage: ssu -i -o -m [METHOD] -t [-n threads] [-a alpha] [-f] [--vaw]" << std::endl; - std::cout << " [--mode [MODE]] [--start starting-stripe] [--stop stopping-stripe] [--partial-pattern ]" << std::endl; - std::cout << " [--n-partials number_of_partitions] [--report-bare] [--format|-r out-mode]" << std::endl; - std::cout << std::endl; - std::cout << " -i\t\tThe input BIOM table." << std::endl; - std::cout << " -t\t\tThe input phylogeny in newick." << std::endl; - std::cout << " -m\t\tThe method, [unweighted | weighted_normalized | weighted_unnormalized | generalized | unweighted_fp32 | weighted_normalized_fp32 | weighted_unnormalized_fp32 | generalized_fp32]." << std::endl; - std::cout << " -o\t\tThe output distance matrix." << std::endl; - std::cout << " -n\t\t[OPTIONAL] The number of threads, default is 1." << std::endl; - std::cout << " -a\t\t[OPTIONAL] Generalized UniFrac alpha, default is 1." << std::endl; - std::cout << " -f\t\t[OPTIONAL] Bypass tips, reduces compute by about 50%." << std::endl; - std::cout << " --vaw\t[OPTIONAL] Variance adjusted, default is to not adjust for variance." << std::endl; - std::cout << " --mode\t[OPTIONAL] Mode of operation:" << std::endl; - std::cout << " \t\t one-off : [DEFAULT] compute UniFrac." << std::endl; - std::cout << " \t\t partial : Compute UniFrac over a subset of stripes." << std::endl; - std::cout << " \t\t partial-report : Start and stop suggestions for partial compute." << std::endl; - std::cout << " \t\t merge-partial : Merge partial UniFrac results." << std::endl; - std::cout << " --start\t[OPTIONAL] If mode==partial, the starting stripe." << std::endl; - std::cout << " --stop\t[OPTIONAL] If mode==partial, the stopping stripe." << std::endl; - std::cout << " --partial-pattern\t[OPTIONAL] If mode==merge-partial, a glob pattern for partial outputs to merge." << std::endl; - std::cout << " --n-partials\t[OPTIONAL] If mode==partial-report, the number of partitions to compute." << std::endl; - std::cout << " --report-bare\t[OPTIONAL] If mode==partial-report, produce barebones output." << std::endl; - std::cout << " --format|-r\t[OPTIONAL] Output format:" << std::endl; - std::cout << " \t\t ascii : [DEFAULT] Original ASCII format." << std::endl; - std::cout << " \t\t hfd5 : HFD5 format. May be fp32 or fp64, depending on method." << std::endl; - std::cout << " \t\t hdf5_fp32 : HFD5 format, using fp32 precision." << std::endl; - std::cout << " \t\t hdf5_fp64 : HFD5 format, using fp64 precision." << std::endl; - std::cout << " --pcoa\t[OPTIONAL] Number of PCoA dimensions to compute (default: 10, do not compute if 0)" << std::endl; - std::cout << " --diskbuf\t[OPTIONAL] Use a disk buffer to reduce memory footprint. Provide path to a fast partition (ideally NVMe)." << std::endl; - std::cout << std::endl; - std::cout << "Citations: " << std::endl; - std::cout << " For UniFrac, please see:" << std::endl; - std::cout << " McDonald et al. Nature Methods 2018; DOI: 10.1038/s41592-018-0187-8" << std::endl; - std::cout << " Lozupone and Knight Appl Environ Microbiol 2005; DOI: 10.1128/AEM.71.12.8228-8235.2005" << std::endl; - std::cout << " Lozupone et al. Appl Environ Microbiol 2007; DOI: 10.1128/AEM.01996-06" << std::endl; - std::cout << " Hamady et al. ISME 2010; DOI: 10.1038/ismej.2009.97" << std::endl; - std::cout << " Lozupone et al. ISME 2011; DOI: 10.1038/ismej.2010.133" << std::endl; - std::cout << " For Generalized UniFrac, please see: " << std::endl; - std::cout << " Chen et al. Bioinformatics 2012; DOI: 10.1093/bioinformatics/bts342" << std::endl; - std::cout << " For Variance Adjusted UniFrac, please see: " << std::endl; - std::cout << " Chang et al. BMC Bioinformatics 2011; DOI: 10.1186/1471-2105-12-118" << std::endl; - std::cout << std::endl; - std::cout << "Runtime progress can be obtained by issuing a SIGUSR1 signal. If running with " << std::endl; - std::cout << "multiple threads, this signal will only be honored if issued to the master PID. " << std::endl; - std::cout << "The report will yield the following information: " << std::endl; - std::cout << std::endl; - std::cout << "tid: start: stop: k: total:" << std::endl; - std::cout << std::endl; - std::cout << "The proportion of the tree that has been evaluated can be determined from (k / total)." << std::endl; - std::cout << std::endl; -} - -const char* compute_status_messages[7] = {"No error.", - "The tree file cannot be found.", - "The table file cannot be found.", - "The table file contains an empty table.", - "An unknown method was requested.", - "Table observation IDs are not a subset of the tree tips. This error can also be triggered if a node name contains a single quote (this is unlikely).", - "Error creating the output."}; - - -// https://stackoverflow.com/questions/8401777/simple-glob-in-c-on-unix-system -inline std::vector glob(const std::string& pat){ - using namespace std; - glob_t glob_result; - glob(pat.c_str(),GLOB_TILDE,NULL,&glob_result); - vector ret; - for(unsigned int i=0;i partials = glob(partial_pattern); - partial_dyn_mat_t** partial_mats = (partial_dyn_mat_t**)malloc(sizeof(partial_dyn_mat_t*) * partials.size()); - for(size_t i = 0; i < partials.size(); i++) { - IOStatus io_err = read_partial_header(partials[i].c_str(), &partial_mats[i]); - if(io_err != read_okay) { - std::ostringstream msg; - msg << "Unable to parse file (" << partials[i] << "); err " << io_err; - err(msg.str()); - return EXIT_FAILURE; - } - } - - const char * mmap_dir_c = mmap_dir.empty() ? NULL : mmap_dir.c_str(); - - int status; - if (format_val==format_hdf5_fp64) { - status = mode_merge_partial_fp64(output_filename.c_str(), format_val, pcoa_dims, partials.size(), partial_mats, mmap_dir_c); - } else if (format_val==format_hdf5_fp32) { - status = mode_merge_partial_fp32(output_filename.c_str(), format_val, pcoa_dims, partials.size(), partial_mats, mmap_dir_c); - } else { - status = mode_merge_partial_fp64(output_filename.c_str(), format_val, pcoa_dims, partials.size(), partial_mats, mmap_dir_c); - } - - for(size_t i = 0; i < partials.size(); i++) { - destroy_partial_dyn_mat(&partial_mats[i]); - } - - return status; -} - -int mode_partial(std::string table_filename, std::string tree_filename, - std::string output_filename, std::string method_string, - bool vaw, double g_unifrac_alpha, bool bypass_tips, - unsigned int nthreads, int start_stripe, int stop_stripe) { - if(output_filename.empty()) { - err("output filename missing"); - return EXIT_FAILURE; - } - - if(table_filename.empty()) { - err("table filename missing"); - return EXIT_FAILURE; - } - - if(tree_filename.empty()) { - err("tree filename missing"); - return EXIT_FAILURE; - } - - if(method_string.empty()) { - err("method missing"); - return EXIT_FAILURE; - } - - if(start_stripe < 0) { - err("Starting stripe must be >= 0"); - return EXIT_FAILURE; - } - if(stop_stripe <= start_stripe) { - err("In '--mode partial', the stop and start stripes must be specified, and the stop stripe must be > start stripe"); - return EXIT_FAILURE; - } - - partial_mat_t *result = NULL; - compute_status status; - status = partial(table_filename.c_str(), tree_filename.c_str(), method_string.c_str(), - vaw, g_unifrac_alpha, bypass_tips, nthreads, start_stripe, stop_stripe, &result); - if(status != okay || result == NULL) { - fprintf(stderr, "Compute failed in partial: %s\n", compute_status_messages[status]); - exit(EXIT_FAILURE); - } - - io_status err = write_partial(output_filename.c_str(), result); - destroy_partial_mat(&result); - - if(err != write_okay){ - fprintf(stderr, "Write failed: %s\n", err == open_error ? "could not open output" : "unknown error"); - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; -} - -int mode_one_off(const std::string &table_filename, const std::string &tree_filename, - const std::string &output_filename, const std::string &format_str, Format format_val, - const std::string &method_string, unsigned int pcoa_dims, - bool vaw, double g_unifrac_alpha, bool bypass_tips, - unsigned int nthreads, const std::string &mmap_dir) { - if(output_filename.empty()) { - err("output filename missing"); - return EXIT_FAILURE; - } - - if(table_filename.empty()) { - err("table filename missing"); - return EXIT_FAILURE; - } - - if(tree_filename.empty()) { - err("tree filename missing"); - return EXIT_FAILURE; - } - - if(method_string.empty()) { - err("method missing"); - return EXIT_FAILURE; - } - - compute_status status = okay; - if (format_val==format_ascii) { - mat_t *result = NULL; - - status = one_off(table_filename.c_str(), tree_filename.c_str(), method_string.c_str(), - vaw, g_unifrac_alpha, bypass_tips, nthreads, &result); - if(status != okay || result == NULL) { - fprintf(stderr, "Compute failed in one_off: %s\n", compute_status_messages[status]); - exit(EXIT_FAILURE); - } - - IOStatus iostatus = write_mat(output_filename.c_str(), result); - destroy_mat(&result); - - if(iostatus!=write_okay) { - err("Failed to write output file."); - status = output_error; - } - - } else { - const char * mmap_dir_c = mmap_dir.empty() ? NULL : mmap_dir.c_str(); - - status = unifrac_to_file(table_filename.c_str(), tree_filename.c_str(), output_filename.c_str(), - method_string.c_str(), vaw, g_unifrac_alpha, bypass_tips, nthreads, format_str.c_str(), - pcoa_dims, mmap_dir_c); - - if (status != okay) { - fprintf(stderr, "Compute failed in one_off: %s\n", compute_status_messages[status]); - } - } - - return (status==okay) ? EXIT_SUCCESS : EXIT_FAILURE; -} - -void ssu_sig_handler(int signo) { - if (signo == SIGUSR1) { - printf("Status cannot be reported.\n"); - } -} - -Format get_format(const std::string &format_string, const std::string &method_string) { - Format format_val = format_invalid; - if (format_string.empty()) { - format_val = format_ascii; - } else if (format_string == "ascii") { - format_val = format_ascii; - } else if (format_string == "hdf5_fp32") { - format_val = format_hdf5_fp32; - } else if (format_string == "hdf5_fp64") { - format_val = format_hdf5_fp64; - } else if (format_string == "hdf5") { - if ((method_string=="unweighted_fp32") || (method_string=="weighted_normalized_fp32") || (method_string=="weighted_unnormalized_fp32") || (method_string=="generalized_fp32")) - format_val = format_hdf5_fp32; - else - format_val = format_hdf5_fp64; - } - - return format_val; -} - -int main(int argc, char **argv){ - signal(SIGUSR1, ssu_sig_handler); - InputParser input(argc, argv); - if(input.cmdOptionExists("-h") || input.cmdOptionExists("--help") || argc == 1) { - usage(); - return EXIT_SUCCESS; - } - - unsigned int nthreads; - std::string table_filename = input.getCmdOption("-i"); - std::string tree_filename = input.getCmdOption("-t"); - std::string output_filename = input.getCmdOption("-o"); - std::string method_string = input.getCmdOption("-m"); - std::string nthreads_arg = input.getCmdOption("-n"); - std::string gunifrac_arg = input.getCmdOption("-a"); - std::string mode_arg = input.getCmdOption("--mode"); - std::string start_arg = input.getCmdOption("--start"); - std::string stop_arg = input.getCmdOption("--stop"); - std::string partial_pattern = input.getCmdOption("--partial-pattern"); - std::string npartials = input.getCmdOption("--n-partials"); - std::string report_bare = input.getCmdOption("--report-bare"); - std::string format_arg = input.getCmdOption("--format"); - std::string sformat_arg = input.getCmdOption("-r"); - std::string pcoa_arg = input.getCmdOption("--pcoa"); - std::string diskbuf_arg = input.getCmdOption("--diskbuf"); - - if(nthreads_arg.empty()) { - nthreads = 1; - } else { - nthreads = atoi(nthreads_arg.c_str()); - } - - bool vaw = input.cmdOptionExists("--vaw"); - bool bare = input.cmdOptionExists("--report-bare"); - bool bypass_tips = input.cmdOptionExists("-f"); - double g_unifrac_alpha; - - if(gunifrac_arg.empty()) { - g_unifrac_alpha = 1.0; - } else { - g_unifrac_alpha = atof(gunifrac_arg.c_str()); - } - - int start_stripe; - if(start_arg.empty()) - start_stripe = 0; - else - start_stripe = atoi(start_arg.c_str()); - - int stop_stripe; - if(stop_arg.empty()) - stop_stripe = 0; - else - stop_stripe = atoi(stop_arg.c_str()); - - int n_partials; - if(npartials.empty()) - n_partials = 1; - else - n_partials = atoi(npartials.c_str()); - - if(n_partials<1) { - err("--n-partials cannot be < 1"); - return EXIT_FAILURE; - } - if(n_partials>1000000000) { - err("--n-partials cannot be > 1G"); - return EXIT_FAILURE; - } - - Format format_val = format_invalid; - if(!format_arg.empty()) { - format_val = get_format(format_arg,method_string); - } else { - format_val = get_format(sformat_arg,method_string); - format_arg=sformat_arg; // easier to use a single variable - } - if(format_val==format_invalid) { - err("Invalid format, must be one of ascii|hdf5|hdf5_fp32|hdf5_fp64"); - return EXIT_FAILURE; - } - - unsigned int pcoa_dims; - if(pcoa_arg.empty()) - pcoa_dims = 10; - else - pcoa_dims = atoi(pcoa_arg.c_str()); - - - if(mode_arg.empty() || mode_arg == "one-off") - return mode_one_off(table_filename, tree_filename, output_filename, format_arg, format_val, method_string, pcoa_dims, vaw, g_unifrac_alpha, bypass_tips, nthreads, diskbuf_arg); - else if(mode_arg == "partial") - return mode_partial(table_filename, tree_filename, output_filename, method_string, vaw, g_unifrac_alpha, bypass_tips, nthreads, start_stripe, stop_stripe); - else if(mode_arg == "merge-partial") - return mode_merge_partial(output_filename, format_val, pcoa_dims, partial_pattern, diskbuf_arg); - else if(mode_arg == "partial-report") - return mode_partial_report(table_filename, uint32_t(n_partials), bare); - else - err("Unknown mode. Valid options are: one-off, partial, merge-partial, partial-report"); - - return EXIT_SUCCESS; -} - diff --git a/R/unifrac_cpp/su_R.cpp b/R/unifrac_cpp/su_R.cpp index 0e2b595ef..c65f06f61 100644 --- a/R/unifrac_cpp/su_R.cpp +++ b/R/unifrac_cpp/su_R.cpp @@ -1,44 +1,21 @@ #include -#include #include -#include "api.hpp" - -using namespace std; -using namespace Rcpp; - - -// [[Rcpp::export]] -Rcpp::List unifrac(const char* table, const char* tree, int nthreads){ - mat_t* result = NULL; - const char* method = "unweighted"; - ComputeStatus status; - status = one_off(table, tree, method, false, 1.0, false, nthreads, &result); - vector cf; - //push result->condensed_form into a vector becuase R doesn't like double* - for(int i=0; icf_size; i++){ - cf.push_back(result->condensed_form[i]); - } - return Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, - Rcpp::Named("is_upper_triangle") = result->is_upper_triangle, - Rcpp::Named("cf_size") = result->cf_size, - Rcpp::Named("c_form") = cf); +#include -} +#include "api.hpp" +#include "tree.hpp" // [[Rcpp::export]] -Rcpp::List faith_pd(const char* table, const char* tree){ - r_vec* result = NULL; - ComputeStatus status; - status = faith_pd_one_off(table, tree, &result); - vector values; - for(int i = 0; i < result->n_samples; i++){ - values.push_back(result->values[i]); +Rcpp::NumericVector faith_pd(const Rcpp::S4 & treeSE, bool isRooted){ + + std::vector results = faith_pd_one_off(treeSE, isRooted); + + Rcpp::NumericVector faith = Rcpp::NumericVector(results.size()); + + for(unsigned int i = 0; i < results.size(); i++){ + faith[i] = results[i]; } - - return Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, - Rcpp::Named("faith_pd") = values); - + + return(faith); } - - diff --git a/R/unifrac_cpp/su_R_s.cpp b/R/unifrac_cpp/su_R_s.cpp deleted file mode 100644 index afdeea2de..000000000 --- a/R/unifrac_cpp/su_R_s.cpp +++ /dev/null @@ -1,145 +0,0 @@ -#include -#include -#include -#include "api_s.hpp" -#include "tree_s.hpp" - -#include - -// [[Rcpp::export]] -Rcpp::NumericVector faith_pd(const Rcpp::S4 & treeSE, bool isRooted){ - - std::vector results = faith_pd_one_off(treeSE, isRooted); - - Rcpp::NumericVector faith = Rcpp::NumericVector(results.size()); - - for(unsigned int i = 0; i < results.size(); i++){ - faith[i] = results[i]; - } - - return(faith); -} - - -// [[Rcpp::export]] -Rcpp::LogicalVector rowTree_to_bp(const Rcpp::List & rowTree) { - Rcpp::NumericMatrix edge = rowTree["edge"]; - Rcpp::StringVector tips = rowTree["tip.label"]; - std::vector structure = std::vector(); - - uint32_t ntips = tips.size(); // phylo tips are always numbered from 1 to number of tips; - - std::stack nodes; // Keeps track of the branch's internal nodes - - int currentNode = 0; - int nextNode = 0; - - // Goal: Insert true when a branch starts, a false when it closes, and a true-false for each tip. - - for (unsigned int i = 0; i < edge.nrow(); i++){ - currentNode = edge(i, 0); - nextNode = edge(i, 1); - - if(nodes.size() > 0 && currentNode < nodes.top()) { - // We've exhausted the branch and moved backwards in the tree - do { - nodes.pop(); - structure.push_back(false); - } while(currentNode != nodes.top()); - } - - if(nodes.size() == 0 || currentNode > nodes.top() ) { - // We are either at the root, or entering a new node - // What if the tree is unrooted? - nodes.push(currentNode); - structure.push_back(true); - - } - - if(nextNode <= ntips) { - // We've found a tip - structure.push_back(true); - structure.push_back(false); - } - - if(i == edge.nrow() - 1) { - // We've reached the end of the tree - do { - nodes.pop(); - structure.push_back(false); - } while(nodes.size() > 0); - } - } - - Rcpp::LogicalVector bp = Rcpp::LogicalVector(structure.size()); - - for(unsigned int i = 0; i < structure.size(); i++){ - bp[i] = structure[i]; - } - - return(bp); -} - - -// [[Rcpp::export]] -Rcpp::LogicalVector newick_to_bp(std::string newick) { - char last_structure; - bool potential_single_descendent = false; - int count = 0; - bool in_quote = false; - std::vector structure; - for(auto c = newick.begin(); c != newick.end(); c++) { - if(*c == '\'') - in_quote = !in_quote; - - if(in_quote) - continue; - - switch(*c) { - case '(': - // opening of a node - count++; - structure.push_back(true); - last_structure = *c; - potential_single_descendent = true; - break; - case ')': - // closing of a node - if(potential_single_descendent || (last_structure == ',')) { - // we have a single descendent or a last child (i.e. ",)" scenario) - count += 3; - structure.push_back(true); - structure.push_back(false); - structure.push_back(false); - potential_single_descendent = false; - } else { - // it is possible still to have a single descendent in the case of - // multiple single descendents (e.g., (...()...) ) - count += 1; - structure.push_back(false); - } - last_structure = *c; - break; - case ',': - if(last_structure != ')') { - // we have a new tip - count += 2; - structure.push_back(true); - structure.push_back(false); - } - potential_single_descendent = false; - last_structure = *c; - break; - default: - break; - } - } - - Rcpp::LogicalVector bp = Rcpp::LogicalVector(structure.size()); - - for(unsigned int i = 0; i < structure.size(); i++){ - bp[i] = structure[i]; - } - - return(bp); -} diff --git a/R/unifrac_cpp/task_parameters.hpp b/R/unifrac_cpp/task_parameters.hpp deleted file mode 100644 index 8f3d53277..000000000 --- a/R/unifrac_cpp/task_parameters.hpp +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include - -#ifndef __su_task_parameters - #ifdef __cplusplus - namespace su { - #endif - - /* task specific compute parameters - * - * n_samples the number of samples being processed - * start the first stride to process - * stop the last stride to process - * tid the thread identifier - * bypass_tips ignore tips on compute, reduces compute by ~50% - * g_unifrac_alpha an alpha value for generalized unifrac - */ - struct task_parameters { - uint32_t n_samples; // number of samples - unsigned int start; // starting stripe - unsigned int stop; // stopping stripe - unsigned int tid; // thread ID - bool bypass_tips; // avoid compute at tips - - // task specific arguments below - double g_unifrac_alpha; // generalized unifrac alpha - }; - - #ifdef __cplusplus - } - #endif - -#define __su_task_parameters -#endif - diff --git a/R/unifrac_cpp/test_api.cpp b/R/unifrac_cpp/test_api.cpp deleted file mode 100644 index 11304ae05..000000000 --- a/R/unifrac_cpp/test_api.cpp +++ /dev/null @@ -1,744 +0,0 @@ -#include -#include "api.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * test harness adapted from - * https://github.com/noporpoise/BitArray/blob/master/dev/bit_array_test.c - */ - -const char *suite_name; -char suite_pass; -int suites_run = 0, suites_failed = 0, suites_empty = 0; -int tests_in_suite = 0, tests_run = 0, tests_failed = 0; - -#define QUOTE(str) #str -#define ASSERT(x) {tests_run++; tests_in_suite++; if(!(x)) \ - { fprintf(stderr, "failed assert [%s:%i] %s\n", __FILE__, __LINE__, QUOTE(x)); \ - suite_pass = 0; tests_failed++; }} - -void SUITE_START(const char *name) { - suite_pass = 1; - suite_name = name; - suites_run++; - tests_in_suite = 0; -} - -void SUITE_END() { - printf("Testing %s ", suite_name); - size_t suite_i; - for(suite_i = strlen(suite_name); suite_i < 80-8-5; suite_i++) printf("."); - printf("%s\n", suite_pass ? " pass" : " fail"); - if(!suite_pass) suites_failed++; - if(!tests_in_suite) suites_empty++; -} -/* - * End adapted code - */ - - -//void test_write_mat() { -// SUITE_START("test write mat_t"); -// SUITE_END(); -//} -// -//void test_read_mat() { -// SUITE_START("test read mat_t"); -// SUITE_END(); -//} -// - -template -void fill_test_pm(TMat* pm, int case_id) { - pm->n_samples = 6; - pm->sample_ids = (char**)malloc(sizeof(char*) * 6); - pm->sample_ids[0] = (char*)malloc(sizeof(char) * 2); - pm->sample_ids[0][0] = 'A'; pm->sample_ids[0][1] = '\0'; - pm->sample_ids[1] = (char*)malloc(sizeof(char) * 2); - pm->sample_ids[1][0] = 'B'; pm->sample_ids[1][1] = '\0'; - pm->sample_ids[2] = (char*)malloc(sizeof(char) * 3); - pm->sample_ids[2][0] = 'C'; pm->sample_ids[2][1] = 'x'; pm->sample_ids[2][2] = '\0'; - pm->sample_ids[3] = (char*)malloc(sizeof(char) * 2); - pm->sample_ids[3][0] = 'D'; pm->sample_ids[3][1] = '\0'; - pm->sample_ids[4] = (char*)malloc(sizeof(char) * 2); - pm->sample_ids[4][0] = 'E'; pm->sample_ids[4][1] = '\0'; - pm->sample_ids[5] = (char*)malloc(sizeof(char) * 2); - pm->sample_ids[5][0] = 'F'; pm->sample_ids[5][1] = '\0'; - - if (case_id==0) { - pm->stripe_start = 0; - pm->stripe_stop = 3; - pm->stripe_total = 3; - pm->stripes = (TReal**)malloc(sizeof(TReal*) * 3); - pm->stripes[0] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[0][0] = 1; pm->stripes[0][1] = 2; pm->stripes[0][2] = 3; pm->stripes[0][3] = 4; pm->stripes[0][4] = 5; pm->stripes[0][5] = 6; - pm->stripes[1] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[1][0] = 7; pm->stripes[1][1] = 8; pm->stripes[1][2] = 9; pm->stripes[1][3] = 10; pm->stripes[1][4] = 11; pm->stripes[1][5] = 12; - pm->stripes[2] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[2][0] = 13; pm->stripes[2][1] = 14; pm->stripes[2][2] = 15; pm->stripes[2][3] = 16; pm->stripes[2][4] = 17; pm->stripes[2][5] = 18; - } else if (case_id==1) { - pm->stripe_start = 0; - pm->stripe_stop = 2; - pm->stripe_total = 3; - pm->stripes = (TReal**)malloc(sizeof(TReal*) * 2); - pm->stripes[0] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[0][0] = 1; pm->stripes[0][1] = 2; pm->stripes[0][2] = 3; pm->stripes[0][3] = 4; pm->stripes[0][4] = 5; pm->stripes[0][5] = 6; - pm->stripes[1] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[1][0] = 7; pm->stripes[1][1] = 8; pm->stripes[1][2] = 9; pm->stripes[1][3] = 10; pm->stripes[1][4] = 11; pm->stripes[1][5] = 12; - } else { // assume 2 - pm->stripe_start = 2; - pm->stripe_stop = 3; - pm->stripe_total = 3; - pm->stripes = (TReal**)malloc(sizeof(TReal*) * 1); - pm->stripes[0] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[0][0] = 16; pm->stripes[0][1] = 17; pm->stripes[0][2] = 18; pm->stripes[0][3] = 16; pm->stripes[0][4] = 17; pm->stripes[0][5] = 18; - } - pm->is_upper_triangle = true; -} - -partial_mat_t* make_test_pm(int case_id) { - partial_mat_t* pm = (partial_mat_t*)malloc(sizeof(partial_mat_t)); - - fill_test_pm(pm,case_id); - return pm; -} - -partial_dyn_mat_t* make_test_pdm(int case_id) { - partial_dyn_mat_t* pm = (partial_dyn_mat_t*)malloc(sizeof(partial_dyn_mat_t)); - fill_test_pm(pm,case_id); - pm->offsets = (uint64_t*)calloc(pm->stripe_stop-pm->stripe_start,sizeof(uint64_t)); - pm->filename = strdup("dummy"); - - return pm; -} - -mat_t* mat_three_rep() { - mat_t* res = (mat_t*)malloc(sizeof(mat_t)); - res->n_samples = 6; - res->cf_size = 15; - res->is_upper_triangle = true; - res->condensed_form = (double*)malloc(sizeof(double) * 15); - // using second half of third stripe. the last stripe when operating on even numbers of samples is normally redundant with the first half, - // but that was more annoying in to write up in the tests. - res->condensed_form[0] = 1; res->condensed_form[1] = 7; res->condensed_form[2] = 16; res->condensed_form[3] = 11; res->condensed_form[4] = 6; - res->condensed_form[5] = 2; res->condensed_form[6] = 8; res->condensed_form[7] = 17; res->condensed_form[8] = 12; - res->condensed_form[9] = 3; res->condensed_form[10] = 9; res->condensed_form[11] = 18; - res->condensed_form[12] = 4; res->condensed_form[13] = 10; - res->condensed_form[14] = 5; - res->sample_ids = (char**)malloc(sizeof(char*) * 6); - res->sample_ids[0] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[0][0] = 'A'; res->sample_ids[0][1] = '\0'; - res->sample_ids[1] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[1][0] = 'B'; res->sample_ids[1][1] = '\0'; - res->sample_ids[2] = (char*)malloc(sizeof(char) * 3); - res->sample_ids[2][0] = 'C'; res->sample_ids[2][1] = 'x'; res->sample_ids[2][2] = '\0'; - res->sample_ids[3] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[3][0] = 'D'; res->sample_ids[3][1] = '\0'; - res->sample_ids[4] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[4][0] = 'E'; res->sample_ids[4][1] = '\0'; - res->sample_ids[5] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[5][0] = 'F'; res->sample_ids[5][1] = '\0'; - - return res; -} - -template -TMat* mat_full_three_rep() { - TMat* res = (TMat*)malloc(sizeof(TMat)); - res->n_samples = 6; - res->flags=0; - res->matrix = (TReal*)malloc(sizeof(TReal) * 36); - TReal * m=res->matrix ; - m[ 0] = 0; m[ 1] = 1; m[ 2] = 7; m[ 3] = 16; m[ 4] = 11; m[ 5] = 6; - m[ 6] = 1; m[ 7] = 0; m[ 8] = 2; m[ 9] = 8; m[10] = 17; m[11] = 12; - m[12] = 7; m[13] = 2; m[14] = 0; m[15] = 3; m[16] = 9; m[17] = 18; - m[18] = 16; m[19] = 8; m[20] = 3; m[21] = 0; m[22] = 4; m[23] = 10; - m[24] = 11; m[25] = 17; m[26] = 9; m[27] = 4; m[28] = 0; m[29] = 5; - m[30] = 6; m[31] = 12; m[32] = 18; m[33] = 10; m[34] = 5; m[35] = 0; - - res->sample_ids = (char**)malloc(sizeof(char*) * 6); - res->sample_ids[0] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[0][0] = 'A'; res->sample_ids[0][1] = '\0'; - res->sample_ids[1] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[1][0] = 'B'; res->sample_ids[1][1] = '\0'; - res->sample_ids[2] = (char*)malloc(sizeof(char) * 3); - res->sample_ids[2][0] = 'C'; res->sample_ids[2][1] = 'x'; res->sample_ids[2][2] = '\0'; - res->sample_ids[3] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[3][0] = 'D'; res->sample_ids[3][1] = '\0'; - res->sample_ids[4] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[4][0] = 'E'; res->sample_ids[4][1] = '\0'; - res->sample_ids[5] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[5][0] = 'F'; res->sample_ids[5][1] = '\0'; - - return res; -} - -void test_read_write_partial_mat() { - SUITE_START("test read/write partial_mat_t"); - - partial_mat_t* pm = make_test_pm(0); - - io_status err = write_partial("/tmp/ssu_io.dat", pm); - ASSERT(err == write_okay); - - { - partial_mat_t *obs = NULL; - err = read_partial("/tmp/ssu_io.dat", &obs); - - ASSERT(err == read_okay); - ASSERT(obs->n_samples == 6); - ASSERT(obs->stripe_start == 0); - ASSERT(obs->stripe_stop == 3); - ASSERT(obs->stripe_total == 3); - ASSERT(strcmp(obs->sample_ids[0], "A") == 0); - ASSERT(strcmp(obs->sample_ids[1], "B") == 0); - ASSERT(strcmp(obs->sample_ids[2], "Cx") == 0); - ASSERT(strcmp(obs->sample_ids[3], "D") == 0); - ASSERT(strcmp(obs->sample_ids[4], "E") == 0); - ASSERT(strcmp(obs->sample_ids[5], "F") == 0); - - for(int i = 0; i < 3; i++) { - for(int j = 0; j < 6; j++) { - ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); - } - } - - destroy_partial_mat(&obs); - } - - { - partial_dyn_mat_t *obs = NULL; - err = read_partial_header("/tmp/ssu_io.dat", &obs); - - ASSERT(err == read_okay); - ASSERT(obs->n_samples == 6); - ASSERT(obs->stripe_start == 0); - ASSERT(obs->stripe_stop == 3); - ASSERT(obs->stripe_total == 3); - ASSERT(strcmp(obs->sample_ids[0], "A") == 0); - ASSERT(strcmp(obs->sample_ids[1], "B") == 0); - ASSERT(strcmp(obs->sample_ids[2], "Cx") == 0); - ASSERT(strcmp(obs->sample_ids[3], "D") == 0); - ASSERT(strcmp(obs->sample_ids[4], "E") == 0); - ASSERT(strcmp(obs->sample_ids[5], "F") == 0); - - for(int i = 0; i < 3; i++) { - ASSERT(obs->stripes[i]==NULL); - } - - err = read_partial_one_stripe(obs,1); - ASSERT(err == read_okay); - - ASSERT(obs->stripes[0]==NULL); - ASSERT(obs->stripes[1]!=NULL); - ASSERT(obs->stripes[2]==NULL); - - { - const int i = 1; - for(int j = 0; j < 6; j++) { - ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); - } - } - - err = read_partial_one_stripe(obs,0); - ASSERT(err == read_okay); - - ASSERT(obs->stripes[0]!=NULL); - ASSERT(obs->stripes[1]!=NULL); - ASSERT(obs->stripes[2]==NULL); - - { - const int i = 0; - for(int j = 0; j < 6; j++) { - ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); - } - } - - err = read_partial_one_stripe(obs,2); - ASSERT(err == read_okay); - - - for(int i = 0; i < 3; i++) { - ASSERT(obs->stripes[i]!=NULL); - for(int j = 0; j < 6; j++) { - ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); - } - } - - destroy_partial_dyn_mat(&obs); - } - - unlink("/tmp/ssu_io.dat"); - - SUITE_END(); -} - -void test_merge_partial_mat() { - SUITE_START("test merge partial_mat_t"); - - // the easy test - partial_mat_t* pm1 = make_test_pm(1); - partial_mat_t* pm2 = make_test_pm(2); - - mat_t* exp = mat_three_rep(); - - partial_mat_t* pms[2]; - pms[0] = pm1; - pms[1] = pm2; - - mat_t* obs = NULL; - merge_status err = merge_partial(pms, 2, 1, &obs); - ASSERT(err == merge_okay); - ASSERT(obs->cf_size == exp->cf_size); - ASSERT(obs->n_samples == exp->n_samples); - ASSERT(obs->is_upper_triangle == exp->is_upper_triangle); - for(unsigned int i = 0; i < obs->cf_size; i++) { - ASSERT(obs->condensed_form[i] == exp->condensed_form[i]); - } - for(unsigned int i = 0; i < obs->n_samples; i++) - ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); - - // out of order test - - pms[0] = pm2; - pms[1] = pm1; - - obs = NULL; - err = merge_partial(pms, 2, 1, &obs); - ASSERT(err == merge_okay); - ASSERT(obs->cf_size == exp->cf_size); - ASSERT(obs->n_samples == exp->n_samples); - ASSERT(obs->is_upper_triangle == exp->is_upper_triangle); - for(unsigned int i = 0; i < obs->cf_size; i++) { - ASSERT(obs->condensed_form[i] == exp->condensed_form[i]); - } - for(unsigned int i = 0; i < obs->n_samples; i++) - ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); - - // error checking - pm1->stripe_start = 0; - pm1->stripe_stop = 3; - pm1->stripe_total = 9; - pm1->is_upper_triangle = true; - - pm2->stripe_start = 3; - pm2->stripe_stop = 5; - pm2->stripe_total = 9; - pm2->is_upper_triangle = true; - - partial_mat_t* pm3 = make_test_pm(0); - pm3->stripe_start = 6; - pm3->stripe_stop = 9; - pm3->stripe_total = 9; - - partial_mat_t* pms_err[3]; - - pms_err[2] = pm1; - pms_err[0] = pm2; - pms_err[1] = pm3; - - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == incomplete_stripe_set); - - pm2->stripe_start = 2; - pm2->stripe_stop = 6; - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == stripes_overlap); - - pm2->stripe_start = 3; - pm2->sample_ids[2][0] = 'X'; - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == sample_id_consistency); - - pm2->sample_ids[2][0] = 'C'; - pm3->n_samples = 2; - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == partials_mismatch); - - pm3->n_samples = 6; - pm3->stripe_total = 12; - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == partials_mismatch); - - pm3->is_upper_triangle = false; - pm3->stripe_total = 9; - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == square_mismatch); - - SUITE_END(); -} - -void test_merge_partial_dyn_mat() { - SUITE_START("test merge partial_dyn_mat_t"); - - // the easy test - partial_dyn_mat_t* pm1 = make_test_pdm(1); - partial_dyn_mat_t* pm2 = make_test_pdm(2); - - mat_full_fp64_t* exp = mat_full_three_rep(); - - partial_dyn_mat_t* pms[2]; - pms[0] = pm1; - pms[1] = pm2; - - mat_full_fp64_t* obs = NULL; - merge_status err = merge_partial_to_matrix(pms, 2, &obs); - ASSERT(err == merge_okay); - ASSERT(obs->n_samples == exp->n_samples); - for(unsigned int i = 0; i < (obs->n_samples*obs->n_samples); i++) { - ASSERT(obs->matrix[i] == exp->matrix[i]); - } - for(unsigned int i = 0; i < obs->n_samples; i++) { - ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); - } - // out of order test - - // recreate deallocated stripes - ASSERT(pm1->stripes[0]==NULL); - ASSERT(pm1->stripes[1]==NULL); - ASSERT(pm2->stripes[0]==NULL); - destroy_partial_dyn_mat(&pm1); - destroy_partial_dyn_mat(&pm2); - - - pm1 = make_test_pdm(1); - pm2 = make_test_pdm(2); - - pms[0] = pm2; - pms[1] = pm1; - - mat_full_fp32_t* exp2 = mat_full_three_rep(); - mat_full_fp32_t *obs2 = NULL; - err = merge_partial_to_matrix_fp32(pms, 2, &obs2); - ASSERT(err == merge_okay); - ASSERT(obs2->n_samples == exp2->n_samples); - for(unsigned int i = 0; i < (obs2->n_samples*obs2->n_samples); i++) { - ASSERT(obs2->matrix[i] == exp2->matrix[i]); - } - for(unsigned int i = 0; i < obs2->n_samples; i++) - ASSERT(strcmp(obs2->sample_ids[i], exp2->sample_ids[i]) == 0); - - - ASSERT(pm2->stripes[0]==NULL); - ASSERT(pm1->stripes[0]==NULL); - ASSERT(pm1->stripes[1]==NULL); - destroy_partial_dyn_mat(&pm1); - destroy_partial_dyn_mat(&pm2); - - - pm1 = make_test_pdm(1); - pm2 = make_test_pdm(2); - - - // error checking - pm1->stripe_start = 0; - pm1->stripe_stop = 3; - pm1->stripe_total = 9; - pm1->is_upper_triangle = true; - - pm2->stripe_start = 3; - pm2->stripe_stop = 5; - pm2->stripe_total = 9; - pm2->is_upper_triangle = true; - - partial_dyn_mat_t* pm3 = make_test_pdm(0); - pm3->stripe_start = 6; - pm3->stripe_stop = 9; - pm3->stripe_total = 9; - - partial_dyn_mat_t* pms_err[3]; - - pms_err[2] = pm1; - pms_err[0] = pm2; - pms_err[1] = pm3; - - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == incomplete_stripe_set); - - pm2->stripe_start = 2; - pm2->stripe_stop = 6; - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == stripes_overlap); - - pm2->stripe_start = 3; - pm2->sample_ids[2][0] = 'X'; - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == sample_id_consistency); - - pm2->sample_ids[2][0] = 'C'; - pm3->n_samples = 2; - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == partials_mismatch); - - pm3->n_samples = 6; - pm3->stripe_total = 12; - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == partials_mismatch); - - /* - * Disable for now... not dealing properly with is_upper_triangle == false - - pm3->is_upper_triangle = false; - pm3->stripe_total = 9; - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == square_mismatch); - */ - - destroy_mat_full_fp64(&obs); - // note, we cannot cleanly destroy the partial_dyn_mat_t structures that have been hacked by hand - - SUITE_END(); -} - -void test_merge_partial_io() { - SUITE_START("test merge partial_io"); - - // the easy test - partial_mat_t* s1 = make_test_pm(1); - partial_mat_t* s2 = make_test_pm(2); - - io_status ierr; - - ierr = write_partial("/tmp/ssu_io_1.dat", s1); - ASSERT(ierr == write_okay); - - ierr = write_partial("/tmp/ssu_io_2.dat", s2); - ASSERT(ierr == write_okay); - - partial_dyn_mat_t* pm1 = NULL; - partial_dyn_mat_t* pm2 = NULL; - - ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1); - ASSERT(ierr == read_okay); - - ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2); - ASSERT(ierr == read_okay); - - mat_full_fp64_t* exp = mat_full_three_rep(); - - partial_dyn_mat_t* pms[2]; - pms[0] = pm1; - pms[1] = pm2; - - mat_full_fp64_t* obs = NULL; - merge_status err = merge_partial_to_matrix(pms, 2, &obs); - ASSERT(err == merge_okay); - ASSERT(obs->n_samples == exp->n_samples); - for(unsigned int i = 0; i < (obs->n_samples*obs->n_samples); i++) { - ASSERT(obs->matrix[i] == exp->matrix[i]); - } - for(unsigned int i = 0; i < obs->n_samples; i++) { - ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); - } - ASSERT(pm1->stripes[0]==NULL); - ASSERT(pm1->stripes[1]==NULL); - ASSERT(pm2->stripes[0]==NULL); - - destroy_mat_full_fp64(&obs); - destroy_partial_dyn_mat(&pm1); - destroy_partial_dyn_mat(&pm2); - - // out of order test - partial_dyn_mat_t* pm1b = NULL; - partial_dyn_mat_t* pm2b = NULL; - - ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1b); - ASSERT(ierr == read_okay); - - ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2b); - ASSERT(ierr == read_okay); - - pms[0] = pm2b; - pms[1] = pm1b; - - mat_full_fp32_t* exp2 = mat_full_three_rep(); - mat_full_fp32_t *obs2 = NULL; - err = merge_partial_to_matrix_fp32(pms, 2, &obs2); - ASSERT(err == merge_okay); - ASSERT(obs2->n_samples == exp2->n_samples); - for(unsigned int i = 0; i < (obs2->n_samples*obs2->n_samples); i++) { - ASSERT(obs2->matrix[i] == exp2->matrix[i]); - } - for(unsigned int i = 0; i < obs2->n_samples; i++) - ASSERT(strcmp(obs2->sample_ids[i], exp2->sample_ids[i]) == 0); - - - ASSERT(pm2b->stripes[0]==NULL); - ASSERT(pm1b->stripes[0]==NULL); - ASSERT(pm1b->stripes[1]==NULL); - - destroy_mat_full_fp32(&obs2); - destroy_partial_dyn_mat(&pm1b); - destroy_partial_dyn_mat(&pm2b); - - unlink("/tmp/ssu_io_1.dat"); - unlink("/tmp/ssu_io_2.dat"); - - SUITE_END(); -} - -void test_merge_partial_mmap() { - SUITE_START("test merge partial_mmap"); - - // the easy test - partial_mat_t* s1 = make_test_pm(1); - partial_mat_t* s2 = make_test_pm(2); - - io_status ierr; - - ierr = write_partial("/tmp/ssu_io_1.dat", s1); - ASSERT(ierr == write_okay); - - ierr = write_partial("/tmp/ssu_io_2.dat", s2); - ASSERT(ierr == write_okay); - - partial_dyn_mat_t* pm1 = NULL; - partial_dyn_mat_t* pm2 = NULL; - - ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1); - ASSERT(ierr == read_okay); - - ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2); - ASSERT(ierr == read_okay); - - mat_full_fp64_t* exp = mat_full_three_rep(); - - partial_dyn_mat_t* pms[2]; - pms[0] = pm1; - pms[1] = pm2; - - mat_full_fp32_t* obs = NULL; - merge_status err = merge_partial_to_mmap_matrix_fp32(pms, 2, "/tmp", &obs); - ASSERT(err == merge_okay); - ASSERT(obs->n_samples == exp->n_samples); - ASSERT(obs->flags != 0); - for(unsigned int i = 0; i < (obs->n_samples*obs->n_samples); i++) { - ASSERT(obs->matrix[i] == exp->matrix[i]); - } - for(unsigned int i = 0; i < obs->n_samples; i++) { - ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); - } - ASSERT(pm1->stripes[0]==NULL); - ASSERT(pm1->stripes[1]==NULL); - ASSERT(pm2->stripes[0]==NULL); - - destroy_mat_full_fp32(&obs); - destroy_partial_dyn_mat(&pm1); - destroy_partial_dyn_mat(&pm2); - - // test failure due to FS problems - - ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1); - ASSERT(ierr == read_okay); - - ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2); - ASSERT(ierr == read_okay); - - pms[0] = pm1; - pms[1] = pm2; - - - err = merge_partial_to_mmap_matrix_fp32(pms, 2, "/santa/goes/skiing", &obs); - ASSERT(err != merge_okay); - destroy_partial_dyn_mat(&pm1); - destroy_partial_dyn_mat(&pm2); - - destroy_partial_mat(&s1); - destroy_partial_mat(&s2); - - unlink("/tmp/ssu_io_1.dat"); - unlink("/tmp/ssu_io_2.dat"); - - - SUITE_END(); -} - -void test_to_file_one(const char *method) { - - static const char h5name[]="/tmp/ssu_t1.h5"; - struct stat sbuf; - - ComputeStatus urc; - int frc; - - // ensure file does not already exist - frc=stat(h5name,&sbuf); - if (frc == 0) { - unlink(h5name); - frc=stat(h5name,&sbuf); - } - ASSERT(frc != 0); - - urc=unifrac_to_file("test.biom","test.tre",h5name,method,false,1.0,false,1,"hdf5",0,NULL); - ASSERT(urc == okay); - - // first, we check it does exist - frc=stat(h5name,&sbuf); - ASSERT(frc == 0); - - { - try { - H5::H5File file(h5name, H5F_ACC_RDONLY); - H5::DataSet mds(file.openDataSet("matrix")); - H5::DataSpace dataspace(mds.getSpace()); - - ASSERT(dataspace.isSimple() == true); - ASSERT(dataspace.getSimpleExtentNdims() == 2); - - hsize_t dims[2]; - dataspace.getSimpleExtentDims(dims, NULL); - ASSERT(dims[0] == 6); - ASSERT(dims[1] == 6); - } catch(...) { - int rc=1; - ASSERT(rc == 0); // if we get here is always an error, just to get a nice message - } - } - - unlink(h5name); - -} - -void test_to_file() { - SUITE_START("test unifrac_to_file"); - - test_to_file_one("unweighted"); - test_to_file_one("unweighted_fp32"); - test_to_file_one("weighted_normalized"); - test_to_file_one("weighted_normalized_fp32"); - test_to_file_one("weighted_unnormalized"); - test_to_file_one("weighted_unnormalized_fp32"); - test_to_file_one("generalized"); - test_to_file_one("generalized_fp32"); - - SUITE_END(); -} - -int main(int argc, char** argv) { - /* one_off and partial are executed as integration tests */ - - //test_write_mat(); - //test_read_mat(); - test_read_write_partial_mat(); - test_merge_partial_mat(); - test_merge_partial_dyn_mat(); - test_merge_partial_io(); - test_merge_partial_mmap(); - test_to_file(); - - printf("\n"); - printf(" %i / %i suites failed\n", suites_failed, suites_run); - printf(" %i / %i suites empty\n", suites_empty, suites_run); - printf(" %i / %i tests failed\n", tests_failed, tests_run); - - printf("\n THE END.\n"); - - return tests_failed ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/R/unifrac_cpp/test_ska.cpp b/R/unifrac_cpp/test_ska.cpp deleted file mode 100644 index 949c9d669..000000000 --- a/R/unifrac_cpp/test_ska.cpp +++ /dev/null @@ -1,516 +0,0 @@ -#include -#include "skbio_alt.hpp" -#include -#include -#include - -#include "api.hpp" - -/* - * test harness adapted from - * https://github.com/noporpoise/BitArray/blob/master/dev/bit_array_test.c - */ -const char *suite_name; -char suite_pass; -int suites_run = 0, suites_failed = 0, suites_empty = 0; -int tests_in_suite = 0, tests_run = 0, tests_failed = 0; - -#define QUOTE(str) #str -#define ASSERT(x) {tests_run++; tests_in_suite++; if(!(x)) \ - { fprintf(stderr, "failed assert [%s:%i] %s\n", __FILE__, __LINE__, QUOTE(x)); \ - suite_pass = 0; tests_failed++; }} - -void SUITE_START(const char *name) { - suite_pass = 1; - suite_name = name; - suites_run++; - tests_in_suite = 0; -} - -void SUITE_END() { - printf("Testing %s ", suite_name); - size_t suite_i; - for(suite_i = strlen(suite_name); suite_i < 80-8-5; suite_i++) printf("."); - printf("%s\n", suite_pass ? " pass" : " fail"); - if(!suite_pass) suites_failed++; - if(!tests_in_suite) suites_empty++; -} -/* - * End adapted code - */ - - -void test_center_mat() { - SUITE_START("test center mat"); - - // unweighted unifrac of test.biom - double matrix[] = { - 0.0000000000, 0.2000000000, 0.5714285714, 0.6000000000, 0.5000000000, 0.2000000000, - 0.2000000000, 0.0000000000, 0.4285714286 ,0.6666666667, 0.6000000000, 0.3333333333, - 0.5714285714, 0.4285714286, 0.0000000000, 0.7142857143, 0.8571428571, 0.4285714286, - 0.6000000000, 0.6666666667, 0.7142857143, 0.0000000000, 0.3333333333, 0.4000000000, - 0.5000000000, 0.6000000000, 0.8571428571, 0.3333333333, 0.0000000000, 0.6000000000, - 0.2000000000, 0.3333333333, 0.4285714286, 0.4000000000, 0.6000000000, 0.0000000000}; - - const uint32_t n_samples = 6; - - double exp[] = { 0.05343726, 0.04366213, -0.0329743 , -0.07912698, -0.00495654, 0.01995843, - 0.04366213, 0.073887 , 0.04867914, -0.11112434, -0.04973167,-0.00537226, - -0.0329743 , 0.04867914, 0.20714475, -0.07737528, -0.17044974, 0.02497543, - -0.07912698, -0.11112434, -0.07737528, 0.14830877, 0.11192366, 0.00739418, - -0.00495654, -0.04973167, -0.17044974, 0.11192366, 0.18664966,-0.07343537, - 0.01995843, -0.00537226, 0.02497543, 0.00739418, -0.07343537, 0.02647959 }; - - { - double *centered = (double *) malloc(6*6*sizeof(double)); - - su::mat_to_centered(matrix, n_samples, centered); - - for(int i = 0; i < (6*6); i++) { - //printf("%i %f %f\n",i,float(centered[i]),float(exp[i])); - ASSERT(fabs(centered[i] - exp[i]) < 0.000001); - } - - free(centered); - } - - float *matrix_fp32 = (float *) malloc(6*6*sizeof(float)); - for(int i = 0; i < (6*6); i++) matrix_fp32[i] = matrix[i]; - - - { - float *centered_fp32 = (float *) malloc(6*6*sizeof(float)); - - su::mat_to_centered(matrix_fp32, n_samples, centered_fp32); - - for(int i = 0; i < (6*6); i++) { - //printf("%i %f %f\n",i,float(centered_fp32[i]),float(exp[i])); - ASSERT(fabs(centered_fp32[i] - exp[i]) < 0.000001); - } - - free(centered_fp32); - } - - free(matrix_fp32); - - - SUITE_END(); -} - -void test_pcoa() { - SUITE_START("test pcoa"); - - // unweighted unifrac of crawford.biom - double matrix[] = { - 0. , 0.71836067 , 0.71317361 , 0.69746044 , 0.62587207 , 0.72826674 - , 0.72065895 , 0.72640581 , 0.73606053, - 0.71836067 , 0. , 0.70302967 , 0.73407301 , 0.6548042 , 0.71547381 - , 0.78397813 , 0.72318399 , 0.76138933, - 0.71317361 , 0.70302967 , 0. , 0.61041275 , 0.62331299 , 0.71848305 - , 0.70416337 , 0.75258475 , 0.79249029, - 0.69746044 , 0.73407301 , 0.61041275 , 0. , 0.6439278 , 0.70052733 - , 0.69832716 , 0.77818938 , 0.72959894, - 0.62587207 , 0.6548042 , 0.62331299 , 0.6439278 , 0. , 0.75782689 - , 0.71005144 , 0.75065046 , 0.78944369, - 0.72826674 , 0.71547381 , 0.71848305 , 0.70052733 , 0.75782689 , 0. - , 0.63593642 , 0.71283615 , 0.58314638, - 0.72065895 , 0.78397813 , 0.70416337 , 0.69832716 , 0.71005144 , 0.63593642 - , 0. , 0.69200762 , 0.68972056, - 0.72640581 , 0.72318399 , 0.75258475 , 0.77818938 , 0.75065046 , 0.71283615 - , 0.69200762 , 0. , 0.71514083, - 0.73606053 , 0.76138933 , 0.79249029 , 0.72959894 , 0.78944369 , 0.58314638 - , 0.68972056 , 0.71514083 , 0. }; - - - const uint32_t n_samples = 9; - - // Test centering - - double exp2[] = { - 0.22225336 , -0.025481 , -0.03491711 , -0.02614685 , 0.01899659 , -0.05103589 - , -0.03971812 , -0.02699392 , -0.03695706, - -0.025481 , 0.24282669 , -0.01744751 , -0.04206624 , 0.0107569 , -0.03151438 - , -0.07706765 , -0.0143721 , -0.0456347, - -0.03491711 , -0.01744751 , 0.21652901 , 0.02791465 , 0.0177328 , -0.04682078 - , -0.03082866 , -0.04921529 , -0.08294711, - -0.02614685 , -0.04206624 , 0.02791465 , 0.21190401 , 0.00235833 , -0.03639361 - , -0.02904855 , -0.07112525 , -0.03739649, - 0.01899659 , 0.0107569 , 0.0177328 , 0.00235833 , 0.20745566 , -0.08039931 - , -0.03952883 , -0.05229812 , -0.08507403, - -0.05103589 , -0.03151438 , -0.04682078 , -0.03639361 , -0.08039931 , 0.20604732 - , 0.00964595 , -0.02533192 , 0.05580262, - -0.03971812 , -0.07706765 , -0.03082866 , -0.02904855 , -0.03952883 , 0.00964595 - , 0.21765972 , -0.00489531 , -0.00621856, - -0.02699392 , -0.0143721 , -0.04921529 , -0.07112525 , -0.05229812 , -0.02533192 - , -0.00489531 , 0.2514242 , -0.00719229, - -0.03695706 , -0.0456347 , -0.08294711 , -0.03739649 , -0.08507403 , 0.05580262 - , -0.00621856 , -0.00719229 , 0.24561762 }; - - double *centered = (double *) malloc(9*9*sizeof(double)); - - su::mat_to_centered(matrix, n_samples, centered); - - for(int i = 0; i < (9*9); i++) { - //printf("%i %f %f\n",i,float(centered[i]),float(exp[i])); - ASSERT(fabs(centered[i] - exp2[i]) < 0.000001); - } - - // Test eigens - - double exp3a[] = {0.45752162, 0.3260088 , 0.2791141 , 0.26296948, 0.20924533}; - double exp3b[] = { - -0.17316152, 0.17579996, 0.23301609, -0.74519625, -0.05194624, - -0.19959264, 0.53235665, -0.53370018, 0.2173474 , 0.26736004, - -0.35794942, -0.27956624, 0.01114096, 0.40488848, -0.13121464, - -0.2296467 , -0.47494333, -0.12571292, 0.02313551, -0.46916459, - -0.44501584, 0.05597451, 0.07717711, -0.15881922, 0.24594442, - 0.40335552, -0.16290597, -0.30327343, 0.03778646, 0.21664806, - 0.23769142, -0.29034629, 0.46813757, 0.13858945, 0.58624834, - 0.2407584 , 0.51300752, 0.48211607, 0.34422672, -0.42416046, - 0.52356078, -0.0693768 , -0.30890127, -0.26195855, -0.23971493}; - - { - double *eigenvalues; - double *eigenvectors; - su::find_eigens_fast(n_samples, 5, centered, eigenvalues, eigenvectors); - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(eigenvalues[i]),float(exp3a[i])); - ASSERT(fabs(eigenvalues[i] - exp3a[i]) < 0.000001); - } - - // signs may flip, that's normal - for(int i = 0; i < (5*9); i++) { - //printf("%i %f %f %f\n",i,float(eigenvectors[i]),float(exp3b[i]),float(fabs(eigenvectors[i]) - fabs(exp3b[i]))); - ASSERT( fabs(fabs(eigenvectors[i]) - fabs(exp3b[i])) < 0.000001); - } - - free(eigenvectors); - free(eigenvalues); - } - free(centered); - - - // Test PCoA (incudes the above calls - - double *exp4a = exp3a; // same eigenvals; - double exp4b[] = { - -0.11712705, 0.10037682, -0.12310531, -0.38214073, -0.02376195, - -0.13500515, 0.30396064, 0.28196047, 0.11145694, 0.12229942, - -0.24211822, -0.15962444, -0.00588591, 0.20762904, -0.06002196, - -0.15533382, -0.27117925, 0.06641571, 0.01186402, -0.21461156, - -0.30101024, 0.03195987, -0.04077363, -0.08144337, 0.1125032 , - 0.27283106, -0.09301471, 0.16022314, 0.0193771 , 0.09910206, - 0.16077529, -0.16577955, -0.24732293, 0.07106943, 0.26816958, - 0.16284981, 0.29291283, -0.25470794, 0.17652136, -0.19402517, - 0.35413832, -0.0396122 , 0.1631964 , -0.13433378, -0.10965362}; - double exp4c[] = {0.22630343, 0.16125338, 0.13805791, 0.13007231, 0.10349879}; - - { - double *eigenvalues; - double *samples; - double *proportion_explained; - - su::pcoa(matrix, n_samples, 5, eigenvalues, samples, proportion_explained); - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(eigenvalues[i]),float(exp4a[i])); - ASSERT(fabs(eigenvalues[i] - exp4a[i]) < 0.000001); - } - - // signs may flip, that's normal - for(int i = 0; i < (5*9); i++) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp4b[i]),float(fabs(samples[i]) - fabs(exp4b[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp4b[i])) < 0.000001); - } - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp4c[i])); - ASSERT(fabs(proportion_explained[i] - exp4c[i]) < 0.000001); - } - - free(eigenvalues); - free(samples); - free(proportion_explained); - } - - // Test PCoA mixed mode - { - float *eigenvalues_fp32; - float *samples_fp32; - float *proportion_explained_fp32; - - su::pcoa(matrix, n_samples, 5, eigenvalues_fp32, samples_fp32, proportion_explained_fp32); - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(eigenvalues_fp32[i]),float(exp4a[i])); - ASSERT(fabs(eigenvalues_fp32[i] - exp4a[i]) < 0.000001); - } - - // signs may flip, that's normal - for(int i = 0; i < (5*9); i++) { - //printf("%i %f %f %f\n",i,float(samples_fp32[i]),float(exp4b[i]),float(fabs(samples_fp32[i]) - fabs(exp4b[i]))); - ASSERT( fabs(fabs(samples_fp32[i]) - fabs(exp4b[i])) < 0.000001); - } - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(proportion_explained_fp32[i]),float(exp4c[i])); - ASSERT(fabs(proportion_explained_fp32[i] - exp4c[i]) < 0.000001); - } - - free(eigenvalues_fp32); - free(samples_fp32); - free(proportion_explained_fp32); - } - - // test in-place - { - double *eigenvalues; - double *samples; - double *proportion_explained; - - su::pcoa_inplace(matrix, n_samples, 5, eigenvalues, samples, proportion_explained); - // Note: matrix content has been destroyed - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(eigenvalues[i]),float(exp4a[i])); - ASSERT(fabs(eigenvalues[i] - exp4a[i]) < 0.000001); - } - - // signs may flip, that's normal - for(int i = 0; i < (5*9); i++) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp4b[i]),float(fabs(samples[i]) - fabs(exp4b[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp4b[i])) < 0.000001); - } - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp4c[i])); - ASSERT(fabs(proportion_explained[i] - exp4c[i]) < 0.000001); - } - - free(eigenvalues); - free(samples); - free(proportion_explained); - } - - SUITE_END(); -} - -void test_pcoa_big() { - SUITE_START("test pcoa big"); - - //too big to inline, use support file - FILE *fptr = fopen("test_ska_pcoa_big.dat", "r"); - ASSERT(fptr != NULL) - - unsigned int n_samples = 0; - fscanf(fptr,"# unifrac %u\n",&n_samples); - ASSERT(n_samples == 57); - - - // first 57 rows/cols of unweighted unifrac of EMP - double matrix[57*57]; - for (unsigned int i=0; i<(57*57); i++) - fscanf(fptr,"%lf\n",&(matrix[i])); - - unsigned int n_dims = 0; - fscanf(fptr,"# pcoa %u\n",&n_dims); - ASSERT(n_dims == 7); - - double exp1[7]; - for (unsigned int i=0; i<(7); i++) - fscanf(fptr,"%lf\n",&(exp1[i])); - - double exp2[7*57]; - for (unsigned int i=0; i<(7*57); i++) - fscanf(fptr,"%lf\n",&(exp2[i])); - - double exp3[7]; - for (unsigned int i=0; i<(7); i++) - fscanf(fptr,"%lf\n",&(exp3[i])); - - fclose(fptr); - - { - double *eigenvalues; - double *samples; - double *proportion_explained; - - su::pcoa(matrix, n_samples, n_dims, eigenvalues, samples, proportion_explained); - - // last three eignes are very close to each other and could come back in reverse order - - for(unsigned int i = 0; i < n_dims ; i++) { - //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); - const double max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo - ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); - } - - // signs may flip, that's normal - for(unsigned int i = 0; i < (n_samples*n_dims); i++) { - if ((i%n_dims)<4) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.05) - } else { - // any of the 3 will do - unsigned int ibase = (i/n_dims)*n_dims; - //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); - ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); - } - } - - for(unsigned int i = 0; i < n_dims; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); - const double max_err = (i<4) ? 0.001 : 0.01; - ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); - } - - free(proportion_explained); - free(samples); - free(eigenvalues); - } - - { - float *eigenvalues; - float *samples; - float *proportion_explained; - - su::pcoa(matrix, n_samples, n_dims, eigenvalues, samples, proportion_explained); - - // last three eignes are very close to each other and could come back in reverse order - - for(unsigned int i = 0; i < n_dims ; i++) { - //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); - const float max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo - ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); - } - - // signs may flip, that's normal - for(unsigned int i = 0; i < (n_samples*n_dims); i++) { - if ((i%n_dims)<4) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.1) - } else { - // any of the 3 will do - unsigned int ibase = (i/n_dims)*n_dims; - //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); - ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); - } - } - - for(unsigned int i = 0; i < n_dims; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); - const float max_err = (i<4) ? 0.001 : 0.01; - ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); - } - - free(proportion_explained); - free(samples); - free(eigenvalues); - } - - { - float matrix_fp32[57*57]; - for (unsigned int i=0; i<(57*57); i++) - matrix_fp32[i] = matrix[i]; - - { - float *eigenvalues; - float *samples; - float *proportion_explained; - - su::pcoa(matrix_fp32, n_samples, n_dims, eigenvalues, samples, proportion_explained); - - // last three eignes are very close to each other and could come back in reverse order - - for(unsigned int i = 0; i < n_dims ; i++) { - //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); - const float max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo - ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); - } - - // signs may flip, that's normal - for(unsigned int i = 0; i < (n_samples*n_dims); i++) { - if ((i%n_dims)<4) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.1) - } else { - // any of the 3 will do - unsigned int ibase = (i/n_dims)*n_dims; - //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); - ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); - } - } - - for(unsigned int i = 0; i < n_dims; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); - const float max_err = (i<4) ? 0.001 : 0.01; - ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); - } - - free(proportion_explained); - free(samples); - free(eigenvalues); - } - - { - float *eigenvalues; - float *samples; - float *proportion_explained; - - su::pcoa_inplace(matrix_fp32, n_samples, n_dims, eigenvalues, samples, proportion_explained); - // Note: content of matrix_fp32 has been destroyed - - // last three eignes are very close to each other and could come back in reverse order - - for(unsigned int i = 0; i < n_dims ; i++) { - //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); - const float max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo - ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); - } - - // signs may flip, that's normal - for(unsigned int i = 0; i < (n_samples*n_dims); i++) { - if ((i%n_dims)<4) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.1) - } else { - // any of the 3 will do - unsigned int ibase = (i/n_dims)*n_dims; - //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); - ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); - } - } - - for(unsigned int i = 0; i < n_dims; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); - const float max_err = (i<4) ? 0.001 : 0.01; - ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); - } - - free(proportion_explained); - free(samples); - free(eigenvalues); - } - - } - - - - SUITE_END(); -} - -int main(int argc, char** argv) { - test_center_mat(); - test_pcoa(); - test_pcoa_big(); - - printf("\n"); - printf(" %i / %i suites failed\n", suites_failed, suites_run); - printf(" %i / %i suites empty\n", suites_empty, suites_run); - printf(" %i / %i tests failed\n", tests_failed, tests_run); - - printf("\n THE END.\n"); - - return tests_failed ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/R/unifrac_cpp/test_su.cpp b/R/unifrac_cpp/test_su.cpp deleted file mode 100644 index 9ef3d3341..000000000 --- a/R/unifrac_cpp/test_su.cpp +++ /dev/null @@ -1,1872 +0,0 @@ -#include -#include "api.hpp" -#include "tree.hpp" -#include "biom.hpp" -#include "unifrac.hpp" -#include "unifrac_internal.hpp" -#include -#include -#include - -/* - * test harness adapted from - * https://github.com/noporpoise/BitArray/blob/master/dev/bit_array_test.c - */ -const char *suite_name; -char suite_pass; -int suites_run = 0, suites_failed = 0, suites_empty = 0; -int tests_in_suite = 0, tests_run = 0, tests_failed = 0; - -#define QUOTE(str) #str -#define ASSERT(x) {tests_run++; tests_in_suite++; if(!(x)) \ - { fprintf(stderr, "failed assert [%s:%i] %s\n", __FILE__, __LINE__, QUOTE(x)); \ - suite_pass = 0; tests_failed++; }} - -void SUITE_START(const char *name) { - suite_pass = 1; - suite_name = name; - suites_run++; - tests_in_suite = 0; -} - -void SUITE_END() { - printf("Testing %s ", suite_name); - size_t suite_i; - for(suite_i = strlen(suite_name); suite_i < 80-8-5; suite_i++) printf("."); - printf("%s\n", suite_pass ? " pass" : " fail"); - if(!suite_pass) suites_failed++; - if(!tests_in_suite) suites_empty++; -} -/* - * End adapted code - */ - -std::vector _bool_array_to_vector(bool *arr, unsigned int n) { - std::vector vec; - - for(unsigned int i = 0; i < n; i++) - vec.push_back(arr[i]); - - return vec; -} - -std::vector _uint32_array_to_vector(uint32_t *arr, unsigned int n) { - std::vector vec; - - for(unsigned int i = 0; i < n; i++) - vec.push_back(arr[i]); - - return vec; -} - -std::vector _double_array_to_vector(double *arr, unsigned int n) { - std::vector vec; - - for(unsigned int i = 0; i < n; i++) - vec.push_back(arr[i]); - - return vec; -} - -std::vector _string_array_to_vector(std::string *arr, unsigned int n) { - std::vector vec; - - for(unsigned int i = 0; i < n; i++) - vec.push_back(arr[i]); - - return vec; -} - -bool vec_almost_equal(std::vector a, std::vector b) { - if(a.size() != b.size()) { - return false; - } - for(unsigned int i = 0; i < a.size(); i++) { - if(!(fabs(a[i] - b[i]) < 0.000001)) { // sufficient given the tests - return false; - } - } - return true; -} - - -void test_bptree_constructor_simple() { - SUITE_START("bptree constructor simple"); - //01234567 - //11101000 - su::BPTree tree = su::BPTree("(('123:foo; bar':1,b:2)c);"); - - unsigned int exp_nparens = 8; - - std::vector exp_structure; - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(false); - exp_structure.push_back(true); - exp_structure.push_back(false); - exp_structure.push_back(false); - exp_structure.push_back(false); - - std::vector exp_openclose; - exp_openclose.push_back(7); - exp_openclose.push_back(6); - exp_openclose.push_back(3); - exp_openclose.push_back(2); - exp_openclose.push_back(5); - exp_openclose.push_back(4); - exp_openclose.push_back(1); - exp_openclose.push_back(0); - - std::vector exp_names; - exp_names.push_back(std::string()); - exp_names.push_back(std::string("c")); - exp_names.push_back(std::string("123:foo; bar")); - exp_names.push_back(std::string()); - exp_names.push_back(std::string("b")); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - - std::vector exp_lengths; - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(1.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(2.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(tree.get_openclose() == exp_openclose); - ASSERT(tree.lengths == exp_lengths); - ASSERT(tree.names == exp_names); - - SUITE_END(); -} - -void test_bptree_constructor_from_existing() { - SUITE_START("bptree constructor from_existing"); - //01234567 - //11101000 - su::BPTree existing = su::BPTree("(('123:foo; bar':1,b:2)c);"); - su::BPTree tree = su::BPTree(existing.get_structure(), existing.lengths, existing.names); - - unsigned int exp_nparens = 8; - - std::vector exp_structure; - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(false); - exp_structure.push_back(true); - exp_structure.push_back(false); - exp_structure.push_back(false); - exp_structure.push_back(false); - - std::vector exp_openclose; - exp_openclose.push_back(7); - exp_openclose.push_back(6); - exp_openclose.push_back(3); - exp_openclose.push_back(2); - exp_openclose.push_back(5); - exp_openclose.push_back(4); - exp_openclose.push_back(1); - exp_openclose.push_back(0); - - std::vector exp_names; - exp_names.push_back(std::string()); - exp_names.push_back(std::string("c")); - exp_names.push_back(std::string("123:foo; bar")); - exp_names.push_back(std::string()); - exp_names.push_back(std::string("b")); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - - std::vector exp_lengths; - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(1.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(2.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(tree.get_openclose() == exp_openclose); - ASSERT(tree.lengths == exp_lengths); - ASSERT(tree.names == exp_names); - - SUITE_END(); -} - -void test_bptree_mask() { - SUITE_START("bptree mask"); - //01234567 - //11101000 - //111000 - std::vector mask = {true, true, true, true, false, false, true, true}; - su::BPTree base = su::BPTree("(('123:foo; bar':1,b:2)c);"); - su::BPTree tree = base.mask(mask, base.lengths); - unsigned int exp_nparens = 6; - - std::vector exp_structure; - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(false); - exp_structure.push_back(false); - exp_structure.push_back(false); - - std::vector exp_openclose; - exp_openclose.push_back(5); - exp_openclose.push_back(4); - exp_openclose.push_back(3); - exp_openclose.push_back(2); - exp_openclose.push_back(1); - exp_openclose.push_back(0); - - std::vector exp_names; - exp_names.push_back(std::string()); - exp_names.push_back(std::string("c")); - exp_names.push_back(std::string("123:foo; bar")); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - - std::vector exp_lengths; - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(1.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(tree.get_openclose() == exp_openclose); - ASSERT(tree.lengths == exp_lengths); - ASSERT(tree.names == exp_names); - - SUITE_END(); -} - -void test_bptree_constructor_single_descendent() { - SUITE_START("bptree constructor single descendent"); - - su::BPTree tree = su::BPTree("(((a)b)c,((d)e)f,g)r;"); - - unsigned int exp_nparens = 16; - - bool structure_arr[] = {1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0}; - std::vector exp_structure = _bool_array_to_vector(structure_arr, exp_nparens); - - double length_arr[] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; - std::vector exp_lengths = _double_array_to_vector(length_arr, exp_nparens); - - std::string names_arr[] = {"r", "c", "b", "a", "", "", "", "f", "e", "d", "", "", "", "g", "", ""}; - std::vector exp_names = _string_array_to_vector(names_arr, exp_nparens); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(vec_almost_equal(tree.lengths, exp_lengths)); - ASSERT(tree.names == exp_names); - - SUITE_END(); -} - -void test_bptree_constructor_complex() { - SUITE_START("bp tree constructor complex"); - su::BPTree tree = su::BPTree("(((a:1,b:2.5)c:6,d:8,(e),(f,g,(h:1,i:2)j:1)k:1.2)l,m:2)r;"); - - unsigned int exp_nparens = 30; - - bool structure_arr[] = {1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0}; - std::vector exp_structure = _bool_array_to_vector(structure_arr, exp_nparens); - - double length_arr[] = {0, 0, 6, 1, 0, 2.5, 0, 0, 8, 0, 0, 0, 0, 0, 1.2, 0, 0, 0, 0, 1, 1, 0, 2, 0, 0, 0, 0, 2, 0, 0}; - std::vector exp_lengths = _double_array_to_vector(length_arr, exp_nparens); - - std::string names_arr[] = {"r", "l", "c", "a", "", "b", "", "", "d", "", "", "e", "", "", "k", "f", "", "g", "", "j", "h", "", "i", "", "", "", "", "m", "", ""}; - std::vector exp_names = _string_array_to_vector(names_arr, exp_nparens); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(vec_almost_equal(tree.lengths, exp_lengths)); - ASSERT(tree.names == exp_names); - SUITE_END(); -} - -void test_bptree_constructor_semicolon() { - SUITE_START("bp tree constructor semicolon"); - su::BPTree tree = su::BPTree("((a,(b,c):5)'d','e; foo':10,((f))g)r;"); - - unsigned int exp_nparens = 20; - - bool structure_arr[] = {1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0}; - std::vector exp_structure = _bool_array_to_vector(structure_arr, exp_nparens); - - double length_arr[] = {0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0}; - std::vector exp_lengths = _double_array_to_vector(length_arr, exp_nparens); - - std::string names_arr[] = {"r", "d", "a", "", "", "b", "", "c", "", "", "", "e; foo", "", "g", "", "f", "", "", "", ""}; - std::vector exp_names = _string_array_to_vector(names_arr, exp_nparens); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(vec_almost_equal(tree.lengths, exp_lengths)); - ASSERT(tree.names == exp_names); - SUITE_END(); -} - -void test_bptree_constructor_edgecases() { - SUITE_START("bp tree constructor edgecases"); - - su::BPTree tree1 = su::BPTree("((a,b));"); - bool structure_arr1[] = {1, 1, 1, 0, 1, 0, 0, 0}; - std::vector exp_structure1 = _bool_array_to_vector(structure_arr1, 8); - - su::BPTree tree2 = su::BPTree("(a);"); - bool structure_arr2[] = {1, 1, 0, 0}; - std::vector exp_structure2 = _bool_array_to_vector(structure_arr2, 4); - - su::BPTree tree3 = su::BPTree("();"); - bool structure_arr3[] = {1, 1, 0, 0}; - std::vector exp_structure3 = _bool_array_to_vector(structure_arr3, 4); - - su::BPTree tree4 = su::BPTree("((a,b),c);"); - bool structure_arr4[] = {1, 1, 1, 0, 1, 0, 0, 1, 0, 0}; - std::vector exp_structure4 = _bool_array_to_vector(structure_arr4, 10); - - su::BPTree tree5 = su::BPTree("(a,(b,c));"); - bool structure_arr5[] = {1, 1, 0, 1, 1, 0, 1, 0, 0, 0}; - std::vector exp_structure5 = _bool_array_to_vector(structure_arr5, 10); - - ASSERT(tree1.get_structure() == exp_structure1); - ASSERT(tree2.get_structure() == exp_structure2); - ASSERT(tree3.get_structure() == exp_structure3); - ASSERT(tree4.get_structure() == exp_structure4); - ASSERT(tree5.get_structure() == exp_structure5); - - SUITE_END(); -} - -void test_bptree_constructor_quoted_comma() { - SUITE_START("quoted comma bug"); - su::BPTree tree = su::BPTree("((3,'foo,bar')x,c)r;"); - std::vector exp_names = {"r", "x", "3", "", "foo,bar", "", "", "c", "", ""}; - ASSERT(exp_names.size() == tree.names.size()); - - for(unsigned int i = 0; i < tree.names.size(); i++) { - ASSERT(exp_names[i] == tree.names[i]); - } - SUITE_END(); -} - -void test_bptree_constructor_quoted_parens() { - SUITE_START("quoted parens"); - su::BPTree tree = su::BPTree("((3,'foo(b)ar')x,c)r;"); - std::vector exp_names = {"r", "x", "3", "", "foo(b)ar", "", "", "c", "", ""}; - ASSERT(exp_names.size() == tree.names.size()); - - for(unsigned int i = 0; i < tree.names.size(); i++) { - ASSERT(exp_names[i] == tree.names[i]); - } - SUITE_END(); -} -void test_bptree_postorder() { - SUITE_START("postorderselect"); - - // fig1 from https://www.dcc.uchile.cl/~gnavarro/ps/tcs16.2.pdf - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - uint32_t exp[] = {2, 4, 7, 6, 1, 11, 15, 17, 14, 13, 0}; - uint32_t obs[tree.nparens / 2]; - - for(unsigned int i = 0; i < (tree.nparens / 2); i++) - obs[i] = tree.postorderselect(i); - - std::vector exp_v = _uint32_array_to_vector(exp, tree.nparens / 2); - std::vector obs_v = _uint32_array_to_vector(obs, tree.nparens / 2); - - ASSERT(obs_v == exp_v); - SUITE_END(); -} - -void test_bptree_preorder() { - SUITE_START("preorderselect"); - - // fig1 from https://www.dcc.uchile.cl/~gnavarro/ps/tcs16.2.pdf - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - uint32_t exp[] = {0, 1, 2, 4, 6, 7, 11, 13, 14, 15, 17}; - uint32_t obs[tree.nparens / 2]; - - for(unsigned int i = 0; i < (tree.nparens / 2); i++) - obs[i] = tree.preorderselect(i); - - std::vector exp_v = _uint32_array_to_vector(exp, tree.nparens / 2); - std::vector obs_v = _uint32_array_to_vector(obs, tree.nparens / 2); - - ASSERT(obs_v == exp_v); - SUITE_END(); -} - -void test_bptree_parent() { - SUITE_START("parent"); - - // fig1 from https://www.dcc.uchile.cl/~gnavarro/ps/tcs16.2.pdf - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - uint32_t exp[] = {0, 1, 1, 1, 1, 1, 6, 6, 1, 0, 0, 0, 0, 13, 14, 14, 14, 14, 13, 0}; - - // all the -2 and +1 garbage is to avoid testing the root. - uint32_t obs[tree.nparens - 2]; - - for(int i = 0; i < (int(tree.nparens) - 2); i++) - obs[i] = tree.parent(i+1); - - std::vector exp_v = _uint32_array_to_vector(exp, tree.nparens - 2); - std::vector obs_v = _uint32_array_to_vector(obs, tree.nparens - 2); - - ASSERT(obs_v == exp_v); - SUITE_END(); -} - -void test_biom_constructor() { - SUITE_START("biom constructor"); - - su::biom table = su::biom("test.biom"); - uint32_t exp_n_samples = 6; - uint32_t exp_n_obs = 5; - - std::string sids[] = {"Sample1", "Sample2", "Sample3", "Sample4", "Sample5", "Sample6"}; - std::vector exp_sids = _string_array_to_vector(sids, exp_n_samples); - - std::string oids[] = {"GG_OTU_1", "GG_OTU_2","GG_OTU_3", "GG_OTU_4", "GG_OTU_5"}; - std::vector exp_oids = _string_array_to_vector(oids, exp_n_obs); - - uint32_t s_indptr[] = {0, 2, 5, 9, 11, 12, 15}; - std::vector exp_s_indptr = _uint32_array_to_vector(s_indptr, exp_n_samples + 1); - - uint32_t o_indptr[] = {0, 1, 6, 9, 13, 15}; - std::vector exp_o_indptr = _uint32_array_to_vector(o_indptr, exp_n_obs + 1); - - uint32_t exp_nnz = 15; - - ASSERT(table.n_samples == exp_n_samples); - ASSERT(table.n_obs == exp_n_obs); - ASSERT(table.nnz == exp_nnz); - ASSERT(table.sample_ids == exp_sids); - ASSERT(table.obs_ids == exp_oids); - ASSERT(table.sample_indptr == exp_s_indptr); - ASSERT(table.obs_indptr == exp_o_indptr); - - SUITE_END(); -} - -void test_biom_get_obs_data() { - SUITE_START("biom get obs data"); - - su::biom table = su::biom("test.biom"); - double exp0[] = {0.0, 0.0, 1.0, 0.0, 0.0, 0.0}; - std::vector exp0_vec = _double_array_to_vector(exp0, 6); - double exp1[] = {5.0, 1.0, 0.0, 2.0, 3.0, 1.0}; - std::vector exp1_vec = _double_array_to_vector(exp1, 6); - double exp2[] = {0.0, 0.0, 1.0, 4.0, 0.0, 2.0}; - std::vector exp2_vec = _double_array_to_vector(exp2, 6); - double exp3[] = {2.0, 1.0, 1.0, 0.0, 0.0, 1.0}; - std::vector exp3_vec = _double_array_to_vector(exp3, 6); - double exp4[] = {0.0, 1.0, 1.0, 0.0, 0.0, 0.0}; - std::vector exp4_vec = _double_array_to_vector(exp4, 6); - - double *out = (double*)malloc(sizeof(double) * 6); - std::vector obs_vec; - - table.get_obs_data(std::string("GG_OTU_1").c_str(), out); - obs_vec = _double_array_to_vector(out, 6); - ASSERT(vec_almost_equal(obs_vec, exp0_vec)); - - table.get_obs_data(std::string("GG_OTU_2").c_str(), out); - obs_vec = _double_array_to_vector(out, 6); - ASSERT(vec_almost_equal(obs_vec, exp1_vec)); - - table.get_obs_data(std::string("GG_OTU_3").c_str(), out); - obs_vec = _double_array_to_vector(out, 6); - ASSERT(vec_almost_equal(obs_vec, exp2_vec)); - - table.get_obs_data(std::string("GG_OTU_4").c_str(), out); - obs_vec = _double_array_to_vector(out, 6); - ASSERT(vec_almost_equal(obs_vec, exp3_vec)); - - table.get_obs_data(std::string("GG_OTU_5").c_str(), out); - obs_vec = _double_array_to_vector(out, 6); - ASSERT(vec_almost_equal(obs_vec, exp4_vec)); - - free(out); - SUITE_END(); -} - -void test_bptree_leftchild() { - SUITE_START("test bptree left child"); - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - - uint32_t exp[] = {1, 2, 0, 0, 7, 0, 0, 14, 15, 0, 0}; - std::vector structure = tree.get_structure(); - - uint32_t exp_pos = 0; - for(unsigned int i = 0; i < tree.nparens; i++) { - if(structure[i]) - ASSERT(tree.leftchild(i) == exp[exp_pos++]); - } - SUITE_END(); -} - -void test_bptree_rightchild() { - SUITE_START("test bptree right child"); - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - - uint32_t exp[] = {13, 6, 0, 0, 7, 0, 0, 14, 17, 0, 0}; - std::vector structure = tree.get_structure(); - - uint32_t exp_pos = 0; - for(unsigned int i = 0; i < tree.nparens; i++) { - if(structure[i]) - ASSERT(tree.rightchild(i) == exp[exp_pos++]); - } - SUITE_END(); -} - -void test_bptree_rightsibling() { - SUITE_START("test bptree rightsibling"); - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - - uint32_t exp[] = {0, 11, 4, 6, 0, 0, 13, 0, 0, 17, 0}; - std::vector structure = tree.get_structure(); - - uint32_t exp_pos = 0; - for(unsigned int i = 0; i < tree.nparens; i++) { - if(structure[i]) - ASSERT(tree.rightsibling(i) == exp[exp_pos++]); - } - SUITE_END(); -} - -void test_propstack_constructor() { - SUITE_START("test propstack constructor"); - su::PropStack ps(10); - // nothing to test directly... - SUITE_END(); -} - -void test_propstack_push_and_pop() { - SUITE_START("test propstack push and pop"); - su::PropStack ps(10); - - double *vec1 = ps.pop(1); - double *vec2 = ps.pop(2); - double *vec3 = ps.pop(3); - double *vec1_obs; - double *vec2_obs; - double *vec3_obs; - - ps.push(1); - ps.push(2); - ps.push(3); - - vec3_obs = ps.pop(4); - vec2_obs = ps.pop(5); - vec1_obs = ps.pop(6); - - ASSERT(vec1 == vec1_obs); - ASSERT(vec2 == vec2_obs); - ASSERT(vec3 == vec3_obs); - SUITE_END(); -} - -void test_propstack_get() { - SUITE_START("test propstack get"); - su::PropStack ps(10); - - double *vec1 = ps.pop(1); - double *vec2 = ps.pop(2); - double *vec3 = ps.pop(3); - - double *vec1_obs = ps.get(1); - double *vec2_obs = ps.get(2); - double *vec3_obs = ps.get(3); - - ASSERT(vec1 == vec1_obs); - ASSERT(vec2 == vec2_obs); - ASSERT(vec3 == vec3_obs); - SUITE_END(); -} - -void test_unifrac_set_proportions() { - SUITE_START("test unifrac set proportions"); - // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 - // ( ( ) ( ( ) ( ) ) ( ( ) ( ) ) ) - su::BPTree tree = su::BPTree("(GG_OTU_1,(GG_OTU_2,GG_OTU_3),(GG_OTU_5,GG_OTU_4));"); - su::biom table = su::biom("test.biom"); - su::PropStack ps(table.n_samples); - - double *obs = ps.pop(4); // GG_OTU_2 - double exp4[] = {0.714285714286, 0.333333333333, 0.0, 0.333333333333, 1.0, 0.25}; - set_proportions(obs, tree, 4, table, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - double exp6[] = {0.0, 0.0, 0.25, 0.666666666667, 0.0, 0.5}; - set_proportions(obs, tree, 6, table, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - double exp3[] = {0.71428571, 0.33333333, 0.25, 1.0, 1.0, 0.75}; - set_proportions(obs, tree, 3, table, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp3[i]) < 0.000001); - SUITE_END(); -} - -void test_unifrac_set_proportions_range() { - SUITE_START("test unifrac set proportions range"); - // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 - // ( ( ) ( ( ) ( ) ) ( ( ) ( ) ) ) - su::BPTree tree = su::BPTree("(GG_OTU_1,(GG_OTU_2,GG_OTU_3),(GG_OTU_5,GG_OTU_4));"); - su::biom table = su::biom("test.biom"); - - const double exp4[] = {0.714285714286, 0.333333333333, 0.0, 0.333333333333, 1.0, 0.25}; - const double exp6[] = {0.0, 0.0, 0.25, 0.666666666667, 0.0, 0.5}; - const double exp3[] = {0.71428571, 0.33333333, 0.25, 1.0, 1.0, 0.75}; - - - // first the whole table - { - su::PropStack ps(table.n_samples); - - double *obs = ps.pop(4); // GG_OTU_2 - set_proportions_range(obs, tree, 4, table, 0, table.n_samples, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - set_proportions_range(obs, tree, 6, table, 0, table.n_samples, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - set_proportions_range(obs, tree, 3, table, 0, table.n_samples, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp3[i]) < 0.000001); - } - - // beginning - { - su::PropStack ps(3); - - double *obs = ps.pop(4); // GG_OTU_2 - set_proportions_range(obs, tree, 4, table, 0, 3, ps); - for(unsigned int i = 0; i < 3; i++) - ASSERT(fabs(obs[i] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - set_proportions_range(obs, tree, 6, table, 0, 3, ps); - for(unsigned int i = 0; i < 3; i++) - ASSERT(fabs(obs[i] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - set_proportions_range(obs, tree, 3, table, 0, 3, ps); - for(unsigned int i = 0; i < 3; i++) - ASSERT(fabs(obs[i] - exp3[i]) < 0.000001); - } - - - // end - { - su::PropStack ps(4); - - double *obs = ps.pop(4); // GG_OTU_2 - set_proportions_range(obs, tree, 4, table, 2, table.n_samples, ps); - for(unsigned int i = 2; i < table.n_samples; i++) - ASSERT(fabs(obs[i-2] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - set_proportions_range(obs, tree, 6, table, 2, table.n_samples, ps); - for(unsigned int i = 2; i < table.n_samples; i++) - ASSERT(fabs(obs[i-2] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - set_proportions_range(obs, tree, 3, table, 2, table.n_samples, ps); - for(unsigned int i = 2; i < table.n_samples; i++) - ASSERT(fabs(obs[i-2] - exp3[i]) < 0.000001); - } - - - // middle - { - const unsigned int start = 1; - const unsigned int end = 4; - su::PropStack ps(end-start); - - double *obs = ps.pop(4); // GG_OTU_2 - set_proportions_range(obs, tree, 4, table, start, end, ps); - for(unsigned int i =start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - set_proportions_range(obs, tree, 6, table, start, end, ps); - for(unsigned int i = start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - set_proportions_range(obs, tree, 3, table, start, end, ps); - for(unsigned int i = start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp3[i]) < 0.000001); - } - - SUITE_END(); -} - -void test_unifrac_set_proportions_range_float() { - SUITE_START("test unifrac set proportions range float"); - // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 - // ( ( ) ( ( ) ( ) ) ( ( ) ( ) ) ) - su::BPTree tree = su::BPTree("(GG_OTU_1,(GG_OTU_2,GG_OTU_3),(GG_OTU_5,GG_OTU_4));"); - su::biom table = su::biom("test.biom"); - - const float exp4[] = {0.714285714286, 0.333333333333, 0.0, 0.333333333333, 1.0, 0.25}; - const float exp6[] = {0.0, 0.0, 0.25, 0.666666666667, 0.0, 0.5}; - const float exp3[] = {0.71428571, 0.33333333, 0.25, 1.0, 1.0, 0.75}; - - // just midle - { - const unsigned int start = 1; - const unsigned int end = 4; - su::PropStack ps(end-start); - - float *obs = ps.pop(4); // GG_OTU_2 - set_proportions_range(obs, tree, 4, table, start, end, ps); - for(unsigned int i =start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - set_proportions_range(obs, tree, 6, table, start, end, ps); - for(unsigned int i = start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - set_proportions_range(obs, tree, 3, table, start, end, ps); - for(unsigned int i = start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp3[i]) < 0.000001); - } - - SUITE_END(); -} - - - -void test_unifrac_deconvolute_stripes() { - SUITE_START("test deconvolute stripes"); - std::vector stripes; - double s1[] = {1, 1, 1, 1, 1, 1}; - double s2[] = {2, 2, 2, 2, 2, 2}; - double s3[] = {3, 3, 3, 3, 3, 3}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - - double exp[6][6] = { {0, 1, 2, 3, 2, 1}, - {1, 0, 1, 2, 3, 2}, - {2, 1, 0, 1, 2, 3}, - {3, 2, 1, 0, 1, 2}, - {2, 3, 2, 1, 0, 1}, - {1, 2, 3, 2, 1, 0} }; - double **obs = su::deconvolute_stripes(stripes, 6); - for(unsigned int i = 0; i < 6; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(exp[i][j] == obs[i][j]); - } - } - free(obs); - SUITE_END(); -} - -void test_unifrac_stripes_to_condensed_form_even() { - SUITE_START("test stripes_to_condensed_form even samples"); - std::vector stripes; - double s1[] = {0, 9, 17, 24, 30, 35, 39, 42, 44, 8}; - double s2[] = {1, 10, 18, 25, 31, 36, 40, 43, 7, 16}; - double s3[] = {2, 11, 19, 26, 32, 37, 41, 6, 15, 23}; - double s4[] = {3, 12, 20, 27, 33, 38, 5, 14, 22, 29}; - double s5[] = {4, 13, 21, 28, 34, 4, 13, 21, 28, 34}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - double exp[45] = {/* 0, */ 0, 1, 2, 3, 4, 5, 6, 7, 8, - /* *, 0, */ 9, 10, 11, 12, 13, 14, 15, 16, - /* *, *, 0, */ 17, 18, 19, 20, 21, 22, 23, - /* *, *, *, 0, */ 24, 25, 26, 27, 28, 29, - /* *, *, *, *, 0, */ 30, 31, 32, 33, 34, - /* *, *, *, *, *, 0, */ 35, 36, 37, 38, - /* *, *, *, *, *, *, 0, */ 39, 40, 41, - /* *, *, *, *, *, *, *, 0, */ 42, 43, - /* *, *, *, *, *, *, *, *, 0, */ 44}; - /* *, *, *, *, *, *, *, *, *, *, 0 */ - - double *obs = (double*)malloc(sizeof(double) * 45); - su::stripes_to_condensed_form(stripes, 10, obs, 0, 5); - for(unsigned int i = 0; i < 45; i++) { - ASSERT(exp[i] == obs[i]); - } - free(obs); - SUITE_END(); -} - -void test_unifrac_stripes_to_condensed_form_odd() { - SUITE_START("test stripes_to_condensed_form odd samples"); - std::vector stripes; - double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0}; - double s2[] = {20, 19, 18, 17, 16, 15, 14 ,13, 12, 11, 1}; - double s3[] = {21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 2}; - double s4[] = {40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 3}; - double s5[] = {41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 4}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - double exp[55] = {/* 0, */ 1, 20, 21, 40, 41, 47, 33, 29, 11, 0, - /* 1, 0, */ 2, 19, 22, 39, 42, 48, 32, 30, 1, - /*20, 2, 0, */ 3, 18, 23, 38, 43, 49, 31, 2, - /*21, 19, 3, 0, */ 4, 17, 24, 37, 44, 50, 3, - /*40, 22, 18, 4, 0, */ 5, 16, 25 ,36, 45 , 4, - /*41, 39, 23, 17, 5, 0, */ 6, 15, 26, 35, 46, - /*47, 42, 38, 24, 16, 6, 0, */ 7, 14, 27, 34, - /*33, 48, 43, 37, 25, 15, 7, 0,*/ 8, 13, 28, - /*29, 32, 49, 44, 36, 26, 14, 8, 0, */ 9, 12, - /*11, 30, 31, 50, 45, 35, 27, 13, 9, 0,*/ 10}; - /* 0, 1, 2, 3, 4, 46, 34, 28, 12, 10, 0}; */ - double *obs = (double*)malloc(sizeof(double) * 55); - su::stripes_to_condensed_form(stripes, 11, obs, 0, 5); - for(unsigned int i = 0; i < 55; i++) { - ASSERT(exp[i] == obs[i]); - } - free(obs); - SUITE_END(); -} - -void test_unifrac_stripes_to_condensed_form_odd2() { - SUITE_START("test stripes_to_condensed_form odd(2) samples"); - std::vector stripes; - double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9}; - double s2[] = {18, 17, 16, 15, 14, 13, 12 ,11, 10}; - double s3[] = {19, 20, 21, 22, 23, 24, 25, 26, 27}; - double s4[] = {36, 35, 34, 33, 32, 31, 30, 29, 28}; - double s5[] = {31, 30, 29, 28, 36, 35, 34, 33, 32}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - double exp[36] = {/* 0, */ 1, 18, 19, 36, 31, 25, 11, 9, - /* 1, 0, */ 2, 17, 20, 35, 30, 26, 10, - /*20, 2, 0, */ 3, 16, 21, 34, 29, 27, - /*21, 19, 3, 0, */ 4, 15, 22, 33, 28, - /*40, 22, 18, 4, 0, */ 5, 14, 23 ,32, - /*41, 39, 23, 17, 5, 0, */ 6, 13, 24, - /*47, 42, 38, 24, 16, 6, 0, */ 7, 12, - /*47, 42, 38, 24, 16, 6, 7, 0, */ 8}; - /* 0, 1, 2, 3, 4, 46, 34, 8, 8, 0}; */ - double *obs = (double*)malloc(sizeof(double) * 36); - su::stripes_to_condensed_form(stripes, 9, obs, 0, 5); - for(unsigned int i = 0; i < 36; i++) { - ASSERT(exp[i] == obs[i]); - } - free(obs); - SUITE_END(); -} - -class ValidatedMemoryStripes : public su::MemoryStripes { - private: - const uint32_t n_stripes; - mutable std::vector stripe_status; // 0 new, 1 allocated, 2 deallocated, 3 reallocated, 6 deallocate after rellocation - public: - ValidatedMemoryStripes(uint32_t _n_stripes, std::vector &_stripes) - : su::MemoryStripes(_stripes) - , n_stripes(_n_stripes) - , stripe_status(n_stripes) - { - for (uint32_t i=0; i2); - return out; - } - - -}; - - -void test_unifrac_stripes_to_matrix_even() { - SUITE_START("test stripes_to_matrix even samples"); - std::vector stripes; - double s1[] = {0, 9, 17, 24, 30, 35, 39, 42, 44, 8}; - double s2[] = {1, 10, 18, 25, 31, 36, 40, 43, 7, 16}; - double s3[] = {2, 11, 19, 26, 32, 37, 41, 6, 15, 23}; - double s4[] = {3, 12, 20, 27, 33, 38, 5, 14, 22, 29}; - double s5[] = {4, 13, 21, 28, 34, 4, 13, 21, 28, 34}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - // test also double to float conversion - float exp[100] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, - 0, 0, 9, 10, 11, 12, 13, 14, 15, 16, - 1, 9, 0, 17, 18, 19, 20, 21, 22, 23, - 2, 10, 17, 0, 24, 25, 26, 27, 28, 29, - 3, 11, 18, 24, 0, 30, 31, 32, 33, 34, - 4, 12, 19, 25, 30, 0, 35, 36, 37, 38, - 5, 13, 20, 26, 31, 35, 0, 39, 40, 41, - 6, 14, 21, 27, 32, 36, 39, 0, 42, 43, - 7, 15, 22, 28, 33, 37, 40, 42, 0, 44, - 8, 16, 23, 29, 34, 38, 41, 43, 44, 0}; - { - float *obs = (float*)malloc(sizeof(float) * 100); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix_fp32(vs, 10, 5, obs); - for(unsigned int i = 0; i < 100; i++) { - ASSERT(exp[i] == obs[i]); - } - - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - - free(obs); - } - - { // small tiles - float *obs = (float*)malloc(sizeof(float) * 100); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix_fp32(vs, 10, 5, obs, 4); - for(unsigned int i = 0; i < 100; i++) { - ASSERT(exp[i] == obs[i]); - } - - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - - free(obs); - } - - { // large tiles - float *obs = (float*)malloc(sizeof(float) * 100); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix_fp32(vs, 10, 5, obs, 128); - for(unsigned int i = 0; i < 100; i++) { - ASSERT(exp[i] == obs[i]); - } - - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - - free(obs); - } - - - // test also intermediate, 2-step procedure - double *obsC = (double*)malloc(sizeof(double) * 45); - su::stripes_to_condensed_form(stripes, 10, obsC, 0, 5); - - float *obs2 = (float*)malloc(sizeof(float) * 100); - su::condensed_form_to_matrix_fp32(obsC, 10, obs2); - - for(unsigned int i = 0; i < 100; i++) { - ASSERT(exp[i] == obs2[i]); - } - - free(obs2); - free(obsC); - SUITE_END(); -} - -void test_unifrac_stripes_to_matrix_odd() { - SUITE_START("test stripes_to_matrix odd samples"); - std::vector stripes; - double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0}; - double s2[] = {20, 19, 18, 17, 16, 15, 14 ,13, 12, 11, 1}; - double s3[] = {21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 2}; - double s4[] = {40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 3}; - double s5[] = {41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 4}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - double exp[121] = { 0, 1, 20, 21, 40, 41, 47, 33, 29, 11, 0, - 1, 0, 2, 19, 22, 39, 42, 48, 32, 30, 1, - 20, 2, 0, 3, 18, 23, 38, 43, 49, 31, 2, - 21, 19, 3, 0, 4, 17, 24, 37, 44, 50, 3, - 40, 22, 18, 4, 0, 5, 16, 25 ,36, 45 , 4, - 41, 39, 23, 17, 5, 0, 6, 15, 26, 35, 46, - 47, 42, 38, 24, 16, 6, 0, 7, 14, 27, 34, - 33, 48, 43, 37, 25, 15, 7, 0, 8, 13, 28, - 29, 32, 49, 44, 36, 26, 14, 8, 0, 9, 12, - 11, 30, 31, 50, 45, 35, 27, 13, 9, 0, 10, - 0, 1, 2, 3, 4, 46, 34, 28, 12, 10, 0}; - - { - double *obs = (double*)malloc(sizeof(double) * 121); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 11, 5, obs); - for(unsigned int i = 0; i < 121; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - { // small tiling - double *obs = (double*)malloc(sizeof(double) * 121); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 11, 5, obs,4); - for(unsigned int i = 0; i < 121; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - { // large tiling - double *obs = (double*)malloc(sizeof(double) * 121); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 11, 5, obs,128); - for(unsigned int i = 0; i < 121; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - - // test also intermediate, 2-step procedure - double *obsC = (double*)malloc(sizeof(double) * 55); - su::stripes_to_condensed_form(stripes, 11, obsC, 0, 5); - - double *obs2 = (double*)malloc(sizeof(double) * 121); - su::condensed_form_to_matrix(obsC, 11, obs2); - - for(unsigned int i = 0; i < 121; i++) { - ASSERT(exp[i] == obs2[i]); - } - - free(obs2); - free(obsC); - SUITE_END(); -} - -void test_unifrac_stripes_to_matrix_odd2() { - SUITE_START("test stripes_to_matrix odd(2) samples"); - std::vector stripes; - double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9}; - double s2[] = {18, 17, 16, 15, 14, 13, 12 ,11, 10}; - double s3[] = {19, 20, 21, 22, 23, 24, 25, 26, 27}; - double s4[] = {36, 35, 34, 33, 32, 31, 30, 29, 28}; - double s5[] = {31, 30, 29, 28, 36, 35, 34, 33, 32}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - double exp[81] = { 0, 1, 18, 19, 36, 31, 25, 11, 9, - 1, 0, 2, 17, 20, 35, 30, 26, 10, - 18, 2, 0, 3, 16, 21, 34, 29, 27, - 19, 17, 3, 0, 4, 15, 22, 33, 28, - 36, 20, 16, 4, 0, 5, 14, 23 ,32, - 31, 35, 21, 15, 5, 0, 6, 13, 24, - 25, 30, 34, 22, 14, 6, 0, 7, 12, - 11, 26, 29, 33, 23, 13, 7, 0, 8, - 9, 10, 27, 28, 32, 24, 12, 8, 0}; - - { - double *obs = (double*)malloc(sizeof(double) * 81); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 9, 5, obs); - for(unsigned int i = 0; i < 81; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - { // small tile - double *obs = (double*)malloc(sizeof(double) * 81); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 9, 5, obs,4); - for(unsigned int i = 0; i < 81; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - { // large tile - double *obs = (double*)malloc(sizeof(double) * 81); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 9, 5, obs,128); - for(unsigned int i = 0; i < 81; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - // test also intermediate, 2-step procedure - double *obsC = (double*)malloc(sizeof(double) * 36); - su::stripes_to_condensed_form(stripes, 9, obsC, 0, 5); - - double *obs2 = (double*)malloc(sizeof(double) * 81); - su::condensed_form_to_matrix(obsC, 9, obs2); - - for(unsigned int i = 0; i < 81; i++) { - ASSERT(exp[i] == obs2[i]); - } - - free(obs2); - free(obsC); - SUITE_END(); -} - - -void test_unnormalized_weighted_unifrac() { - SUITE_START("test unnormalized weighted unifrac"); - - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - std::vector exp; - double stride1[] = {1.52380952, 1.25, 2.75, 1.33333333, 2., 1.07142857}; - double stride2[] = {2.17857143, 2.66666667, 3.25, 1.0, 1.14285714, 1.83333333}; - double stride3[] = {1.9047619, 2.66666667, 1.75, 1.9047619, 2.66666667, 1.75}; - exp.push_back(stride1); - exp.push_back(stride2); - exp.push_back(stride3); - std::vector strides = su::make_strides(6); - std::vector strides_total = su::make_strides(6); - - su::task_parameters task_p; - task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = false; - - std::vector tasks; - tasks.push_back(task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::weighted_unnormalized, - false, - std::ref(strides), - std::ref(strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); - } - free(strides[i]); - } - SUITE_END(); -} - -void test_generalized_unifrac() { - SUITE_START("test generalized unifrac"); - - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - // weighted normalized unifrac as computed above - std::vector w_exp; - double w_stride1[] = {0.38095238, 0.33333333, 0.73333333, 0.33333333, 0.5, 0.26785714}; - double w_stride2[] = {0.58095238, 0.66666667, 0.86666667, 0.25, 0.28571429, 0.45833333}; - double w_stride3[] = {0.47619048, 0.66666667, 0.46666667, 0.47619048, 0.66666667, 0.46666667}; - w_exp.push_back(w_stride1); - w_exp.push_back(w_stride2); - w_exp.push_back(w_stride3); - std::vector w_strides = su::make_strides(6); - std::vector w_strides_total = su::make_strides(6); - su::task_parameters w_task_p; - w_task_p.start = 0; w_task_p.stop = 3; w_task_p.tid = 0; w_task_p.n_samples = 6; w_task_p.bypass_tips = false; - w_task_p.g_unifrac_alpha = 1.0; - - std::vector tasks; - tasks.push_back(w_task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::generalized, - false, - std::ref(w_strides), - std::ref(w_strides_total), - std::ref(threads), - std::ref(tasks)); - - // as computed by GUniFrac v1.0 - // Sample1 Sample2 Sample3 Sample4 Sample5 Sample6 - //Sample1 0.0000000 0.4408392 0.6886965 0.7060606 0.5833333 0.3278410 - //Sample2 0.4408392 0.0000000 0.5102041 0.7500000 0.8000000 0.5208125 - //Sample3 0.6886965 0.5102041 0.0000000 0.8649351 0.9428571 0.5952381 - //Sample4 0.7060606 0.7500000 0.8649351 0.0000000 0.5000000 0.4857143 - //Sample5 0.5833333 0.8000000 0.9428571 0.5000000 0.0000000 0.7485714 - //Sample6 0.3278410 0.5208125 0.5952381 0.4857143 0.7485714 0.0000000 - std::vector d0_exp; - double d0_stride1[] = {0.4408392, 0.5102041, 0.8649351, 0.5000000, 0.7485714, 0.3278410}; - double d0_stride2[] = {0.6886965, 0.7500000, 0.9428571, 0.4857143, 0.5833333, 0.5208125}; - double d0_stride3[] = {0.7060606, 0.8000000, 0.5952381, 0.7060606, 0.8000000, 0.5952381}; - d0_exp.push_back(d0_stride1); - d0_exp.push_back(d0_stride2); - d0_exp.push_back(d0_stride3); - std::vector d0_strides = su::make_strides(6); - std::vector d0_strides_total = su::make_strides(6); - su::task_parameters d0_task_p; - d0_task_p.start = 0; d0_task_p.stop = 3; d0_task_p.tid = 0; d0_task_p.n_samples = 6; d0_task_p.bypass_tips = false; - d0_task_p.g_unifrac_alpha = 0.0; - - tasks.clear(); - tasks.push_back(d0_task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::generalized, - false, - std::ref(d0_strides), - std::ref(d0_strides_total), - std::ref(threads), - std::ref(tasks)); - - // as computed by GUniFrac v1.0 - // Sample1 Sample2 Sample3 Sample4 Sample5 Sample6 - //Sample1 0.0000000 0.4040518 0.6285560 0.5869439 0.4082483 0.2995673 - //Sample2 0.4040518 0.0000000 0.4160597 0.7071068 0.7302479 0.4860856 - //Sample3 0.6285560 0.4160597 0.0000000 0.8005220 0.9073159 0.5218198 - //Sample4 0.5869439 0.7071068 0.8005220 0.0000000 0.4117216 0.3485667 - //Sample5 0.4082483 0.7302479 0.9073159 0.4117216 0.0000000 0.6188282 - //Sample6 0.2995673 0.4860856 0.5218198 0.3485667 0.6188282 0.0000000 - std::vector d05_exp; - double d05_stride1[] = {0.4040518, 0.4160597, 0.8005220, 0.4117216, 0.6188282, 0.2995673}; - double d05_stride2[] = {0.6285560, 0.7071068, 0.9073159, 0.3485667, 0.4082483, 0.4860856}; - double d05_stride3[] = {0.5869439, 0.7302479, 0.5218198, 0.5869439, 0.7302479, 0.5218198}; - d05_exp.push_back(d05_stride1); - d05_exp.push_back(d05_stride2); - d05_exp.push_back(d05_stride3); - std::vector d05_strides = su::make_strides(6); - std::vector d05_strides_total = su::make_strides(6); - su::task_parameters d05_task_p; - d05_task_p.start = 0; d05_task_p.stop = 3; d05_task_p.tid = 0; d05_task_p.n_samples = 6; d05_task_p.bypass_tips = false; - d05_task_p.g_unifrac_alpha = 0.5; - - tasks.clear(); - tasks.push_back(d05_task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::generalized, - false, - std::ref(d05_strides), - std::ref(d05_strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(w_strides[i][j] - w_exp[i][j]) < 0.000001); - ASSERT(fabs(d0_strides[i][j] - d0_exp[i][j]) < 0.000001); - ASSERT(fabs(d05_strides[i][j] - d05_exp[i][j]) < 0.000001); - } - free(w_strides[i]); - free(d0_strides[i]); - free(d05_strides[i]); - } - SUITE_END(); -} - -void test_vaw_unifrac_weighted_normalized() { - SUITE_START("test vaw weighted normalized unifrac"); - - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - // as computed by GUniFrac, the original implementation of VAW-UniFrac - // could not be found. - // Sample1 Sample2 Sample3 Sample4 Sample5 Sample6 - //Sample1 0.0000000 0.4086040 0.6240185 0.4639481 0.2857143 0.2766318 - //Sample2 0.4086040 0.0000000 0.3798594 0.6884992 0.6807616 0.4735781 - //Sample3 0.6240185 0.3798594 0.0000000 0.7713254 0.8812897 0.5047114 - //Sample4 0.4639481 0.6884992 0.7713254 0.0000000 0.6666667 0.2709298 - //Sample5 0.2857143 0.6807616 0.8812897 0.6666667 0.0000000 0.4735991 - //Sample6 0.2766318 0.4735781 0.5047114 0.2709298 0.4735991 0.0000000 - // weighted normalized unifrac as computed above - - std::vector w_exp; - double w_stride1[] = {0.4086040, 0.3798594, 0.7713254, 0.6666667, 0.4735991, 0.2766318}; - double w_stride2[] = {0.6240185, 0.6884992, 0.8812897, 0.2709298, 0.2857143, 0.4735781}; - double w_stride3[] = {0.4639481, 0.6807616, 0.5047114, 0.4639481, 0.6807616, 0.5047114}; - w_exp.push_back(w_stride1); - w_exp.push_back(w_stride2); - w_exp.push_back(w_stride3); - std::vector w_strides = su::make_strides(6); - std::vector w_strides_total = su::make_strides(6); - su::task_parameters w_task_p; - w_task_p.start = 0; w_task_p.stop = 3; w_task_p.tid = 0; w_task_p.n_samples = 6; w_task_p.bypass_tips = false; - w_task_p.g_unifrac_alpha = 1.0; - - std::vector tasks; - tasks.push_back(w_task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::weighted_normalized, - true, - std::ref(w_strides), - std::ref(w_strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(w_strides[i][j] - w_exp[i][j]) < 0.000001); - } - free(w_strides[i]); - } - SUITE_END(); -} - - -void test_make_strides() { - SUITE_START("test make stripes"); - std::vector exp; - double stride[] = {0., 0., 0.}; - exp.push_back(stride); - exp.push_back(stride); - exp.push_back(stride); - - std::vector obs = su::make_strides(3); - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(obs[i][j] - exp[i][j]) < 0.000001); - } - free(obs[i]); - } -} - -void test_faith_pd() { - SUITE_START("test faith PD"); - - // Note this tree is binary (opposed to example below) - su::BPTree tree = su::BPTree("((GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1):2,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - // make vector of expectations from faith PD - double exp[6] = {6., 7., 8., 5., 4., 7.}; - - // run faith PD to get obs - double obs[6] = {0, 0, 0, 0, 0, 0}; - - su::faith_pd(table, tree, obs); - - // ASSERT that results = expectation - for (unsigned int i = 0; i < 6; i++){ - ASSERT(fabs(exp[i]-obs[i]) < 0.000001) - } - SUITE_END(); -} - -void test_faith_pd_shear(){ - SUITE_START("test faith PD extra OTUs in tree"); - - su::BPTree tree = su::BPTree("((GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1,GG_OTU_ex:9):1):2,(GG_OTU_5:1,GG_OTU_4:1,GG_OTU_ex2:12):1);"); - su::biom table = su::biom("test.biom"); - - // make vector of expectations from faith PD - double exp[6] = {6., 7., 8., 5., 4., 7.}; - - // run faith PD to get obs - double obs[6] = {0, 0, 0, 0, 0, 0}; - - std::unordered_set to_keep(table.obs_ids.begin(), \ - table.obs_ids.end()); \ - su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - su::faith_pd(table, tree_sheared, obs); - - // ASSERT that results = expectation - for (unsigned int i = 0; i < 6; i++){ - ASSERT(fabs(exp[i]-obs[i]) < 0.000001) - } - SUITE_END(); -} - -void test_unweighted_unifrac() { - SUITE_START("test unweighted unifrac"); - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - std::vector exp; - double stride1[] = {0.2, 0.42857143, 0.71428571, 0.33333333, 0.6, 0.2}; - double stride2[] = {0.57142857, 0.66666667, 0.85714286, 0.4, 0.5, 0.33333333}; - double stride3[] = {0.6, 0.6, 0.42857143, 0.6, 0.6, 0.42857143}; - exp.push_back(stride1); - exp.push_back(stride2); - exp.push_back(stride3); - std::vector strides = su::make_strides(6); - std::vector strides_total = su::make_strides(6); - - su::task_parameters task_p; - task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = false; - - std::vector tasks; - tasks.push_back(task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::unweighted, - false, - std::ref(strides), - std::ref(strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); - } - free(strides[i]); - } - SUITE_END(); -} - -void test_unweighted_unifrac_fast() { - SUITE_START("test unweighted unifrac no tips"); - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - std::vector exp; - double stride1[] = {0., 0., 0.5, 0., 0.5, 0.}; - double stride2[] = {0., 0.5, 0.5, 0.5, 0.5, 0.}; - double stride3[] = {0.5, 0.5, 0., 0.5, 0.5, 0.}; - exp.push_back(stride1); - exp.push_back(stride2); - exp.push_back(stride3); - std::vector strides = su::make_strides(6); - std::vector strides_total = su::make_strides(6); - - su::task_parameters task_p; - task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = true; - - std::vector tasks; - tasks.push_back(task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::unweighted, - false, - std::ref(strides), - std::ref(strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); - } - free(strides[i]); - } - SUITE_END(); -} - -void test_normalized_weighted_unifrac() { - SUITE_START("test normalized weighted unifrac"); - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - std::vector exp; - double stride1[] = {0.38095238, 0.33333333, 0.73333333, 0.33333333, 0.5, 0.26785714}; - double stride2[] = {0.58095238, 0.66666667, 0.86666667, 0.25, 0.28571429, 0.45833333}; - double stride3[] = {0.47619048, 0.66666667, 0.46666667, 0.47619048, 0.66666667, 0.46666667}; - exp.push_back(stride1); - exp.push_back(stride2); - exp.push_back(stride3); - std::vector strides = su::make_strides(6); - std::vector strides_total = su::make_strides(6); - - su::task_parameters task_p; - task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = false; - - - std::vector tasks; - tasks.push_back(task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::weighted_normalized, - false, - std::ref(strides), - std::ref(strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); - } - free(strides[i]); - } - SUITE_END(); -} - -void test_bptree_shear_simple() { - SUITE_START("test bptree shear simple"); - su::BPTree tree = su::BPTree("((3:2,4:3,(6:5)5:4)2:1,7:6,((10:9,11:10)9:8)8:7)r"); - - // simple - std::unordered_set to_keep = {"4", "6", "7", "10", "11"}; - - uint32_t exp_nparens = 20; - std::vector exp_structure = {true, true, true, false, true, true, false, false, false, true, - false, true, true, true, false, true, false, false, false, false}; - std::vector exp_names = {"r", "2", "4", "", "5", "6", "", "", "", "7", "", "8", "9", "10", "", - "11", "", "", "", ""}; - std::vector exp_lengths = {0, 1, 3, 0, 4, 5, 0, 0, 0, 6, 0, 7, 8, 9, 0, 10, 0, 0, 0, 0}; - - su::BPTree obs = tree.shear(to_keep); - ASSERT(obs.get_structure() == exp_structure); - ASSERT(exp_nparens == obs.nparens); - ASSERT(vec_almost_equal(exp_lengths, obs.lengths)); - ASSERT(obs.names == exp_names); - SUITE_END(); -} - -void test_bptree_shear_deep() { - SUITE_START("test bptree shear deep"); - su::BPTree tree = su::BPTree("((3:2,4:3,(6:5)5:4)2:1,7:6,((10:9,11:10)9:8)8:7)r"); - - // deep - std::unordered_set to_keep = {"10", "11"}; - - uint32_t exp_nparens = 10; - std::vector exp_structure = {true, true, true, true, false, true, false, false, false, false}; - std::vector exp_names = {"r", "8", "9", "10", "", "11", "", "", "", ""}; - std::vector exp_lengths = {0, 7, 8, 9, 0, 10, 0, 0, 0, 0}; - - su::BPTree obs = tree.shear(to_keep); - ASSERT(exp_nparens == obs.nparens); - ASSERT(obs.get_structure() == exp_structure); - ASSERT(vec_almost_equal(exp_lengths, obs.lengths)); - ASSERT(obs.names == exp_names); - SUITE_END(); -} - -void test_test_table_ids_are_subset_of_tree() { - SUITE_START("test test_table_ids_are_subset_of_tree"); - - su::BPTree tree = su::BPTree("(a:1,b:2)r;"); - su::biom table = su::biom("test.biom"); - std::string expected = "GG_OTU_1"; - std::string observed = su::test_table_ids_are_subset_of_tree(table, tree); - ASSERT(observed == expected); - - su::BPTree tree2 = su::BPTree("(GG_OTU_1,GG_OTU_5,GG_OTU_6,GG_OTU_2,GG_OTU_3,GG_OTU_4);"); - su::biom table2 = su::biom("test.biom"); - expected = ""; - observed = su::test_table_ids_are_subset_of_tree(table2, tree2); - ASSERT(observed == expected); - SUITE_END(); -} - - -void test_bptree_get_tip_names() { - SUITE_START("test bptree get_tip_names"); - su::BPTree tree = su::BPTree("((a:2,b:3,(c:5)d:4)e:1,f:6,((g:9,h:10)i:8)j:7)r"); - - std::unordered_set expected = {"a", "b", "c", "f", "g", "h"}; - std::unordered_set observed = tree.get_tip_names(); - ASSERT(observed == expected); - SUITE_END(); -} - -void test_bptree_collapse_simple() { - SUITE_START("test bptree collapse simple"); - su::BPTree tree = su::BPTree("((3:2,4:3,(6:5)5:4)2:1,7:6,((10:9,11:10)9:8)8:7)r"); - - uint32_t exp_nparens = 18; - std::vector exp_structure = {true, true, true, false, true, false, true, false, false, - true, false, true, true, false, true, false, false, false}; - std::vector exp_names = {"r", "2", "3", "", "4", "", "6", "", "", "7", "", "9", "10", "", "11", "", "", ""}; - std::vector exp_lengths = {0, 1, 2, 0, 3, 0, 9, 0, 0, 6, 0, 15, 9, 0, 10, 0, 0, 0}; - - su::BPTree obs = tree.collapse(); - - ASSERT(obs.get_structure() == exp_structure); - ASSERT(exp_nparens == obs.nparens); - ASSERT(vec_almost_equal(exp_lengths, obs.lengths)); - ASSERT(obs.names == exp_names); - SUITE_END(); -} - -void test_bptree_collapse_edge() { - SUITE_START("test bptree collapse edge case against root"); - - su::BPTree tree = su::BPTree("((a),b)r;"); - su::BPTree exp = su::BPTree("(a,b)r;"); - su::BPTree obs = tree.collapse(); - ASSERT(obs.get_structure() == exp.get_structure()); - ASSERT(obs.names == exp.names); - ASSERT(vec_almost_equal(obs.lengths, exp.lengths)); - - SUITE_END(); -} - -void test_unifrac_sample_counts() { - SUITE_START("test unifrac sample counts"); - su::biom table = su::biom("test.biom"); - double* obs = table.sample_counts; - double exp[] = {7, 3, 4, 6, 3, 4}; - for(unsigned int i = 0; i < 6; i++) - ASSERT(obs[i] == exp[i]); - SUITE_END(); -} - -void test_set_tasks() { - SUITE_START("test set tasks"); - std::vector obs(1); - std::vector exp(1); - - exp[0].g_unifrac_alpha = 1.0; - exp[0].n_samples = 100; - exp[0].bypass_tips = false; - exp[0].start = 0; - exp[0].stop = 100; - exp[0].tid = 0; - - set_tasks(obs, 1.0, 100, 0, 100, false, 1); - ASSERT(obs[0].g_unifrac_alpha == exp[0].g_unifrac_alpha); - ASSERT(obs[0].n_samples == exp[0].n_samples); - ASSERT(obs[0].start == exp[0].start); - ASSERT(obs[0].stop == exp[0].stop); - ASSERT(obs[0].tid == exp[0].tid); - - std::vector obs2(2); - std::vector exp2(2); - - exp2[0].g_unifrac_alpha = 1.0; - exp2[0].n_samples = 100; - exp2[0].bypass_tips = false; - exp2[0].start = 0; - exp2[0].stop = 50; - exp2[0].tid = 0; - exp2[1].g_unifrac_alpha = 1.0; - exp2[1].n_samples = 100; - exp2[1].bypass_tips = false; - exp2[1].start = 50; - exp2[1].stop = 100; - exp2[1].tid = 1; - - set_tasks(obs2, 1.0, 100, 0, 100, false, 2); - for(unsigned int i=0; i < 2; i++) { - ASSERT(obs2[i].g_unifrac_alpha == exp2[i].g_unifrac_alpha); - ASSERT(obs2[i].n_samples == exp2[i].n_samples); - ASSERT(obs2[i].start == exp2[i].start); - ASSERT(obs2[i].stop == exp2[i].stop); - ASSERT(obs2[i].tid == exp2[i].tid); - } - - std::vector obs3(3); - std::vector exp3(3); - - exp3[0].g_unifrac_alpha = 1.0; - exp3[0].n_samples = 100; - exp3[0].bypass_tips = false; - exp3[0].start = 25; - exp3[0].stop = 50; - exp3[0].tid = 0; - exp3[1].g_unifrac_alpha = 1.0; - exp3[1].n_samples = 100; - exp3[1].bypass_tips = false; - exp3[1].start = 50; - exp3[1].stop = 75; - exp3[1].tid = 1; - exp3[2].g_unifrac_alpha = 1.0; - exp3[2].n_samples = 100; - exp3[2].bypass_tips = false; - exp3[2].start = 75; - exp3[2].stop = 100; - exp3[2].tid = 2; - - set_tasks(obs3, 1.0, 100, 25, 100, false, 3); - for(unsigned int i=0; i < 3; i++) { - ASSERT(obs3[i].g_unifrac_alpha == exp3[i].g_unifrac_alpha); - ASSERT(obs3[i].n_samples == exp3[i].n_samples); - ASSERT(obs3[i].start == exp3[i].start); - ASSERT(obs3[i].stop == exp3[i].stop); - ASSERT(obs3[i].tid == exp3[i].tid); - } - - std::vector obs4(3); - std::vector exp4(3); - - exp4[0].g_unifrac_alpha = 1.0; - exp4[0].n_samples = 100; - exp4[0].bypass_tips = false; - exp4[0].start = 26; - exp4[0].stop = 51; - exp4[0].tid = 0; - exp4[1].g_unifrac_alpha = 1.0; - exp4[1].n_samples = 100; - exp4[1].bypass_tips = false; - exp4[1].start = 51; - exp4[1].stop = 76; - exp4[1].tid = 1; - exp4[2].g_unifrac_alpha = 1.0; - exp4[2].n_samples = 100; - exp4[2].bypass_tips = false; - exp4[2].start = 76; - exp4[2].stop = 100; - exp4[2].tid = 2; - - set_tasks(obs4, 1.0, 100, 26, 100, false, 3); - for(unsigned int i=0; i < 3; i++) { - ASSERT(obs4[i].g_unifrac_alpha == exp4[i].g_unifrac_alpha); - ASSERT(obs4[i].n_samples == exp4[i].n_samples); - ASSERT(obs4[i].start == exp4[i].start); - ASSERT(obs4[i].stop == exp4[i].stop); - ASSERT(obs4[i].tid == exp4[i].tid); - } - - // set_tasks boundary bug - std::vector obs16(16); - std::vector exp16(16); - set_tasks(obs16, 1.0, 9511, 0, 0, false, 16); - exp16[15].start = 4459; - exp16[15].stop = 4756; - ASSERT(obs16[15].start == exp16[15].start); - ASSERT(obs16[15].stop == exp16[15].stop); - SUITE_END(); -} - -void test_bptree_constructor_newline_bug() { - SUITE_START("test bptree constructor newline bug"); - su::BPTree tree = su::BPTree("((362be41f31fd26be95ae43a8769b91c0:0.116350803,(a16679d5a10caa9753f171977552d920:0.105836235,((a7acc2abb505c3ee177a12e514d3b994:0.008268754,(4e22aa3508b98813f52e1a12ffdb74ad:0.03144211,8139c4ac825dae48454fb4800fb87896:0.043622957)0.923:0.046588301)0.997:0.120902074,((2d3df7387323e2edcbbfcb6e56a02710:0.031543994,3f6752aabcc291b67a063fb6492fd107:0.091571442)0.759:0.016335166,((d599ebe277afb0dfd4ad3c2176afc50e:5e-09,84d0affc7243c7d6261f3a7d680b873f:0.010245188)0.883:0.048993011,51121722488d0c3da1388d1b117cd239:0.119447926)0.763:0.035660204)0.921:0.058191474)0.776:0.02854575)0.657:0.052060833)0.658:0.032547569,(99647b51f775c8ddde8ed36a7d60dbcd:0.173334268,(f18a9c8112372e2916a66a9778f3741b:0.194813398,(5833416522de0cca717a1abf720079ac:5e-09,(2bf1067d2cd4f09671e3ebe5500205ca:0.031692682,(b32621bcd86cb99e846d8f6fee7c9ab8:0.031330707,1016319c25196d73bdb3096d86a9df2f:5e-09)0.058:0.01028612)0.849:0.010284866)0.791:0.041353384)0.922:0.109470534):0.022169824000000005)root;\n\n"); - SUITE_END(); -} - -int main(int argc, char** argv) { - test_bptree_constructor_simple(); - test_bptree_constructor_newline_bug(); - test_bptree_constructor_from_existing(); - test_bptree_constructor_single_descendent(); - test_bptree_constructor_complex(); - test_bptree_constructor_semicolon(); - test_bptree_constructor_edgecases(); - test_bptree_constructor_quoted_comma(); - test_bptree_constructor_quoted_parens(); - test_bptree_postorder(); - test_bptree_preorder(); - test_bptree_parent(); - test_bptree_leftchild(); - test_bptree_rightchild(); - test_bptree_rightsibling(); - test_bptree_get_tip_names(); - test_bptree_mask(); - test_bptree_shear_simple(); - test_bptree_shear_deep(); - test_bptree_collapse_simple(); - test_bptree_collapse_edge(); - - test_biom_constructor(); - test_biom_get_obs_data(); - - test_propstack_constructor(); - test_propstack_push_and_pop(); - test_propstack_get(); - - test_unifrac_set_proportions(); - test_unifrac_set_proportions_range(); - test_unifrac_set_proportions_range_float(); - test_unifrac_deconvolute_stripes(); - test_unifrac_stripes_to_condensed_form_even(); - test_unifrac_stripes_to_condensed_form_odd(); - test_unifrac_stripes_to_condensed_form_odd2(); - test_unifrac_stripes_to_matrix_even(); - test_unifrac_stripes_to_matrix_odd(); - test_unifrac_stripes_to_matrix_odd2(); - test_unweighted_unifrac(); - test_unweighted_unifrac_fast(); - test_unnormalized_weighted_unifrac(); - test_normalized_weighted_unifrac(); - test_generalized_unifrac(); - test_vaw_unifrac_weighted_normalized(); - test_unifrac_sample_counts(); - test_set_tasks(); - test_test_table_ids_are_subset_of_tree(); - - test_faith_pd(); - test_faith_pd_shear(); - - printf("\n"); - printf(" %i / %i suites failed\n", suites_failed, suites_run); - printf(" %i / %i suites empty\n", suites_empty, suites_run); - printf(" %i / %i tests failed\n", tests_failed, tests_run); - - printf("\n THE END.\n"); - - return tests_failed ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/R/unifrac_cpp/tree.cpp b/R/unifrac_cpp/tree.cpp index 3dc6f98e7..059631c21 100644 --- a/R/unifrac_cpp/tree.cpp +++ b/R/unifrac_cpp/tree.cpp @@ -1,55 +1,77 @@ #include "tree.hpp" + #include #include +#include + using namespace su; -BPTree::BPTree(std::string newick) { - openclose = std::vector(); - lengths = std::vector(); - names = std::vector(); - excess = std::vector(); +BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names, bool rooted) { + isRooted = rooted; + + structure = input_structure; + lengths = input_lengths; + names = input_names; + + nparens = structure.size(); + openclose = std::vector(); select_0_index = std::vector(); select_1_index = std::vector(); - structure = std::vector(); - structure.reserve(500000); // a fair sized tree... avoid reallocs, and its not _that_ much waste if this is wrong - - // three pass for parse. not ideal, but easier to map from IOW code - newick_to_bp(newick); - - // resize is correct here as we are not performing a push_back openclose.resize(nparens); - lengths.resize(nparens); - names.resize(nparens); select_0_index.resize(nparens / 2); select_1_index.resize(nparens / 2); excess.resize(nparens); structure_to_openclose(); - newick_to_metadata(newick); index_and_cache(); } -BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names) { - structure = input_structure; - lengths = input_lengths; - names = input_names; +BPTree::BPTree(const Rcpp::S4 & treeSE, bool rooted) { - nparens = structure.size(); - + isRooted = rooted; + + //Initialize vectors openclose = std::vector(); + lengths = std::vector(); + names = std::vector(); + excess = std::vector(); + select_0_index = std::vector(); select_1_index = std::vector(); + + //Load the tree structure + structure = std::vector(); + structure.reserve(500000); // a fair sized tree... avoid reallocs, and its not _that_ much waste if this is wrong + const Rcpp::List & rowTree = treeSE.slot("rowTree"); + rowTree_to_bp(rowTree); //Also sets the size of nparens + + std::cout << "BP ok\n"; + + //Resize vectors + // resize is correct here as we are not performing a push_back openclose.resize(nparens); + lengths.resize(nparens); + names.resize(nparens); + excess.resize(nparens); + select_0_index.resize(nparens / 2); select_1_index.resize(nparens / 2); - excess.resize(nparens); - + + //Builds a vector that lets us find the corresponding indices for each true/false pair structure_to_openclose(); - index_and_cache(); + std::cout << "structure ok\n"; + //Get metadata + rowTree_to_metadata(rowTree); + std::cout << "metadata ok\n"; + + //Finalize + index_and_cache(); // This causes a crash for some reason + std::cout << "cache ok\n"; } + BPTree BPTree::mask(std::vector topology_mask, std::vector in_lengths) { std::vector new_structure = std::vector(); @@ -79,7 +101,7 @@ BPTree BPTree::mask(std::vector topology_mask, std::vector in_leng } } - return BPTree(new_structure, new_lengths, new_names); + return BPTree(new_structure, new_lengths, new_names, isRooted); } std::unordered_set BPTree::get_tip_names() { @@ -184,7 +206,7 @@ void BPTree::index_and_cache() { auto k1 = select_1_index.begin(); auto e_it = excess.begin(); unsigned int e = 0; - + for(; i != structure.end(); i++, idx++ ) { if(*i) { *(k1++) = idx; @@ -264,61 +286,63 @@ int32_t BPTree::bwd(uint32_t i, int d) const { return -1; } -void BPTree::newick_to_bp(std::string newick) { - char last_structure; - bool potential_single_descendent = false; - int count = 0; - bool in_quote = false; - for(auto c = newick.begin(); c != newick.end(); c++) { - if(*c == '\'') - in_quote = !in_quote; - - if(in_quote) - continue; - - switch(*c) { - case '(': - // opening of a node - count++; - structure.push_back(true); - last_structure = *c; - potential_single_descendent = true; - break; - case ')': - // closing of a node - if(potential_single_descendent || (last_structure == ',')) { - // we have a single descendent or a last child (i.e. ",)" scenario) - count += 3; - structure.push_back(true); - structure.push_back(false); - structure.push_back(false); - potential_single_descendent = false; - } else { - // it is possible still to have a single descendent in the case of - // multiple single descendents (e.g., (...()...) ) - count += 1; - structure.push_back(false); - } - last_structure = *c; - break; - case ',': - if(last_structure != ')') { - // we have a new tip - count += 2; - structure.push_back(true); - structure.push_back(false); - } - potential_single_descendent = false; - last_structure = *c; - break; - default: - break; +// The algorithms that this class uses need the tree to be stored in a binary format +// In terms of the Newick format, an opening bracket corresponds to a TRUE, a closing bracket to a FALSE, and a tip to a TRUE FALSE +// This functions assumes that the tree representation is in cladewise order - Ensure this with ape's reorder.phylo() function +// Need to check whether tree being rooted or not affects construction +// If rooted, root is by definition ntips+1 +// If unrooted, root is chosen arbitrarily? +void BPTree::rowTree_to_bp(const Rcpp::List & rowTree) { + Rcpp::List phylo = rowTree["phylo"]; + Rcpp::NumericMatrix edge = phylo["edge"]; + Rcpp::StringVector tips = phylo["tip.label"]; + + uint32_t ntips = tips.size(); // phylo tips are always numbered from 1 to number of tips; + + std::stack nodes; // Keeps track of the branch's internal nodes + + int currentNode = 0; + int nextNode = 0; + + // Goal: Insert true when a branch starts, a false when it closes, and a true-false for each tip. + + for (unsigned int i = 0; i < edge.nrow(); i++){ + currentNode = edge(i, 0); + nextNode = edge(i, 1); + + if(nodes.size() > 0 && currentNode < nodes.top()) { + // We've exhausted the branch and moved backwards in the tree + do { + nodes.pop(); + structure.push_back(false); + } while(currentNode != nodes.top()); + } + + if(nodes.size() == 0 || currentNode > nodes.top() ) { + // We are either at the root, or entering a new node + // What if the tree is unrooted? + nodes.push(currentNode); + structure.push_back(true); + + } + + if(nextNode <= ntips) { + // We've found a tip + structure.push_back(true); + structure.push_back(false); + } + + if(i == edge.nrow() - 1) { + // We've reached the end of the tree + do { + nodes.pop(); + structure.push_back(false); + } while(nodes.size() > 0); } } nparens = structure.size(); } - void BPTree::structure_to_openclose() { std::stack oc; unsigned int open_idx; @@ -335,123 +359,76 @@ void BPTree::structure_to_openclose() { } } } -// trim from end -// from http://stackoverflow.com/a/217605 -static inline std::string &rtrim(std::string &s) { - s.erase(std::find_if(s.rbegin(), s.rend(), - std::not1(std::ptr_fun(std::isspace))).base(), s.end()); - return s; -} - -//// WEIRDNESS. THIS SOLVES IT WITH THE RTRIM. ISOLATE, MOVE TO CONSTRUCTOR. -void BPTree::newick_to_metadata(std::string newick) { - newick = rtrim(newick); +//Add metadata (lengths and names) to the tree representation +//I think we can just iterate through the structure, and whenever we hit a true decide if it's a leaf or not, and then add the corresponding label/length +//edge.length has (nodes + tips) elements - leaves at the start, nodes at the end +//tip.label has (tips) elements +//root.edge and node.labels are optional, giving the length of the root and the internal node (including root) labels, respectively +void BPTree::rowTree_to_metadata(const Rcpp::List & rowTree) { + Rcpp::List phylo = rowTree["phylo"]; + Rcpp::NumericVector edgelength = phylo["edge.length"]; + Rcpp::NumericMatrix edges = phylo["edge"]; + Rcpp::StringVector tips = phylo["tip.label"]; - std::string::iterator start = newick.begin(); - std::string::iterator end = newick.end(); - std::string token; - char last_structure = '\0'; + const uint32_t n_edges = edgelength.size(); + uint32_t ntips = tips.size(); - unsigned int structure_idx = 0; - unsigned int lag = 0; - unsigned int open_idx; - - while(start != end) { - token = tokenize(start, end); - // this sucks. - if(token.length() == 1 && is_structure_character(token[0])) { - switch(token[0]) { - case '(': - structure_idx++; - break; - case ')': - case ',': - structure_idx++; - if(last_structure == ')') - lag++; - break; - } - } else { - // puts us on the corresponding closing parenthesis - structure_idx += lag; - lag = 0; + //Used to find the correct lengths for the nodes - Includes the root + std::vector edge_v(n_edges + 1, 0.0); + + for(unsigned int i = 0; i < n_edges; i++){ + edge_v.at(edges(i,1) - 1) = edgelength[i]; + } + + if(phylo.containsElementNamed("root.edge")) { + edge_v.at(ntips) = phylo["root.edge"]; + } + + bool hasNodeLabels = false; + Rcpp::StringVector nodes; + + if(phylo.containsElementNamed("node.labels")) { + hasNodeLabels = true; + nodes = phylo["node.labels"]; + } + + unsigned int tip_idx = 0; // tip indices run from 0 to ntips-1 + unsigned int node_idx = 0; // node indices run from ntips to ntips + nnodes - 1 + unsigned int edge_idx = 0; // Used to store the index of the edge for picking lengths; + + for(unsigned int i = 0; i < structure.size(); i++) { + if(structure[i]){ + std::string label = std::string(); + double length = 0.0; - open_idx = open(structure_idx); - set_node_metadata(open_idx, token); - // std::cout << structure_idx << " <-> " << open_idx << " " << token << std::endl; - // make sure to advance an extra position if we are a leaf as the - // as a leaf is by definition a 10, and doing a single advancement - // would put the structure to token mapping out of sync - if(isleaf(open_idx)) - structure_idx += 2; - else - structure_idx += 1; + if(isleaf(i)){ + //Tips can be expected to have both a length and a label + label = Rcpp::as(tips[tip_idx]); + length = edge_v[tip_idx]; + tip_idx++; + } + else{ + //Nodes always have lengths (except the root, which may have it optionally, but defaults to 0.0) + //Nodes may also optionally have labels (which includes the root label) + length = edge_v[ntips + node_idx]; + if(hasNodeLabels){ + label = Rcpp::as(nodes[node_idx]); + } + node_idx++; + } + set_node_metadata(i,label, length); } - last_structure = token[0]; } } -void BPTree::set_node_metadata(unsigned int open_idx, std::string &token) { - double length = 0.0; - std::string name = std::string(); - unsigned int colon_idx = token.find_last_of(':'); - - if(colon_idx == 0) - length = std::stof(token.substr(1)); - else if(colon_idx < token.length()) { - name = token.substr(0, colon_idx); - length = std::stof(token.substr(colon_idx + 1)); - } else - name = token; - +//This takes a label and a length and assigns them to the correct places +void BPTree::set_node_metadata(unsigned int open_idx, std::string name, double length) { names[open_idx] = name; lengths[open_idx] = length; } -inline bool BPTree::is_structure_character(char c) const { - return (c == '(' || c == ')' || c == ',' || c == ';'); -} - -std::string BPTree::tokenize(std::string::iterator &start, const std::string::iterator &end) { - bool inquote = false; - bool isquote = false; - char c; - std::string token; - - do { - c = *start; - start++; - - if(c == '\n') { - continue; - } - - isquote = c == '\''; - - if(inquote && isquote) { - inquote = false; - continue; - } else if(!inquote && isquote) { - inquote = true; - continue; - } - - if(is_structure_character(c) && !inquote) { - if(token.length() == 0) - token.push_back(c); - break; - } - - token.push_back(c); - - - } while(start != end); - - return token; -} - std::vector BPTree::get_structure() { return structure; } diff --git a/R/unifrac_cpp/tree.hpp b/R/unifrac_cpp/tree.hpp index 99f61dfec..189036244 100644 --- a/R/unifrac_cpp/tree.hpp +++ b/R/unifrac_cpp/tree.hpp @@ -16,6 +16,8 @@ #include #include +#include + namespace su { class BPTree { public: @@ -26,19 +28,20 @@ namespace su { /* total number of parentheses */ uint32_t nparens; - /* default constructor - * - * @param newick A newick string - */ - BPTree(std::string newick); - /* constructor from a defined topology * * @param input_structure A boolean vector defining the topology * @param input_lengths A vector of double of the branch lengths * @param input_names A vector of str of the vertex names */ - BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names); + BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names, bool rooted); + + /* constructor from a TreeSummarizedExperiment + * + * @param treeSE An R treeSE object + */ + BPTree(const Rcpp::S4 & treeSE, bool rooted); + ~BPTree(); /* postorder tree traversal @@ -106,6 +109,7 @@ namespace su { } std::cout << std::endl; } + BPTree mask(std::vector topology_mask, std::vector in_lengths); // mask self BPTree shear(std::unordered_set to_keep); @@ -118,16 +122,16 @@ namespace su { std::vector select_0_index; // cache of select 0 std::vector select_1_index; // cache of select 1 std::vector excess; + bool isRooted; // Is the tree rooted or not? void index_and_cache(); // construct the select caches - void newick_to_bp(std::string newick); // convert a newick string to parentheses + void rowTree_to_bp(const Rcpp::List & rowTree); // convert ape tree structure to boolean structure + void rowTree_to_metadata(const Rcpp::List & rowTree); // assign attributes void newick_to_metadata(std::string newick); // convert newick to attributes void structure_to_openclose(); // set the cache mapping between parentheses pairs - void set_node_metadata(unsigned int open_idx, std::string &token); // set attributes for a node - bool is_structure_character(char c) const; // test if a character is a newick structure + void set_node_metadata(unsigned int open_idx, std::string label, double length); // set attributes for a node inline uint32_t open(uint32_t i) const; // obtain the index of the opening for a given parenthesis inline uint32_t close(uint32_t i) const; // obtain the index of the closing for a given parenthesis - std::string tokenize(std::string::iterator &start, const std::string::iterator &end); // newick -> tokens int32_t bwd(uint32_t i, int32_t d) const; int32_t enclose(uint32_t i) const; diff --git a/R/unifrac_cpp/tree_s.cpp b/R/unifrac_cpp/tree_s.cpp deleted file mode 100644 index a8e700856..000000000 --- a/R/unifrac_cpp/tree_s.cpp +++ /dev/null @@ -1,438 +0,0 @@ -#include "tree_s.hpp" -#include -#include - -#include - -using namespace su; - -BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names, bool rooted) { - isRooted = rooted; - - structure = input_structure; - lengths = input_lengths; - names = input_names; - - nparens = structure.size(); - - openclose = std::vector(); - select_0_index = std::vector(); - select_1_index = std::vector(); - openclose.resize(nparens); - select_0_index.resize(nparens / 2); - select_1_index.resize(nparens / 2); - excess.resize(nparens); - - structure_to_openclose(); - index_and_cache(); -} - -BPTree::BPTree(const Rcpp::S4 & treeSE, bool rooted) { - - isRooted = rooted; - - //Initialize vectors - openclose = std::vector(); - lengths = std::vector(); - names = std::vector(); - excess = std::vector(); - - select_0_index = std::vector(); - select_1_index = std::vector(); - - //Load the tree structure - structure = std::vector(); - structure.reserve(500000); // a fair sized tree... avoid reallocs, and its not _that_ much waste if this is wrong - const Rcpp::List & rowTree = treeSE.slot("rowTree"); - rowTree_to_bp(rowTree); //Also sets the size of nparens - - std::cout << "BP ok\n"; - - //Resize vectors - // resize is correct here as we are not performing a push_back - openclose.resize(nparens); - lengths.resize(nparens); - names.resize(nparens); - excess.resize(nparens); - - select_0_index.resize(nparens / 2); - select_1_index.resize(nparens / 2); - - //Builds a vector that lets us find the corresponding indices for each true/false pair - structure_to_openclose(); - std::cout << "structure ok\n"; - //Get metadata - rowTree_to_metadata(rowTree); - std::cout << "metadata ok\n"; - - //Finalize - index_and_cache(); // This causes a crash for some reason - std::cout << "cache ok\n"; -} - - -BPTree BPTree::mask(std::vector topology_mask, std::vector in_lengths) { - - std::vector new_structure = std::vector(); - std::vector new_lengths = std::vector(); - std::vector new_names = std::vector(); - - uint32_t count = 0; - for(auto i = topology_mask.begin(); i != topology_mask.end(); i++) { - if(*i) - count++; - } - - new_structure.resize(count); - new_lengths.resize(count); - new_names.resize(count); - - auto mask_it = topology_mask.begin(); - auto base_it = this->structure.begin(); - uint32_t new_idx = 0; - uint32_t old_idx = 0; - for(; mask_it != topology_mask.end(); mask_it++, base_it++, old_idx++) { - if(*mask_it) { - new_structure[new_idx] = this->structure[old_idx]; - new_lengths[new_idx] = in_lengths[old_idx]; - new_names[new_idx] = this->names[old_idx]; - new_idx++; - } - } - - return BPTree(new_structure, new_lengths, new_names, isRooted); -} - -std::unordered_set BPTree::get_tip_names() { - std::unordered_set observed; - - for(unsigned int i = 0; i < this->nparens; i++) { - if(this->isleaf(i)) { - observed.insert(this->names[i]); - } - } - - return observed; -} - -BPTree BPTree::shear(std::unordered_set to_keep) { - std::vector shearmask = std::vector(this->nparens); - int32_t p; - - for(unsigned int i = 0; i < this->nparens; i++) { - if(this->isleaf(i) && to_keep.count(this->names[i]) > 0) { - shearmask[i] = true; - shearmask[i+1] = true; - - p = this->parent(i); - while(p != -1 && !shearmask[p]) { - shearmask[p] = true; - shearmask[this->close(p)] = true; - p = this->parent(p); - } - } - } - return this->mask(shearmask, this->lengths); -} - -BPTree BPTree::collapse() { - std::vector collapsemask = std::vector(this->nparens); - std::vector new_lengths = std::vector(this->lengths); - - uint32_t current, first, last; - - for(uint32_t i = 0; i < this->nparens / 2; i++) { - current = this->preorderselect(i); - - if(this->isleaf(current) or (current == 0)) { // 0 == root - collapsemask[current] = true; - collapsemask[this->close(current)] = true; - } else { - first = this->leftchild(current); - last = this->rightchild(current); - - if(first == last) { - new_lengths[first] = new_lengths[first] + new_lengths[current]; - } else { - collapsemask[current] = true; - collapsemask[this->close(current)] = true; - } - } - } - - return this->mask(collapsemask, new_lengths); -} - /* - mask = bit_array_create(self.B.size) - bit_array_set_bit(mask, self.root()) - bit_array_set_bit(mask, self.close(self.root())) - - new_lengths = self._lengths.copy() - new_lengths_ptr = new_lengths.data - - with nogil: - for i in range(n): - current = self.preorderselect(i) - - if self.isleaf(current): - bit_array_set_bit(mask, current) - bit_array_set_bit(mask, self.close(current)) - else: - first = self.fchild(current) - last = self.lchild(current) - - if first == last: - new_lengths_ptr[first] = new_lengths_ptr[first] + \ - new_lengths_ptr[current] - else: - bit_array_set_bit(mask, current) - bit_array_set_bit(mask, self.close(current)) - - new_bp = self._mask_from_self(mask, new_lengths) - bit_array_free(mask) - return new_bp -*/ - - -BPTree::~BPTree() { -} - -void BPTree::index_and_cache() { - // should probably do the open/close in here too - unsigned int idx = 0; - auto i = structure.begin(); - auto k0 = select_0_index.begin(); - auto k1 = select_1_index.begin(); - auto e_it = excess.begin(); - unsigned int e = 0; - - for(; i != structure.end(); i++, idx++ ) { - if(*i) { - *(k1++) = idx; - *(e_it++) = ++e; - } - else { - *(k0++) = idx; - *(e_it++) = --e; - } - } -} - -uint32_t BPTree::postorderselect(uint32_t k) const { - return open(select_0_index[k]); -} - -uint32_t BPTree::preorderselect(uint32_t k) const { - return select_1_index[k]; -} - -inline uint32_t BPTree::open(uint32_t i) const { - return structure[i] ? i : openclose[i]; -} - -inline uint32_t BPTree::close(uint32_t i) const { - return structure[i] ? openclose[i] : i; -} - -bool BPTree::isleaf(unsigned int idx) const { - return (structure[idx] && !structure[idx + 1]); -} - -uint32_t BPTree::leftchild(uint32_t i) const { - // aka fchild - if(isleaf(i)) - return 0; // this is awkward, using 0 which is root, but a root cannot be a child. edge case - else - return i + 1; -} - -uint32_t BPTree::rightchild(uint32_t i) const { - // aka lchild - if(isleaf(i)) - return 0; // this is awkward, using 0 which is root, but a root cannot be a child. edge case - else - return open(close(i) - 1); -} - -uint32_t BPTree::rightsibling(uint32_t i) const { - // aka nsibling - uint32_t position = close(i) + 1; - if(position >= nparens) - return 0; // will return 0 if no sibling as root cannot have a sibling - else if(structure[position]) - return position; - else - return 0; -} - -int32_t BPTree::parent(uint32_t i) const { - return enclose(i); -} - -int32_t BPTree::enclose(uint32_t i) const { - if(structure[i]) - return bwd(i, -2) + 1; - else - return bwd(i - 1, -2) + 1; -} - -int32_t BPTree::bwd(uint32_t i, int d) const { - uint32_t target_excess = excess[i] + d; - for(int current_idx = i - 1; current_idx >= 0; current_idx--) { - if(excess[current_idx] == target_excess) - return current_idx; - } - return -1; -} - -// The algorithms that this class uses need the tree to be stored in a binary format -// In terms of the Newick format, an opening bracket corresponds to a TRUE, a closing bracket to a FALSE, and a tip to a TRUE FALSE -// This functions assumes that the tree representation is in cladewise order - Ensure this with ape's reorder.phylo() function -// Need to check whether tree being rooted or not affects construction -// If rooted, root is by definition ntips+1 -// If unrooted, root is chosen arbitrarily? -void BPTree::rowTree_to_bp(const Rcpp::List & rowTree) { - Rcpp::List phylo = rowTree["phylo"]; - Rcpp::NumericMatrix edge = phylo["edge"]; - Rcpp::StringVector tips = phylo["tip.label"]; - - uint32_t ntips = tips.size(); // phylo tips are always numbered from 1 to number of tips; - - std::stack nodes; // Keeps track of the branch's internal nodes - - int currentNode = 0; - int nextNode = 0; - - // Goal: Insert true when a branch starts, a false when it closes, and a true-false for each tip. - - for (unsigned int i = 0; i < edge.nrow(); i++){ - currentNode = edge(i, 0); - nextNode = edge(i, 1); - - if(nodes.size() > 0 && currentNode < nodes.top()) { - // We've exhausted the branch and moved backwards in the tree - do { - nodes.pop(); - structure.push_back(false); - } while(currentNode != nodes.top()); - } - - if(nodes.size() == 0 || currentNode > nodes.top() ) { - // We are either at the root, or entering a new node - // What if the tree is unrooted? - nodes.push(currentNode); - structure.push_back(true); - - } - - if(nextNode <= ntips) { - // We've found a tip - structure.push_back(true); - structure.push_back(false); - } - - if(i == edge.nrow() - 1) { - // We've reached the end of the tree - do { - nodes.pop(); - structure.push_back(false); - } while(nodes.size() > 0); - } - } - nparens = structure.size(); -} - -void BPTree::structure_to_openclose() { - std::stack oc; - unsigned int open_idx; - unsigned int i = 0; - - for(auto it = structure.begin(); it != structure.end(); it++, i++) { - if(*it) { - oc.push(i); - } else { - open_idx = oc.top(); - oc.pop(); - openclose[i] = open_idx; - openclose[open_idx] = i; - } - } -} - -//Add metadata (lengths and names) to the tree representation -//I think we can just iterate through the structure, and whenever we hit a true decide if it's a leaf or not, and then add the corresponding label/length -//edge.length has (nodes + tips) elements - leaves at the start, nodes at the end -//tip.label has (tips) elements -//root.edge and node.labels are optional, giving the length of the root and the internal node (including root) labels, respectively -void BPTree::rowTree_to_metadata(const Rcpp::List & rowTree) { - Rcpp::List phylo = rowTree["phylo"]; - Rcpp::NumericVector edgelength = phylo["edge.length"]; - Rcpp::NumericMatrix edges = phylo["edge"]; - Rcpp::StringVector tips = phylo["tip.label"]; - - const uint32_t n_edges = edgelength.size(); - uint32_t ntips = tips.size(); - - //Used to find the correct lengths for the nodes - Includes the root - std::vector edge_v(n_edges + 1, 0.0); - - for(unsigned int i = 0; i < n_edges; i++){ - edge_v.at(edges(i,1) - 1) = edgelength[i]; - } - - if(phylo.containsElementNamed("root.edge")) { - edge_v.at(ntips) = phylo["root.edge"]; - } - - bool hasNodeLabels = false; - Rcpp::StringVector nodes; - - if(phylo.containsElementNamed("node.labels")) { - hasNodeLabels = true; - nodes = phylo["node.labels"]; - } - - unsigned int tip_idx = 0; // tip indices run from 0 to ntips-1 - unsigned int node_idx = 0; // node indices run from ntips to ntips + nnodes - 1 - unsigned int edge_idx = 0; // Used to store the index of the edge for picking lengths; - - for(unsigned int i = 0; i < structure.size(); i++) { - if(structure[i]){ - std::string label = std::string(); - double length = 0.0; - - if(isleaf(i)){ - //Tips can be expected to have both a length and a label - label = Rcpp::as(tips[tip_idx]); - length = edge_v[tip_idx]; - tip_idx++; - } - - else{ - //Nodes always have lengths (except the root, which may have it optionally, but defaults to 0.0) - //Nodes may also optionally have labels (which includes the root label) - length = edge_v[ntips + node_idx]; - if(hasNodeLabels){ - label = Rcpp::as(nodes[node_idx]); - } - node_idx++; - } - set_node_metadata(i,label, length); - } - } -} - -//This takes a label and a length and assigns them to the correct places -void BPTree::set_node_metadata(unsigned int open_idx, std::string name, double length) { - names[open_idx] = name; - lengths[open_idx] = length; -} - -std::vector BPTree::get_structure() { - return structure; -} - -std::vector BPTree::get_openclose() { - return openclose; -} - diff --git a/R/unifrac_cpp/tree_s.hpp b/R/unifrac_cpp/tree_s.hpp deleted file mode 100644 index 189036244..000000000 --- a/R/unifrac_cpp/tree_s.hpp +++ /dev/null @@ -1,142 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#ifndef __UNIFRAC_TREE_H -#define __UNIFRAC_TREE_H 1 - -#include -#include -#include -#include -#include - -#include - -namespace su { - class BPTree { - public: - /* tracked attributes */ - std::vector lengths; - std::vector names; - - /* total number of parentheses */ - uint32_t nparens; - - /* constructor from a defined topology - * - * @param input_structure A boolean vector defining the topology - * @param input_lengths A vector of double of the branch lengths - * @param input_names A vector of str of the vertex names - */ - BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names, bool rooted); - - /* constructor from a TreeSummarizedExperiment - * - * @param treeSE An R treeSE object - */ - BPTree(const Rcpp::S4 & treeSE, bool rooted); - - ~BPTree(); - - /* postorder tree traversal - * - * Get the index position of the ith node in a postorder tree - * traversal. - * - * @param i The ith node in a postorder traversal - */ - uint32_t postorderselect(uint32_t i)const ; - - /* preorder tree traversal - * - * Get the index position of the ith node in a preorder tree - * traversal. - * - * @param i The ith node in a preorder traversal - */ - uint32_t preorderselect(uint32_t i) const; - - /* Test if the node at an index position is a leaf - * - * @param i The node to evaluate - */ - bool isleaf(uint32_t i) const; - - /* Get the left child of a node - * - * @param i The node to obtain the left child from - */ - uint32_t leftchild(uint32_t i) const ; - - /* Get the right child of a node - * - * @param i The node to obtain the right child from - */ - uint32_t rightchild(uint32_t i) const; - - /* Get the right sibling of a node - * - * @param i The node to obtain the right sibling from - */ - uint32_t rightsibling(uint32_t i) const; - - /* Get the parent of a node - * - * @param i The node to obtain the parent of - */ - int32_t parent(uint32_t i) const; - - /* get the names at the tips of the tree */ - std::unordered_set get_tip_names(); - - /* public getters */ - std::vector get_structure(); - std::vector get_openclose(); - - /* serialize the structure as a sequence of 1s and 0s */ - void print() { - for(auto c = structure.begin(); c != structure.end(); c++) { - if(*c) - std::cout << "1"; - else - std::cout << "0"; - } - std::cout << std::endl; - } - - BPTree mask(std::vector topology_mask, std::vector in_lengths); // mask self - - BPTree shear(std::unordered_set to_keep); - - BPTree collapse(); - - private: - std::vector structure; // the topology - std::vector openclose; // cache'd mapping between parentheses - std::vector select_0_index; // cache of select 0 - std::vector select_1_index; // cache of select 1 - std::vector excess; - bool isRooted; // Is the tree rooted or not? - - void index_and_cache(); // construct the select caches - void rowTree_to_bp(const Rcpp::List & rowTree); // convert ape tree structure to boolean structure - void rowTree_to_metadata(const Rcpp::List & rowTree); // assign attributes - void newick_to_metadata(std::string newick); // convert newick to attributes - void structure_to_openclose(); // set the cache mapping between parentheses pairs - void set_node_metadata(unsigned int open_idx, std::string label, double length); // set attributes for a node - inline uint32_t open(uint32_t i) const; // obtain the index of the opening for a given parenthesis - inline uint32_t close(uint32_t i) const; // obtain the index of the closing for a given parenthesis - - int32_t bwd(uint32_t i, int32_t d) const; - int32_t enclose(uint32_t i) const; - }; -} - -#endif /* UNIFRAC_TREE_H */ - diff --git a/R/unifrac_cpp/unifrac.cpp b/R/unifrac_cpp/unifrac.cpp index 809196ad3..fba907972 100644 --- a/R/unifrac_cpp/unifrac.cpp +++ b/R/unifrac_cpp/unifrac.cpp @@ -10,7 +10,8 @@ #include "tree.hpp" #include "biom_interface.hpp" #include "unifrac.hpp" -#include "affinity.hpp" +#include "unifrac_internal.hpp" + #include #include #include @@ -20,475 +21,41 @@ #include #include -#include "unifrac_internal.hpp" - -// We will always have the CPU version -#define SUCMP_NM su_cpu -#include "unifrac_cmp.hpp" -#undef SUCMP_NM - -#ifdef UNIFRAC_ENABLE_ACC -#define SUCMP_NM su_acc -#include "unifrac_cmp.hpp" -#undef SUCMP_NM -#endif +#include using namespace su; -std::string su::test_table_ids_are_subset_of_tree(su::biom_interface &table, su::BPTree &tree) { - std::unordered_set tip_names = tree.get_tip_names(); - std::unordered_set::const_iterator hit; - std::string a_missing_name = ""; - - for(auto i : table.obs_ids) { - hit = tip_names.find(i); - if(hit == tip_names.end()) { - a_missing_name = i; - break; - } - } - - return a_missing_name; -} - -double** su::deconvolute_stripes(std::vector &stripes, uint32_t n) { - // would be better to just do striped_to_condensed_form - double **dm; - dm = (double**)malloc(sizeof(double*) * n); - if(dm == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double*) * n, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - for(unsigned int i = 0; i < n; i++) { - dm[i] = (double*)malloc(sizeof(double) * n); - if(dm[i] == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double) * n, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - dm[i][i] = 0; - } - - for(unsigned int i = 0; i < stripes.size(); i++) { - double *vec = stripes[i]; - unsigned int k = 0; - for(unsigned int row = 0, col = i + 1; row < n; row++, col++) { - if(col < n) { - dm[row][col] = vec[k]; - dm[col][row] = vec[k]; - } else { - dm[col % n][row] = vec[k]; - dm[row][col % n] = vec[k]; - } - k++; - } - } - return dm; -} - - -void su::stripes_to_condensed_form(std::vector &stripes, uint32_t n, double* cf, unsigned int start, unsigned int stop) { - // n must be >= 2, but that should be enforced upstream as that would imply - // computing unifrac on a single sample. - - uint64_t comb_N = comb_2(n); - for(unsigned int stripe = start; stripe < stop; stripe++) { - // compute the (i, j) position of each element in each stripe - uint64_t i = 0; - uint64_t j = stripe + 1; - for(uint64_t k = 0; k < n; k++, i++, j++) { - if(j == n) { - i = 0; - j = n - (stripe + 1); - } - // determine the position in the condensed form vector for a given (i, j) - // based off of - // https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.squareform.html - uint64_t comb_N_minus_i = comb_2(n - i); - cf[comb_N - comb_N_minus_i + (j - i - 1)] = stripes[stripe][k]; - } - } -} - - -// write in a 2D matrix -// also suitable for writing to disk -template -void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d) { - const uint64_t comb_N = su::comb_2(n); - for(uint64_t i = 0; i < n; i++) { - for(uint64_t j = 0; j < n; j++) { - TReal v; - if(i < j) { // upper triangle - const uint64_t comb_N_minus = su::comb_2(n - i); - v = cf[comb_N - comb_N_minus + (j - i - 1)]; - } else if (i > j) { // lower triangle - const uint64_t comb_N_minus = su::comb_2(n - j); - v = cf[comb_N - comb_N_minus + (i - j - 1)]; - } else { - v = 0.0; - } - buf2d[i*n+j] = v; - } - } -} - - -// make sure it is instantiated -template void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); -template void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); - -void su::condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d) { - su::condensed_form_to_matrix_T(cf,n,buf2d); -} - -void su::condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d) { - su::condensed_form_to_matrix_T(cf,n,buf2d); -} - -/* - * The stripes end up computing the following positions in the distance - * matrix. - * - * x A B C x x - * x x A B C x - * x x x A B C - * C x x x A B - * B C x x x A - * A B C x x x - * - * However, we store those stripes as vectors, ie - * [ A A A A A A ] - */ - - -// Helper class -// Will cache pointers and automatically release stripes when all elements are used -class OnceManagedStripes { - private: - const uint32_t n_samples; - const uint32_t n_stripes; - const ManagedStripes &stripes; - std::vector stripe_ptr; - std::vector stripe_accessed; - - const double *get_stripe(const uint32_t stripe) { - if (stripe_ptr[stripe]==0) stripe_ptr[stripe]=stripes.get_stripe(stripe); - return stripe_ptr[stripe]; - } - - void release_stripe(const uint32_t stripe) { - stripes.release_stripe(stripe); - stripe_ptr[stripe]=0; - } - - public: - OnceManagedStripes(const ManagedStripes &_stripes, const uint32_t _n_samples, const uint32_t _n_stripes) - : n_samples(_n_samples), n_stripes(_n_stripes) - , stripes(_stripes) - , stripe_ptr(n_stripes) - , stripe_accessed(n_stripes) - {} - - ~OnceManagedStripes() - { - for(uint32_t i = 0; i < n_stripes; i++) { - if (stripe_ptr[i]!=0) { - release_stripe(i); - } - } - } - - double get_val(const uint32_t stripe, const uint32_t el) - { - if (stripe_ptr[stripe]==0) stripe_ptr[stripe]=stripes.get_stripe(stripe); - const double *mystripe = stripe_ptr[stripe]; - double val = mystripe[el]; - - stripe_accessed[stripe]++; - if (stripe_accessed[stripe]==n_samples) release_stripe(stripe); // we will not use this stripe anymore - - return val; - } - - -}; - -// write in a 2D matrix -// also suitable for writing to disk -template -void su::stripes_to_matrix_T(const ManagedStripes &_stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size) { - // n_samples must be >= 2, but that should be enforced upstream as that would imply - // computing unifrac on a single sample. - - // tile for for better memory access pattern - const uint32_t TILE = (tile_size>0) ? tile_size : (128/sizeof(TReal)); - const uint32_t n_samples_tup = (n_samples+(TILE-1))/TILE; // round up - - OnceManagedStripes stripes(_stripes, n_samples, n_stripes); - - - for(uint32_t oi = 0; oi < n_samples_tup; oi++) { // off diagonal - // alternate between inner and outer off-diagonal, due to wrap around in stripes - const uint32_t o = ((oi%2)==0) ? \ - (oi/2)*TILE : /* close to diagonal */ \ - (n_samples_tup-(oi/2)-1)*TILE; /* far from diagonal */ - - for(uint32_t d = 0; d < (n_samples-o); d+=TILE) { // diagonal - - uint32_t iOut = d; - uint32_t jOut = d+o; - - uint32_t iMax = std::min(iOut+TILE,n_samples); - uint32_t jMax = std::min(jOut+TILE,n_samples); - - - if (iOut==jOut) { - // on diagonal - for(uint64_t i = iOut; i < iMax; i++) { - buf2d[i*n_samples+i] = 0.0; - - int64_t stripe=0; - - uint64_t j = i+1; - for(; (stripen_stripes) { - // ops, we overshoot... roll back - j-=(stripe-n_stripes); - stripe=n_stripes; - } - for(; (stripe(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size); -template void su::stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size); - -void su::stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size) { - return su::stripes_to_matrix_T(stripes, n_samples, n_stripes, buf2d, tile_size); -} - -void su::stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size) { - return su::stripes_to_matrix_T(stripes, n_samples, n_stripes, buf2d, tile_size); -} - - -void progressbar(float progress) { - // from http://stackoverflow.com/a/14539953 - // - // could encapsulate into a classs for displaying time elapsed etc - int barWidth = 70; - std::cout << "["; - int pos = barWidth * progress; - for (int i = 0; i < barWidth; ++i) { - if (i < pos) std::cout << "="; - else if (i == pos) std::cout << ">"; - else std::cout << " "; - } - std::cout << "] " << int(progress * 100.0) << " %\r"; - std::cout.flush(); -} - // Computes Faith's PD for the samples in `table` over the phylogenetic // tree given by `tree`. // Assure that tree does not contain ids that are not in table -void su::faith_pd(biom_interface &table, - BPTree &tree, - double* result) { - PropStack propstack(table.n_samples); +std::vector su::faith_pd(tse_interface &table, + BPTree &tree) { + PropStack propstack(table.n_samples); // construction seems to go okay + uint32_t node; - double *node_proportions; + std::vector node_proportions; double length; + + std::vector results = std::vector(table.n_samples, 0.0); // for node in postorderselect for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { node = tree.postorderselect(k); + // get branch length length = tree.lengths[node]; - + // get node proportions and set intermediate scores - node_proportions = propstack.pop(node); - set_proportions(node_proportions, tree, node, table, propstack); - + + node_proportions = set_proportions(tree, node, table, propstack); + for (unsigned int sample = 0; sample < table.n_samples; sample++){ // calculate contribution of node to score - result[sample] += (node_proportions[sample] > 0) * length; + results[sample] += (node_proportions[sample] > 0) * length; + //std::cout << node_proportions[sample] << " " << (node_proportions[sample] > 0) * length << " " << results[sample] << " - "; } } -} - - -#ifdef UNIFRAC_ENABLE_ACC - -// test only once, then use persistent value -static int proc_use_acc = -1; - -inline bool use_acc() { - if (proc_use_acc!=-1) return (proc_use_acc!=0); - int has_nvidia_gpu_rc = access("/proc/driver/nvidia/gpus", F_OK); - - bool print_info = false; - - if (const char* env_p = std::getenv("UNIFRAC_GPU_INFO")) { - print_info = true; - std::string env_s(env_p); - if ((env_s=="NO") || (env_s=="N") || (env_s=="no") || (env_s=="n") || - (env_s=="NEVER") || (env_s=="never")) { - print_info = false; - } - } - - - if (has_nvidia_gpu_rc != 0) { - if (print_info) printf("INFO (unifrac): GPU not found, using CPU\n"); - proc_use_acc=0; - return false; - } - - if (const char* env_p = std::getenv("UNIFRAC_USE_GPU")) { - std::string env_s(env_p); - if ((env_s=="NO") || (env_s=="N") || (env_s=="no") || (env_s=="n") || - (env_s=="NEVER") || (env_s=="never")) { - if (print_info) printf("INFO (unifrac): Use of GPU explicitly disabled, using CPU\n"); - proc_use_acc=0; - return false; - } - } - - if (print_info) printf("INFO (unifrac): Using GPU\n"); - proc_use_acc=1; - return true; -} -#endif - -void su::unifrac(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { -#ifdef UNIFRAC_ENABLE_ACC - if (use_acc()) { - su_acc::unifrac(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } else { -#else - if (true) { -#endif - su_cpu::unifrac(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } -} - - -void su::unifrac_vaw(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { -#ifdef UNIFRAC_ENABLE_ACC - if (use_acc()) { - su_acc::unifrac_vaw(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } else { -#else - if (true) { -#endif - su_cpu::unifrac_vaw(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } -} - - -void su::process_stripes(biom_interface &table, - BPTree &tree_sheared, - Method method, - bool variance_adjust, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - std::vector &threads, - std::vector &tasks) { - - // register a signal handler so we can ask the master thread for its - // progress - register_report_status(); - - // cannot use threading with openacc or openmp - for(unsigned int tid = 0; tid < threads.size(); tid++) { - if(variance_adjust) - su::unifrac_vaw( - std::ref(table), - std::ref(tree_sheared), - method, - std::ref(dm_stripes), - std::ref(dm_stripes_total), - &tasks[tid]); - else - su::unifrac( - std::ref(table), - std::ref(tree_sheared), - method, - std::ref(dm_stripes), - std::ref(dm_stripes_total), - &tasks[tid]); - } - - remove_report_status(); + return results; + //return(std::vector()); } diff --git a/R/unifrac_cpp/unifrac.hpp b/R/unifrac_cpp/unifrac.hpp index 842c4aeb2..242b84e7d 100644 --- a/R/unifrac_cpp/unifrac.hpp +++ b/R/unifrac_cpp/unifrac.hpp @@ -13,100 +13,16 @@ #include #include +#include + #ifndef __UNIFRAC -#include "task_parameters.hpp" #include "biom_interface.hpp" +#include "tree.hpp" namespace su { - enum Method {unweighted, weighted_normalized, weighted_unnormalized, generalized, unweighted_fp32, weighted_normalized_fp32, weighted_unnormalized_fp32, generalized_fp32}; - - void faith_pd(biom_interface &table, BPTree &tree, double* result); - - std::string test_table_ids_are_subset_of_tree(biom_interface &table, BPTree &tree); - void unifrac(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const task_parameters* task_p); - - void unifrac_vaw(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const task_parameters* task_p); - - double** deconvolute_stripes(std::vector &stripes, uint32_t n); - - class ManagedStripes { - public: - virtual ~ManagedStripes() {} - virtual const double *get_stripe(uint32_t stripe) const = 0; - virtual void release_stripe(uint32_t stripe) const = 0; - }; - - class MemoryStripes : public ManagedStripes { - private: - const double * const * stripes; // just a pointer, not owned - public: - MemoryStripes(const double * const * _stripes) : stripes(_stripes) {} - MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} - MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} - MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} - MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} - - virtual const double *get_stripe(uint32_t stripe) const {return stripes[stripe];} - virtual void release_stripe(uint32_t stripe) const {}; - }; - - - void stripes_to_condensed_form(std::vector &stripes, uint32_t n, double* cf, unsigned int start, unsigned int stop); - - // tile_size==0 means memory optimized - template void stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size=0); - void stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size=0); - void stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size=0); - - - template void condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d); - void condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); - void condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); - - inline uint64_t comb_2(uint64_t N) { - // based off of _comb_int_long - // https://github.com/scipy/scipy/blob/v0.19.1/scipy/special/_comb.pyx - - // Compute binom(N, k) for integers. - // - // we're disregarding overflow as that practically should not - // happen unless the number of samples processed is in excess - // of 4 billion - uint64_t val, j, M, nterms; - uint64_t k = 2; - - M = N + 1; - nterms = k < (N - k) ? k : N - k; - - val = 1; - - for(j = 1; j < nterms + 1; j++) { - val *= M - j; - val /= j; - } - return val; - } - - // process the stripes described by tasks - void process_stripes(biom_interface &table, - BPTree &tree_sheared, - Method method, - bool variance_adjust, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - std::vector &threads, - std::vector &tasks); + std::vector faith_pd(tse_interface &table, su::BPTree &tree); } + #define __UNIFRAC 1 #endif diff --git a/R/unifrac_cpp/unifrac_cmp.cpp b/R/unifrac_cpp/unifrac_cmp.cpp deleted file mode 100644 index a2c22c91a..000000000 --- a/R/unifrac_cpp/unifrac_cmp.cpp +++ /dev/null @@ -1,395 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include "tree.hpp" -#include "biom_interface.hpp" -#include -#include -#include -#include - -#include "unifrac_internal.hpp" - -#include "unifrac_task.hpp" -// Note: unifrac_task.hpp defines SUCMP_NM, needed by unifrac_cmp.hpp -#include "unifrac_cmp.hpp" - -// embed in this file, to properly instantiate the templatized functions -#include "unifrac_task.cpp" - -using namespace SUCMP_NM; - -template -inline void initialize_sample_counts(TFloat*& _counts, const su::task_parameters* task_p, const su::biom_interface &table) { - const unsigned int n_samples = task_p->n_samples; - const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up - TFloat * counts = NULL; - int err = 0; - err = posix_memalign((void **)&counts, 4096, sizeof(TFloat) * n_samples_r); - if(counts == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(TFloat) * n_samples_r, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - for(unsigned int i = 0; i < n_samples; i++) { - counts[i] = table.sample_counts[i]; - } - // avoid NaNs - for(unsigned int i = n_samples; i < n_samples_r; i++) { - counts[i] = 0.0; - } - - _counts=counts; -} - -template -inline void unifracTT(const su::biom_interface &table, - const su::BPTree &tree, - const bool want_total, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { - int err; - // no processor affinity whenusing openacc or openmp - - if(table.n_samples != task_p->n_samples) { - fprintf(stderr, "Task and table n_samples not equal\n"); - exit(EXIT_FAILURE); - } - const unsigned int n_samples = task_p->n_samples; - const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up - - - su::PropStackMulti propstack_multi(table.n_samples); - - const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; - - su::initialize_stripes(std::ref(dm_stripes), std::ref(dm_stripes_total), want_total, task_p); - - TaskT taskObj(std::ref(dm_stripes), std::ref(dm_stripes_total),max_emb,task_p); - - TFloat *lengths = NULL; - err = posix_memalign((void **)&lengths, 4096, sizeof(TFloat) * max_emb); - if(err != 0) { - fprintf(stderr, "posix_memalign(%d) failed: %d\n", sizeof(TFloat) * max_emb, err); - exit(EXIT_FAILURE); - } -#pragma acc enter data create(lengths[:max_emb]) - - /* - * The values in the example vectors correspond to index positions of an - * element in the resulting distance matrix. So, in the example below, - * the following can be interpreted: - * - * [0 1 2] - * [1 2 3] - * - * As comparing the sample for row 0 against the sample for col 1, the - * sample for row 1 against the sample for col 2, the sample for row 2 - * against the sample for col 3. - * - * In other words, we're computing stripes of a distance matrix. In the - * following example, we're computing over 6 samples requiring 3 - * stripes. - * - * A; stripe == 0 - * [0 1 2 3 4 5] - * [1 2 3 4 5 0] - * - * B; stripe == 1 - * [0 1 2 3 4 5] - * [2 3 4 5 0 1] - * - * C; stripe == 2 - * [0 1 2 3 4 5] - * [3 4 5 0 1 2] - * - * The stripes end up computing the following positions in the distance - * matrix. - * - * x A B C x x - * x x A B C x - * x x x A B C - * C x x x A B - * B C x x x A - * A B C x x x - * - * However, we store those stripes as vectors, ie - * [ A A A A A A ] - * - * We end up performing N / 2 redundant calculations on the last stripe - * (see C) but that is small over large N. - */ - - unsigned int k = 0; // index in tree - const unsigned int max_k = (tree.nparens / 2) - 1; - - const unsigned int num_prop_chunks = propstack_multi.get_num_stacks(); - while (k &propstack = propstack_multi.get_prop_stack(ck); - const unsigned int tstart = propstack_multi.get_start(ck); - const unsigned int tend = propstack_multi.get_end(ck); - unsigned int my_filled_emb = 0; - unsigned int my_k=k_start; - - while ((my_filled_embbypass_tips && tree.isleaf(node)) - continue; - - if (ck==0) { // they all do the same thing, so enough for the first to update the global state - lengths[filled_emb] = tree.lengths[node]; - filled_emb++; - } - taskObj.embed_proportions_range(node_proportions, tstart, tend, my_filled_emb); - my_filled_emb++; - } - if (ck==0) { // they all do the same thing, so enough for the first to update the global state - k=my_k; - } - } - - taskObj.sync_embedded_proportions(filled_emb); -#ifdef _OPENACC - // lengths may be still in use in async mode, wait -#pragma acc wait -#pragma acc update device(lengths[:filled_emb]) -#endif - taskObj._run(filled_emb,lengths); - filled_emb=0; - - su::try_report(task_p, k, max_k); - } - -#pragma acc wait - - if(want_total) { - const uint64_t start_idx = task_p->start; - const uint64_t stop_idx = task_p->stop; - - TFloat * const dm_stripes_buf = taskObj.dm_stripes.buf; - const TFloat * const dm_stripes_total_buf = taskObj.dm_stripes_total.buf; - -#pragma acc parallel loop collapse(2) present(dm_stripes_buf,dm_stripes_total_buf) - for(uint64_t i = start_idx; i < stop_idx; i++) - for(uint64_t j = 0; j < n_samples; j++) { - uint64_t idx = (i-start_idx)*n_samples_r+j; - dm_stripes_buf[idx]=dm_stripes_buf[idx]/dm_stripes_total_buf[idx]; - // taskObj.dm_stripes[i][j] = taskObj.dm_stripes[i][j] / taskObj.dm_stripes_total[i][j]; - } - - } - -#pragma acc exit data delete(lengths[:max_emb]) - free(lengths); -} - -void SUCMP_NM::unifrac(const su::biom_interface &table, - const su::BPTree &tree, - su::Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { - switch(unifrac_method) { - case su::unweighted: - unifracTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized: - unifracTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized: - unifracTT,double>( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized: - unifracTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::unweighted_fp32: - unifracTT,float>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized_fp32: - unifracTT,float>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized_fp32: - unifracTT,float>( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized_fp32: - unifracTT,float>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - default: - fprintf(stderr, "Unknown unifrac task\n"); - exit(1); - break; - } -} - - -template -inline void unifrac_vawTT(const su::biom_interface &table, - const su::BPTree &tree, - const bool want_total, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { - int err; - // no processor affinity whenusing openacc or openmp - - if(table.n_samples != task_p->n_samples) { - fprintf(stderr, "Task and table n_samples not equal\n"); - exit(EXIT_FAILURE); - } - const unsigned int n_samples = task_p->n_samples; - const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up - - su::PropStackMulti propstack_multi(table.n_samples); - su::PropStackMulti countstack_multi(table.n_samples); - - const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; - - TFloat *sample_total_counts; - - initialize_sample_counts(sample_total_counts, task_p, table); -#pragma acc enter data copyin(sample_total_counts[:n_samples_r]) - su::initialize_stripes(std::ref(dm_stripes), std::ref(dm_stripes_total), want_total, task_p); - - TaskT taskObj(std::ref(dm_stripes), std::ref(dm_stripes_total), sample_total_counts, max_emb, task_p); - - TFloat *lengths = NULL; - err = posix_memalign((void **)&lengths, 4096, sizeof(TFloat) * max_emb); - if(err != 0) { - fprintf(stderr, "posix_memalign(%d) failed: %d\n", sizeof(TFloat) * max_emb, err); - exit(EXIT_FAILURE); - } -#pragma acc enter data create(lengths[:max_emb]) - - unsigned int k = 0; // index in tree - const unsigned int max_k = (tree.nparens / 2) - 1; - - const unsigned int num_prop_chunks = propstack_multi.get_num_stacks(); - while (k &propstack = propstack_multi.get_prop_stack(ck); - su::PropStack &countstack = countstack_multi.get_prop_stack(ck); - const unsigned int tstart = propstack_multi.get_start(ck); - const unsigned int tend = propstack_multi.get_end(ck); - unsigned int my_filled_emb = 0; - unsigned int my_k=k_start; - - while ((my_filled_embbypass_tips && tree.isleaf(node)) - continue; - - if (ck==0) { // they all do the same thing, so enough for the first to update the global state - lengths[filled_emb] = tree.lengths[node]; - filled_emb++; - } - taskObj.embed_range(node_proportions, node_counts, tstart, tend, my_filled_emb); - my_filled_emb++; - } - if (ck==0) { // they all do the same thing, so enough for the first to update the global state - k=my_k; - } - } - -#pragma acc wait -#pragma acc update device(lengths[:filled_emb]) - taskObj.sync_embedded(filled_emb); - taskObj._run(filled_emb,lengths); - filled_emb = 0; - - su::try_report(task_p, k, max_k); - } - -#pragma acc wait - if(want_total) { - const uint64_t start_idx = task_p->start; - const uint64_t stop_idx = task_p->stop; - - TFloat * const dm_stripes_buf = taskObj.dm_stripes.buf; - const TFloat * const dm_stripes_total_buf = taskObj.dm_stripes_total.buf; - -#pragma acc parallel loop collapse(2) present(dm_stripes_buf,dm_stripes_total_buf) - for(uint64_t i = start_idx; i < stop_idx; i++) - for(uint64_t j = 0; j < n_samples; j++) { - uint64_t idx = (i-start_idx)*n_samples_r+j; - dm_stripes_buf[idx]=dm_stripes_buf[idx]/dm_stripes_total_buf[idx]; - // taskObj.dm_stripes[i][j] = taskObj.dm_stripes[i][j] / taskObj.dm_stripes_total[i][j]; - } - - } - - -#pragma acc exit data delete(lengths[:max_emb]) -#pragma acc exit data delete(sample_total_counts[:n_samples_r]) - free(lengths); - free(sample_total_counts); -} - -void SUCMP_NM::unifrac_vaw(const su::biom_interface &table, - const su::BPTree &tree, - su::Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { - switch(unifrac_method) { - case su::unweighted: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized: - unifrac_vawTT,double>( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::unweighted_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized_fp32: - unifrac_vawTT,float >( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - default: - fprintf(stderr, "Unknown unifrac task\n"); - exit(1); - break; - } -} - diff --git a/R/unifrac_cpp/unifrac_cmp.hpp b/R/unifrac_cpp/unifrac_cmp.hpp deleted file mode 100644 index 9d5a57877..000000000 --- a/R/unifrac_cpp/unifrac_cmp.hpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#ifdef SUCMP_NM - -/* Note: Allow multiple definitions of this header, using different SUCMP_NM */ - -#include "task_parameters.hpp" -#include "tree.hpp" -#include "biom_interface.hpp" - -#include "unifrac_internal.hpp" - -namespace SUCMP_NM { - - void unifrac(const su::biom_interface &table, - const su::BPTree &tree, - su::Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p); - - void unifrac_vaw(const su::biom_interface &table, - const su::BPTree &tree, - su::Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p); - -} - -#endif /* SUCMP_NM */ diff --git a/R/unifrac_cpp/unifrac_internal.cpp b/R/unifrac_cpp/unifrac_internal.cpp index 1fff6e148..1ae4e61fd 100644 --- a/R/unifrac_cpp/unifrac_internal.cpp +++ b/R/unifrac_cpp/unifrac_internal.cpp @@ -9,7 +9,8 @@ #include "tree.hpp" #include "biom_interface.hpp" -#include "affinity.hpp" +#include "unifrac_internal.hpp" + #include #include #include @@ -18,63 +19,10 @@ #include #include -#include "unifrac_internal.hpp" - -static pthread_mutex_t printf_mutex; -static bool* report_status; - -static int sync_printf(const char *format, ...) { - // https://stackoverflow.com/a/23587285/19741 - va_list args; - va_start(args, format); - - pthread_mutex_lock(&printf_mutex); - vprintf(format, args); - pthread_mutex_unlock(&printf_mutex); - - va_end(args); -} - -static void sig_handler(int signo) { - // http://www.thegeekstuff.com/2012/03/catch-signals-sample-c-code - if (signo == SIGUSR1) { - if(report_status == NULL) - fprintf(stderr, "Cannot report status.\n"); - else { - for(int i = 0; i < CPU_SETSIZE; i++) { - report_status[i] = true; - } - } - } -} +#include using namespace su; -void su::try_report(const su::task_parameters* task_p, unsigned int k, unsigned int max_k) { - if(__builtin_expect(report_status[task_p->tid], false)) { - sync_printf("tid:%u\tstart:%u\tstop:%u\tk:%u\ttotal:%u\n", task_p->tid, task_p->start, task_p->stop, k, max_k); - report_status[task_p->tid] = false; - } -} - -void su::register_report_status() { - // register a signal handler so we can ask the master thread for its - // progress - if (signal(SIGUSR1, sig_handler) == SIG_ERR) - fprintf(stderr, "Can't catch SIGUSR1\n"); - - report_status = (bool*)calloc(sizeof(bool), CPU_SETSIZE); - pthread_mutex_init(&printf_mutex, NULL); -} - -void su::remove_report_status() { - if(report_status != NULL) { - pthread_mutex_destroy(&printf_mutex); - free(report_status); - report_status = NULL; - } -} - template PropStack::PropStack(uint32_t vecsize) : prop_stack() @@ -86,56 +34,26 @@ PropStack::PropStack(uint32_t vecsize) template PropStack::~PropStack() { - // drain stack - for(unsigned int i = 0; i < prop_stack.size(); i++) { - TFloat *vec = prop_stack.top(); - prop_stack.pop(); - free(vec); - } - - // drain the map - for(auto it = prop_map.begin(); it != prop_map.end(); it++) { - TFloat *vec = it->second; - free(vec); - } - prop_map.clear(); } template -TFloat* PropStack::get(uint32_t i) { - return prop_map[i]; +std::vector PropStack::get(uint32_t i) { + if(prop_map.count(i) > 0){ + return prop_map.at(i); + } + else { + return(std::vector()); + } } template -void PropStack::push(uint32_t node) { - TFloat* vec = prop_map[node]; - prop_map.erase(node); - prop_stack.push(vec); +void PropStack::clear(uint32_t i) { + prop_map[i] = std::vector(); } template -TFloat* PropStack::pop(uint32_t node) { - /* - * if we don't have any available vectors, create one - * add it to our record of known vectors so we can track our mallocs - */ - TFloat *vec; - int err = 0; - if(prop_stack.empty()) { - err = posix_memalign((void **)&vec, 32, sizeof(TFloat) * defaultsize); - if(vec == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(TFloat) * defaultsize, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - } - else { - vec = prop_stack.top(); - prop_stack.pop(); - } - +void PropStack::update(uint32_t node, std::vector vec) { prop_map[node] = vec; - return vec; } // make sure they get instantiated @@ -143,148 +61,54 @@ template class su::PropStack; template class su::PropStack; -void su::initialize_stripes(std::vector &dm_stripes, - std::vector &dm_stripes_total, - bool want_total, - const su::task_parameters* task_p) { - int err = 0; - for(unsigned int i = task_p->start; i < task_p->stop; i++){ - err = posix_memalign((void **)&dm_stripes[i], 4096, sizeof(double) * task_p->n_samples); - if(dm_stripes[i] == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(double) * task_p->n_samples, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - for(unsigned int j = 0; j < task_p->n_samples; j++) - dm_stripes[i][j] = 0.; - - if(want_total) { - err = posix_memalign((void **)&dm_stripes_total[i], 4096, sizeof(double) * task_p->n_samples); - if(dm_stripes_total[i] == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes err %d; [%s]:%d\n", - sizeof(double) * task_p->n_samples, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - for(unsigned int j = 0; j < task_p->n_samples; j++) - dm_stripes_total[i][j] = 0.; - } - } -} - template -void su::set_proportions(TFloat* __restrict__ props, - const BPTree &tree, +std::vector su::set_proportions(const BPTree &tree, uint32_t node, - const biom_interface &table, + const tse_interface &table, PropStack &ps, bool normalize) { + + std::vector props = std::vector(); if(tree.isleaf(node)) { - table.get_obs_data(tree.names[node], props); - if (normalize) { -#pragma omp parallel for schedule(static) - for(unsigned int i = 0; i < table.n_samples; i++) { - props[i] /= table.sample_counts[i]; - } - } - + std::string leaf = tree.names[node]; + props = table.get_obs_data(leaf); // Here we basically just need the row for the specified node + //std::cout << "l " << props[0] << " " << props[1] << " " << props[2] << "\n"; + if (normalize) { +//#pragma omp parallel for schedule(static) + for(unsigned int i = 0; i < table.n_samples; i++) { + props[i] /= table.sample_counts[i]; + } + } } else { unsigned int current = tree.leftchild(node); unsigned int right = tree.rightchild(node); -#pragma omp parallel for schedule(static) - for(unsigned int i = 0; i < table.n_samples; i++) - props[i] = 0; - +//#pragma omp parallel for schedule(static) + + for(unsigned int i = 0; i < table.n_samples; i++){ + props.push_back(0); + } + while(current <= right && current != 0) { - TFloat * __restrict__ vec = ps.get(current); // pull from prop map - ps.push(current); // remove from prop map, place back on stack - -#pragma omp parallel for schedule(static) + std::vector vec = ps.get(current); // pull from prop map + ps.clear(current); // remove from prop map, place back on stack +//#pragma omp parallel for schedule(static) for(unsigned int i = 0; i < table.n_samples; i++) props[i] = props[i] + vec[i]; - + current = tree.rightsibling(current); } + + //std::cout << "n " << props[0] << " " << props[1] << " " << props[2] << "\n"; + } + ps.update(node, props); + return(props); } // make sure they get instantiated -template void su::set_proportions(float* __restrict__ props, - const BPTree &tree, - uint32_t node, - const biom_interface &table, - PropStack &ps, - bool normalize); -template void su::set_proportions(double* __restrict__ props, - const BPTree &tree, +template std::vector su::set_proportions(const BPTree &tree, uint32_t node, - const biom_interface &table, + const tse_interface &table, PropStack &ps, bool normalize); - -template -void su::set_proportions_range(TFloat* __restrict__ props, - const BPTree &tree, - uint32_t node, - const biom_interface &table, - unsigned int start, unsigned int end, - PropStack &ps, - bool normalize) { - const unsigned int els = end-start; - if(tree.isleaf(node)) { - table.get_obs_data_range(tree.names[node], start, end, normalize, props); - } else { - const unsigned int right = tree.rightchild(node); - unsigned int current = tree.leftchild(node); - - for(unsigned int i = 0; i < els; i++) - props[i] = 0; - - while(current <= right && current != 0) { - const TFloat * __restrict__ vec = ps.get(current); // pull from prop map - ps.push(current); // remove from prop map, place back on stack - - for(unsigned int i = 0; i < els; i++) - props[i] += vec[i]; - - current = tree.rightsibling(current); - } - } -} - -// make sure they get instantiated -template void su::set_proportions_range(float* __restrict__ props, - const BPTree &tree, - uint32_t node, - const biom_interface &table, - unsigned int start, unsigned int end, - PropStack &ps, - bool normalize); -template void su::set_proportions_range(double* __restrict__ props, - const BPTree &tree, - uint32_t node, - const biom_interface &table, - unsigned int start, unsigned int end, - PropStack &ps, - bool normalize); - -std::vector su::make_strides(unsigned int n_samples) { - uint32_t n_rotations = (n_samples + 1) / 2; - std::vector dm_stripes(n_rotations); - - int err = 0; - for(unsigned int i = 0; i < n_rotations; i++) { - double* tmp; - err = posix_memalign((void **)&tmp, 32, sizeof(double) * n_samples); - if(tmp == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(double) * n_samples, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - for(unsigned int j = 0; j < n_samples; j++) - tmp[j] = 0.0; - dm_stripes[i] = tmp; - } - return dm_stripes; -} - diff --git a/R/unifrac_cpp/unifrac_internal.hpp b/R/unifrac_cpp/unifrac_internal.hpp index b53b0b5ae..1a68597c2 100644 --- a/R/unifrac_cpp/unifrac_internal.hpp +++ b/R/unifrac_cpp/unifrac_internal.hpp @@ -13,84 +13,32 @@ #include #include #include + #include "biom_interface.hpp" -#include "task_parameters.hpp" #include "unifrac.hpp" namespace su { - // helper reporting functions - void register_report_status(); - void remove_report_status(); - void try_report(const su::task_parameters* task_p, unsigned int k, unsigned int max_k); template class PropStack { private: - std::stack prop_stack; - std::unordered_map prop_map; + std::stack> prop_stack; + std::unordered_map> prop_map; uint32_t defaultsize; public: PropStack(uint32_t vecsize); virtual ~PropStack(); - TFloat* pop(uint32_t i); - void push(uint32_t i); - TFloat* get(uint32_t i); - }; - - // Helper class with default constructor - // The default is small enough to fit in L1 cache - template - class PropStackFixed : public PropStack { - public: - static const uint32_t DEF_VEC_SIZE = 1024*sizeof(double)/sizeof(TFloat); - - PropStackFixed() : PropStack(DEF_VEC_SIZE) {} + void clear(uint32_t i); + void update(uint32_t i, std::vector vec); + std::vector get(uint32_t i); }; - // Helper class that splits a large vec_size into several smaller chunks of def_size template - class PropStackMulti { - protected: - const uint32_t vecsize; - std::vector > multi; - - public: - PropStackMulti(uint32_t _vecsize) - : vecsize(_vecsize) - , multi((vecsize + (PropStackFixed::DEF_VEC_SIZE-1))/PropStackFixed::DEF_VEC_SIZE) // round up - {} - ~PropStackMulti() {} - - uint32_t get_num_stacks() const {return (vecsize + (PropStackFixed::DEF_VEC_SIZE-1))/PropStackFixed::DEF_VEC_SIZE;} - - uint32_t get_start(uint32_t idx) const {return idx*PropStackFixed::DEF_VEC_SIZE;} - uint32_t get_end(uint32_t idx) const {return std::min((idx+1)*PropStackFixed::DEF_VEC_SIZE, vecsize);} - - PropStackFixed &get_prop_stack(uint32_t idx) {return multi[idx];} - }; - - template - void set_proportions(TFloat* __restrict__ props, - const BPTree &tree, uint32_t node, - const biom_interface &table, + std::vector set_proportions(const BPTree &tree, uint32_t node, + const tse_interface &table, PropStack &ps, bool normalize = true); - template - void set_proportions_range(TFloat* __restrict__ props, - const BPTree &tree, uint32_t node, - const biom_interface &table,unsigned int start, unsigned int end, - PropStack &ps, - bool normalize = true); - - - void initialize_stripes(std::vector &dm_stripes, - std::vector &dm_stripes_total, - bool want_total, - const su::task_parameters* task_p); - - std::vector make_strides(unsigned int n_samples); - } #endif diff --git a/R/unifrac_cpp/unifrac_internal_s.cpp b/R/unifrac_cpp/unifrac_internal_s.cpp deleted file mode 100644 index 8e1f18a0e..000000000 --- a/R/unifrac_cpp/unifrac_internal_s.cpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include "tree_s.hpp" -#include "biom_interface_s.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -#include "unifrac_internal_s.hpp" - -using namespace su; - -template -PropStack::PropStack(uint32_t vecsize) -: prop_stack() -, prop_map() -, defaultsize(vecsize) -{ - prop_map.reserve(1000); -} - -template -PropStack::~PropStack() { -} - -template -std::vector PropStack::get(uint32_t i) { - if(prop_map.count(i) > 0){ - return prop_map.at(i); - } - else { - return(std::vector()); - } -} - -template -void PropStack::clear(uint32_t i) { - prop_map[i] = std::vector(); -} - -template -void PropStack::update(uint32_t node, std::vector vec) { - prop_map[node] = vec; -} - -// make sure they get instantiated -template class su::PropStack; -template class su::PropStack; - - -template -std::vector su::set_proportions(const BPTree &tree, - uint32_t node, - const tse_interface &table, - PropStack &ps, - bool normalize) { - - std::vector props = std::vector(); - if(tree.isleaf(node)) { - std::string leaf = tree.names[node]; - props = table.get_obs_data(leaf); // Here we basically just need the row for the specified node - //std::cout << "l " << props[0] << " " << props[1] << " " << props[2] << "\n"; - if (normalize) { -//#pragma omp parallel for schedule(static) - for(unsigned int i = 0; i < table.n_samples; i++) { - props[i] /= table.sample_counts[i]; - } - } - } else { - unsigned int current = tree.leftchild(node); - unsigned int right = tree.rightchild(node); - -//#pragma omp parallel for schedule(static) - - for(unsigned int i = 0; i < table.n_samples; i++){ - props.push_back(0); - } - - while(current <= right && current != 0) { - std::vector vec = ps.get(current); // pull from prop map - ps.clear(current); // remove from prop map, place back on stack -//#pragma omp parallel for schedule(static) - for(unsigned int i = 0; i < table.n_samples; i++) - props[i] = props[i] + vec[i]; - - current = tree.rightsibling(current); - } - - //std::cout << "n " << props[0] << " " << props[1] << " " << props[2] << "\n"; - - } - ps.update(node, props); - return(props); -} - -// make sure they get instantiated -template std::vector su::set_proportions(const BPTree &tree, - uint32_t node, - const tse_interface &table, - PropStack &ps, - bool normalize); diff --git a/R/unifrac_cpp/unifrac_internal_s.hpp b/R/unifrac_cpp/unifrac_internal_s.hpp deleted file mode 100644 index 49b0690bc..000000000 --- a/R/unifrac_cpp/unifrac_internal_s.hpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#ifndef __UNIFRAC_INTERNAL -#define __UNIFRAC_INTERNAL 1 - -#include -#include -#include -#include "biom_interface_s.hpp" -#include "unifrac_s.hpp" - -namespace su { - - template - class PropStack { - private: - std::stack> prop_stack; - std::unordered_map> prop_map; - uint32_t defaultsize; - public: - PropStack(uint32_t vecsize); - virtual ~PropStack(); - void clear(uint32_t i); - void update(uint32_t i, std::vector vec); - std::vector get(uint32_t i); - }; - - template - std::vector set_proportions(const BPTree &tree, uint32_t node, - const tse_interface &table, - PropStack &ps, - bool normalize = true); - -} - -#endif diff --git a/R/unifrac_cpp/unifrac_s.cpp b/R/unifrac_cpp/unifrac_s.cpp deleted file mode 100644 index effc4ec0c..000000000 --- a/R/unifrac_cpp/unifrac_s.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include "tree_s.hpp" -#include "biom_interface_s.hpp" -#include "unifrac_s.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -#include "unifrac_internal_s.hpp" - -#include - -using namespace su; - -// Computes Faith's PD for the samples in `table` over the phylogenetic -// tree given by `tree`. -// Assure that tree does not contain ids that are not in table -std::vector su::faith_pd(tse_interface &table, - BPTree &tree) { - PropStack propstack(table.n_samples); // construction seems to go okay - - - uint32_t node; - std::vector node_proportions; - double length; - - std::vector results = std::vector(table.n_samples, 0.0); - - // for node in postorderselect - for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { - node = tree.postorderselect(k); - - // get branch length - length = tree.lengths[node]; - - // get node proportions and set intermediate scores - - node_proportions = set_proportions(tree, node, table, propstack); - - for (unsigned int sample = 0; sample < table.n_samples; sample++){ - // calculate contribution of node to score - results[sample] += (node_proportions[sample] > 0) * length; - //std::cout << node_proportions[sample] << " " << (node_proportions[sample] > 0) * length << " " << results[sample] << " - "; - } - } - return results; - //return(std::vector()); -} diff --git a/R/unifrac_cpp/unifrac_s.hpp b/R/unifrac_cpp/unifrac_s.hpp deleted file mode 100644 index 9d52fba3d..000000000 --- a/R/unifrac_cpp/unifrac_s.hpp +++ /dev/null @@ -1,29 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include -#include -#include -#include -#include - -#include - -#ifndef __UNIFRAC - -#include "task_parameters.hpp" -#include "biom_interface_s.hpp" -#include "tree_s.hpp" - - namespace su { - std::vector faith_pd(tse_interface &table, su::BPTree &tree); - } - -#define __UNIFRAC 1 -#endif diff --git a/R/unifrac_cpp/unifrac_task.cpp b/R/unifrac_cpp/unifrac_task.cpp deleted file mode 100644 index 0a40a7310..000000000 --- a/R/unifrac_cpp/unifrac_task.cpp +++ /dev/null @@ -1,785 +0,0 @@ -#include -#include "unifrac_task.hpp" -#include - - - - -template -void SUCMP_NM::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - - bool * const __restrict__ zcheck = this->zcheck; - TFloat * const __restrict__ sums = this->sums; - - const uint64_t step_size = SUCMP_NM::UnifracUnnormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - // check for zero values and pre-compute single column sums -#ifdef _OPENACC -#pragma acc parallel loop present(embedded_proportions,lengths,zcheck,sums) -#else -#pragma omp parallel for default(shared) -#endif - for(uint64_t k=0; k::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,lengths,zcheck,sums) async -#else - // use dynamic scheduling due to non-homogeneity in the loop -#pragma omp parallel for default(shared) schedule(dynamic,1) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - - if (k>=n_samples) continue; // past the limit - - const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const bool allzero_k = zcheck[k]; - const bool allzero_l1 = zcheck[l1]; - - if (allzero_k && allzero_l1) { - // nothing to do, would have to add 0 - } else { - TFloat my_stripe; - - if (allzero_k || allzero_l1) { - // one side has all zeros - // we can use the distributed property, and use the pre-computed values - - const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 - k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 - - // keep reads in the same place to maximize GPU warp performance - my_stripe = sums[ridx]; - - } else { - // both sides non zero, use the explicit but slow approach - my_stripe = 0.0; - -#pragma acc loop seq - for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracVawUnnormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - const TFloat * const __restrict__ embedded_counts = this->embedded_counts; - const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - - const uint64_t step_size = SUCMP_NM::UnifracVawUnnormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - // point of thread -#ifdef _OPENACC - const unsigned int acc_vector_size = SUCMP_NM::UnifracVawUnnormalizedWeightedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,lengths) async -#else -#pragma omp parallel for default(shared) schedule(dynamic,1) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; - - TFloat my_stripe = dm_stripe[k]; - -#pragma acc loop seq - for (uint64_t emb=0; emb 0) { - TFloat u1 = embedded_proportions[offset + k]; - TFloat v1 = embedded_proportions[offset + l1]; - TFloat diff1 = fabs(u1 - v1); - TFloat length = lengths[emb]; - - my_stripe += (diff1 * length) / vaw; - } - } - - dm_stripe[k] = my_stripe; - } - - } - } - -#ifdef _OPENACC - // next iteration will use the alternative space - std::swap(this->embedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - bool * const __restrict__ zcheck = this->zcheck; - TFloat * const __restrict__ sums = this->sums; - - const uint64_t step_size = SUCMP_NM::UnifracNormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - // check for zero values and pre-compute single column sums -#ifdef _OPENACC -#pragma acc parallel loop present(embedded_proportions,lengths,zcheck,sums) -#else -#pragma omp parallel for default(shared) -#endif - for(uint64_t k=0; k::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,lengths,zcheck,sums) async -#else - // use dynamic scheduling due to non-homogeneity in the loop -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - - if (k>=n_samples) continue; // past the limit - - const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const bool allzero_k = zcheck[k]; - const bool allzero_l1 = zcheck[l1]; - - if (allzero_k && allzero_l1) { - // nothing to do, would have to add 0 - } else { - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - // the totals can always use the distributed property - dm_stripe_total[k] += sums[k] + sums[l1]; - - TFloat my_stripe; - - if (allzero_k || allzero_l1) { - // one side has all zeros - // we can use the distributed property, and use the pre-computed values - - const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 - k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 - - // keep reads in the same place to maximize GPU warp performance - my_stripe = sums[ridx]; - - } else { - // both sides non zero, use the explicit but slow approach - - my_stripe = 0.0; - -#pragma acc loop seq - for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracVawNormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - const TFloat * const __restrict__ embedded_counts = this->embedded_counts; - const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - const uint64_t step_size = SUCMP_NM::UnifracVawNormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - // point of thread -#ifdef _OPENACC - const unsigned int acc_vector_size = SUCMP_NM::UnifracVawNormalizedWeightedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async -#else -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; - - TFloat my_stripe = dm_stripe[k]; - TFloat my_stripe_total = dm_stripe_total[k]; - -#pragma acc loop seq - for (uint64_t emb=0; emb 0) { - TFloat u1 = embedded_proportions[offset + k]; - TFloat v1 = embedded_proportions[offset + l1]; - TFloat diff1 = fabs(u1 - v1); - TFloat length = lengths[emb]; - - my_stripe += (diff1 * length) / vaw; - my_stripe_total += ((u1 + v1) * length) / vaw; - } - } - - dm_stripe[k] = my_stripe; - dm_stripe_total[k] = my_stripe_total; - - } - - } - } - -#ifdef _OPENACC - // next iteration will use the alternative space - std::swap(this->embedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracGeneralizedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - const TFloat g_unifrac_alpha = this->task_p->g_unifrac_alpha; - - const uint64_t step_size = SUCMP_NM::UnifracGeneralizedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - // point of thread -#ifdef _OPENACC - const unsigned int acc_vector_size = SUCMP_NM::UnifracGeneralizedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,lengths) async -#else -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - TFloat my_stripe = dm_stripe[k]; - TFloat my_stripe_total = dm_stripe_total[k]; - -#pragma acc loop seq - for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracVawGeneralizedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - const TFloat g_unifrac_alpha = this->task_p->g_unifrac_alpha; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - const TFloat * const __restrict__ embedded_counts = this->embedded_counts; - const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - const uint64_t step_size = SUCMP_NM::UnifracVawGeneralizedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - // quick hack, to be finished - - // point of thread -#ifdef _OPENACC - const unsigned int acc_vector_size = SUCMP_NM::UnifracVawGeneralizedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async -#else -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; - - TFloat my_stripe = dm_stripe[k]; - TFloat my_stripe_total = dm_stripe_total[k]; - -#pragma acc loop seq - for (uint64_t emb=0; emb 0) { - TFloat u1 = embedded_proportions[offset + k]; - TFloat v1 = embedded_proportions[offset + l1]; - TFloat length = lengths[emb]; - - TFloat sum1 = (u1 + v1) / vaw; - TFloat sub1 = fabs(u1 - v1) / vaw; - TFloat sum_pow1 = pow(sum1, g_unifrac_alpha) * length; - - my_stripe += sum_pow1 * (sub1 / sum1); - my_stripe_total += sum_pow1; - } - } - - dm_stripe[k] = my_stripe; - dm_stripe_total[k] = my_stripe_total; - - } - } - } - -#ifdef _OPENACC - // next iteration will use the alternative space - std::swap(this->embedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracUnweightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const uint64_t * const __restrict__ embedded_proportions = this->embedded_proportions; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - TFloat * const __restrict__ sums = this->sums; - - const uint64_t step_size = SUCMP_NM::UnifracUnweightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - const uint64_t filled_embs_els = filled_embs/64; - const uint64_t filled_embs_rem = filled_embs%64; - - const uint64_t filled_embs_els_round = (filled_embs+63)/64; - - - // pre-compute sums of length elements, since they are likely to be accessed many times - // We will use a 8-bit map, to keep it small enough to keep in L1 cache -#ifdef _OPENACC -#pragma acc parallel loop collapse(2) gang present(lengths,sums) async -#else -#pragma omp parallel for default(shared) -#endif - for (uint64_t emb_el=0; emb_el> 0) & 1) * pl[0]) + (((b8_i >> 1) & 1) * pl[1]) + - (((b8_i >> 2) & 1) * pl[2]) + (((b8_i >> 3) & 1) * pl[3]) + - (((b8_i >> 4) & 1) * pl[4]) + (((b8_i >> 5) & 1) * pl[5]) + - (((b8_i >> 6) & 1) * pl[6]) + (((b8_i >> 7) & 1) * pl[7]); - } - } - } - if (filled_embs_rem>0) { // add also the overflow elements - const uint64_t emb_el=filled_embs_els; -#ifdef _OPENACC -#pragma acc parallel loop gang present(lengths,sums) async -#else - // no advantage of OMP, too small -#endif - for (uint64_t sub8=0; sub8<8; sub8++) { - // we are summing we have enough buffer in sums - const uint64_t emb8 = emb_el*8+sub8; - TFloat * __restrict__ psum = &(sums[emb8<<8]); - -#pragma acc loop vector - // compute all the combinations for this block, set to 0 any past the limit - // as above - for (uint64_t b8_i=0; b8_i<0x100; b8_i++) { - TFloat val= 0; - for (uint64_t li=(emb8*8); li> (li-(emb8*8))) & 1) * lengths[li]; - } - psum[b8_i] = val; - } - } - } - - // point of thread -#ifdef _OPENACC -#pragma acc wait - const unsigned int acc_vector_size = SUCMP_NM::UnifracUnweightedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,sums) async -#else - // use dynamic scheduling due to non-homogeneity in the loop -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - bool did_update = false; - TFloat my_stripe = 0.0; - TFloat my_stripe_total = 0.0; - -#pragma acc loop seq - for (uint64_t emb_el=0; emb_el> 8) & 0xff)] + - psum[0x200+((x1 >> 16) & 0xff)] + - psum[0x300+((x1 >> 24) & 0xff)] + - psum[0x400+((x1 >> 32) & 0xff)] + - psum[0x500+((x1 >> 40) & 0xff)] + - psum[0x600+((x1 >> 48) & 0xff)] + - psum[0x700+((x1 >> 56) )]; - my_stripe_total += psum[ (o1 & 0xff)] + - psum[0x100+((o1 >> 8) & 0xff)] + - psum[0x200+((o1 >> 16) & 0xff)] + - psum[0x300+((o1 >> 24) & 0xff)] + - psum[0x400+((o1 >> 32) & 0xff)] + - psum[0x500+((o1 >> 40) & 0xff)] + - psum[0x600+((o1 >> 48) & 0xff)] + - psum[0x700+((o1 >> 56) )]; - } - } - - if (did_update) { - dm_stripe[k] += my_stripe; - dm_stripe_total[k] += my_stripe_total; - } - - } - - } - } - -#ifdef _OPENACC - // next iteration will use the alternative space - std::swap(this->embedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracVawUnweightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const uint32_t * const __restrict__ embedded_proportions = this->embedded_proportions; - const TFloat * const __restrict__ embedded_counts = this->embedded_counts; - const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - const uint64_t step_size = SUCMP_NM::UnifracVawUnweightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - const uint64_t filled_embs_els = (filled_embs+31)/32; // round up - - // point of thread -#ifdef _OPENACC - const unsigned int acc_vector_size = SUCMP_NM::UnifracVawUnweightedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async -#else -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - TFloat my_stripe = dm_stripe[k]; - TFloat my_stripe_total = dm_stripe_total[k]; - - const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; - -#pragma acc loop seq - for (uint64_t emb_el=0; emb_el 0) { - TFloat length = lengths[emb]; - TFloat lv1 = length / vaw; - - my_stripe += ((x1 >> ei) & 1)*lv1; - my_stripe_total += ((o1 >> ei) & 1)*lv1; - } - } - } - } - - dm_stripe[k] = my_stripe; - dm_stripe_total[k] = my_stripe_total; - - } - - } - } - -#ifdef _OPENACC - // next iteration will use the alternative space - std::swap(this->embedded_proportions,this->embedded_proportions_alt); -#endif -} - diff --git a/R/unifrac_cpp/unifrac_task.hpp b/R/unifrac_cpp/unifrac_task.hpp deleted file mode 100644 index 7424c8e0e..000000000 --- a/R/unifrac_cpp/unifrac_task.hpp +++ /dev/null @@ -1,577 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include "task_parameters.hpp" -#include -#include -#include -#include -#include -#include - - -#ifndef __UNIFRAC_TASKS -#define __UNIFRAC_TASKS 1 - -#ifdef _OPENACC - -#define SUCMP_NM su_acc - - #ifndef SMALLGPU - // defaultt on larger alignment, which improves performance on GPUs like V100 -#define UNIFRAC_BLOCK 64 - #else - // smaller GPUs prefer smaller allignment -#define UNIFRAC_BLOCK 32 - #endif - -#else - -#define SUCMP_NM su_cpu - - -// CPUs don't need such a big alignment -#define UNIFRAC_BLOCK 16 -#endif - -namespace SUCMP_NM { - - // Note: This adds a copy, which is suboptimal - // But was the easiest way to get a contiguous buffer - // And it does allow for fp32 compute, when desired - template - class UnifracTaskVector { - private: - std::vector &dm_stripes; - const su::task_parameters* const task_p; - - public: - const unsigned int start_idx; - const unsigned int n_samples; - const uint64_t n_samples_r; - TFloat* const buf; - - UnifracTaskVector(std::vector &_dm_stripes, const su::task_parameters* _task_p) - : dm_stripes(_dm_stripes), task_p(_task_p) - , start_idx(task_p->start), n_samples(task_p->n_samples) - , n_samples_r(((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK) // round up - , buf((dm_stripes[start_idx]==NULL) ? NULL : new TFloat[n_samples_r*(task_p->stop-start_idx)]) // dm_stripes could be null, in which case keep it null - { - TFloat* const ibuf = buf; - if (ibuf != NULL) { -#ifdef _OPENACC - const uint64_t bufels = n_samples_r * (task_p->stop-start_idx); -#endif - for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { - double * dm_stripe = dm_stripes[stripe]; - TFloat * buf_stripe = this->operator[](stripe); - for(unsigned int j=0; jstop-start_idx); -#pragma acc exit data copyout(ibuf[:bufels]) -#endif - for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { - double * dm_stripe = dm_stripes[stripe]; - TFloat * buf_stripe = this->operator[](stripe); - for(unsigned int j=0; j - class UnifracTaskBase { - public: - UnifracTaskVector dm_stripes; - UnifracTaskVector dm_stripes_total; - - const su::task_parameters* task_p; - - const unsigned int max_embs; - TEmb * embedded_proportions; -#ifdef _OPENACC - protected: - // alternate buffer only needed in async environments, like openacc - TEmb * embedded_proportions_alt; // used as temp - public: -#endif - - UnifracTaskBase(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : dm_stripes(_dm_stripes,_task_p), dm_stripes_total(_dm_stripes_total,_task_p), task_p(_task_p) - , max_embs(_max_embs) - , embedded_proportions(initialize_embedded(dm_stripes.n_samples_r,_max_embs)) -#ifdef _OPENACC - , embedded_proportions_alt(initialize_embedded(dm_stripes.n_samples_r,_max_embs)) -#endif - {} - - /* remove - // Note: not const, since they share a mutable state - UnifracTaskBase(UnifracTaskBase &baseObj) - : dm_stripes(baseObj.dm_stripes), dm_stripes_total(baseObj.dm_stripes_total), task_p(baseObj.task_p) {} - */ - - virtual ~UnifracTaskBase() - { -#ifdef _OPENACC - const uint64_t n_samples_r = dm_stripes.n_samples_r; - const uint64_t bsize = n_samples_r * get_emb_els(max_embs); -#pragma acc exit data delete(embedded_proportions_alt[:bsize]) -#pragma acc exit data delete(embedded_proportions[:bsize]) - free(embedded_proportions_alt); -#endif - free(embedded_proportions); - } - - void sync_embedded_proportions(unsigned int filled_embs) - { -#ifdef _OPENACC - const uint64_t n_samples_r = dm_stripes.n_samples_r; - const uint64_t bsize = n_samples_r * get_emb_els(filled_embs); -#pragma acc update device(embedded_proportions[:bsize]) -#endif - } - - static unsigned int get_emb_els(unsigned int max_embs); - - static TEmb *initialize_embedded(const uint64_t n_samples_r, unsigned int max_embs) { - uint64_t bsize = n_samples_r * get_emb_els(max_embs); - - TEmb* buf = NULL; - int err = posix_memalign((void **)&buf, 4096, sizeof(TEmb) * bsize); - if(buf == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(TEmb) * bsize, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } -#pragma acc enter data create(buf[:bsize]) - return buf; - } - - void embed_proportions_range(const TFloat* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb); - void embed_proportions(const TFloat* __restrict__ in, unsigned int emb) {embed_proportions_range(in,0,dm_stripes.n_samples,emb);} - - - - // - // ===== Internal, do not use directly ======= - // - - - // Just copy from one buffer to another - // May convert between fp formats in the process (if TOut!=double) - template void embed_proportions_range_straight(TOut* __restrict__ out, const TFloat* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) const - { - const unsigned int n_samples = dm_stripes.n_samples; - const uint64_t n_samples_r = dm_stripes.n_samples_r; - const uint64_t offset = emb * n_samples_r; - - for(unsigned int i = start; i < end; i++) { - out[offset + i] = in[i-start]; - } - - if (end==n_samples) { - // avoid NaNs - for(unsigned int i = n_samples; i < n_samples_r; i++) { - out[offset + i] = 0.0; - } - } - } - - // packed bool - // Compute (in[:]>0) on each element, and store only the boolean bit. - // The output values are stored in a multi-byte format, one bit per emb index, - // so it will likely take multiple passes to store all the values - // - // Note: assumes we are processing emb in increasing order, starting from 0 - template void embed_proportions_range_bool(TOut* __restrict__ out, const TFloat* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) const - { - const unsigned int n_packed = sizeof(TOut)*8;// e.g. 32 for unit32_t - const unsigned int n_samples = dm_stripes.n_samples; - const uint64_t n_samples_r = dm_stripes.n_samples_r; - // The output values are stored in a multi-byte format, one bit per emb index - // Compute the element to store the bit into, as well as whichbit in that element - unsigned int emb_block = emb/n_packed; // beginning of the element block - unsigned int emb_bit = emb%n_packed; // bit inside the elements - const uint64_t offset = emb_block * n_samples_r; - - if (emb_bit==0) { - // assign for emb_bit==0, so it clears the other bits - // assumes we processing emb in increasing order, starting from 0 - for(unsigned int i = start; i < end; i++) { - out[offset + i] = (in[i-start] > 0); - } - - if (end==n_samples) { - // avoid NaNs - for(unsigned int i = n_samples; i < n_samples_r; i++) { - out[offset + i] = 0; - } - } - } else { - // just update my bit - for(unsigned int i = start; i < end; i++) { - out[offset + i] |= (TOut(in[i-start] > 0) << emb_bit); - } - - // the rest of the els are already OK - } - } - }; - - // straight embeded_proportions - template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_straight(embedded_proportions,in,start,end,emb);} - template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_straight(embedded_proportions,in,start,end,emb);} - template<> inline void UnifracTaskBase::embed_proportions_range(const float* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_straight(embedded_proportions,in,start,end,emb);} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return max_embs;} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return max_embs;} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return max_embs;} - - //packed bool embeded_proportions - template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} - template<> inline void UnifracTaskBase::embed_proportions_range(const float* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+31)/32;} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+31)/32;} - - template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} - template<> inline void UnifracTaskBase::embed_proportions_range(const float* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+63)/64;} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+63)/64;} - - /* void unifrac tasks - * - * all methods utilize the same function signature. that signature is as follows: - * - * dm_stripes vector the stripes of the distance matrix being accumulated - * into for unique branch length - * dm_stripes vector the stripes of the distance matrix being accumulated - * into for total branch length (e.g., to normalize unweighted unifrac) - * embedded_proportions the proportions vector for a sample, or rather - * the counts vector normalized to 1. this vector is embedded as it is - * duplicated: if A, B and C are proportions for features A, B, and C, the - * vector will look like [A B C A B C]. - * length the branch length of the current node to its parent. - * task_p task specific parameters. - */ - - template - class UnifracTask : public UnifracTaskBase { - protected: - // Use one cache line on CPU - // On GPU, shaing a cache line is actually a good thing - static const unsigned int step_size = 16*4/sizeof(TFloat); - -#ifdef _OPENACC - // Use as big vector size as we can, to maximize cache line reuse - static const unsigned int acc_vector_size = 2048; -#endif - - public: - - UnifracTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTaskBase(_dm_stripes, _dm_stripes_total, _max_embs, _task_p) {} - - /* delete - UnifracTask(UnifracTaskBase &baseObj, const TEmb * _embedded_proportions, unsigned int _max_embs) - : UnifracTaskBase(baseObj) - , embedded_proportions(_embedded_proportions), max_embs(_max_embs) {} - */ - - - virtual ~UnifracTask() {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) = 0; - - protected: - static const unsigned int RECOMMENDED_MAX_EMBS_STRAIGHT = 128-16; // a little less to leave a bit of space of maxed-out L1 - // packed uses 32x less memory,so this should be 32x larger than straight... but there are additional structures, so use half of that - static const unsigned int RECOMMENDED_MAX_EMBS_BOOL = 64*32; - - }; - - - template - class UnifracUnnormalizedWeightedTask : public UnifracTask { - public: - static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; - - UnifracUnnormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) - { - const unsigned int n_samples = this->task_p->n_samples; - - zcheck = NULL; - sums = NULL; - posix_memalign((void **)&zcheck, 4096, sizeof(bool) * n_samples); - posix_memalign((void **)&sums , 4096, sizeof(TFloat) * n_samples); -#pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) - } - - virtual ~UnifracUnnormalizedWeightedTask() - { -#ifdef _OPENACC - const unsigned int n_samples = this->task_p->n_samples; -#pragma acc exit data delete(sums[:n_samples],zcheck[:n_samples]) -#endif - free(sums); - free(zcheck); - } - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - protected: - // temp buffers - bool *zcheck; - TFloat *sums; - }; - template - class UnifracNormalizedWeightedTask : public UnifracTask { - public: - static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; - - UnifracNormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) - { - const unsigned int n_samples = this->task_p->n_samples; - - zcheck = NULL; - sums = NULL; - posix_memalign((void **)&zcheck, 4096, sizeof(bool) * n_samples); - posix_memalign((void **)&sums , 4096, sizeof(TFloat) * n_samples); -#pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) - } - - virtual ~UnifracNormalizedWeightedTask() - { -#ifdef _OPENACC - const unsigned int n_samples = this->task_p->n_samples; -#pragma acc exit data delete(sums[:n_samples],zcheck[:n_samples]) -#endif - free(sums); - free(zcheck); - } - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - protected: - // temp buffers - bool *zcheck; - TFloat *sums; - }; - template - class UnifracUnweightedTask : public UnifracTask { - public: - static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_BOOL; - - // Note: _max_emb MUST be multiple of 64 - UnifracUnweightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) - { - const unsigned int bsize = _max_embs*(0x400/32); - sums = NULL; - posix_memalign((void **)&sums, 4096, sizeof(TFloat) * bsize); -#pragma acc enter data create(sums[:bsize]) - } - - virtual ~UnifracUnweightedTask() - { -#ifdef _OPENACC - const unsigned int bsize = this->max_embs*(0x400/32); -#pragma acc exit data delete(sums[:bsize]) -#endif - free(sums); - } - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - private: - TFloat *sums; // temp buffer - }; - template - class UnifracGeneralizedTask : public UnifracTask { - public: - static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; - - UnifracGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - }; - - /* void unifrac_vaw tasks - * - * all methods utilize the same function signature. that signature is as follows: - * - * dm_stripes vector the stripes of the distance matrix being accumulated - * into for unique branch length - * dm_stripes vector the stripes of the distance matrix being accumulated - * into for total branch length (e.g., to normalize unweighted unifrac) - * embedded_proportions the proportions vector for a sample, or rather - * the counts vector normalized to 1. this vector is embedded as it is - * duplicated: if A, B and C are proportions for features A, B, and C, the - * vector will look like [A B C A B C]. - * embedded_counts the counts vector embedded in the same way and order as - * embedded_proportions. the values of this array are unnormalized feature - * counts for the subtree. - * sample_total_counts the total unnormalized feature counts for all samples - * embedded in the same way and order as embedded_proportions. - * length the branch length of the current node to its parent. - * task_p task specific parameters. - */ - template - class UnifracVawTask : public UnifracTaskBase { - protected: -#ifdef _OPENACC - // The parallel nature of GPUs needs a largish step - #ifndef SMALLGPU - // default to larger step, which makes a big difference for bigger GPUs like V100 - static const unsigned int step_size = 32; - // keep the vector size just big enough to keep the used emb array inside the 32k buffer - static const unsigned int acc_vector_size = 32*32*8/sizeof(TFloat); - #else - // smaller GPUs prefer a slightly smaller step - static const unsigned int step_size = 16; - // keep the vector size just big enough to keep the used emb array inside the 32k buffer - static const unsigned int acc_vector_size = 32*32*8/sizeof(TFloat); - #endif -#else - // The serial nature of CPU cores prefers a small step - static const unsigned int step_size = 4; -#endif - - public: - TFloat * const embedded_counts; - const TFloat * const sample_total_counts; - - static const unsigned int RECOMMENDED_MAX_EMBS = 128; - - UnifracVawTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - const TFloat * _sample_total_counts, - unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTaskBase(_dm_stripes, _dm_stripes_total, _max_embs, _task_p) - , embedded_counts(UnifracTaskBase::initialize_embedded(this->dm_stripes.n_samples_r,_max_embs)), sample_total_counts(_sample_total_counts) {} - - - /* delete - UnifracVawTask(UnifracTaskBase &baseObj, - const TEmb * _embedded_proportions, const TFloat * _sample_total_counts, unsigned int _max_embs) - : UnifracTaskBase(baseObj) - , embedded_proportions(_embedded_proportions), embedded_counts(initialize_embedded()), sample_total_counts(_sample_total_counts), max_embs(_max_embs) {} - */ - - - virtual ~UnifracVawTask() {} - - void sync_embedded_counts(unsigned int filled_embs) - { -#ifdef _OPENACC - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - const uint64_t bsize = n_samples_r * filled_embs; -#pragma acc update device(embedded_counts[:bsize]) -#endif - } - - void sync_embedded(unsigned int filled_embs) { this->sync_embedded_proportions(filled_embs); this->sync_embedded_counts(filled_embs);} - - void embed_range(const TFloat* __restrict__ in_proportions, const TFloat* __restrict__ in_counts, unsigned int start, unsigned int end, unsigned int emb) { - this->embed_proportions_range(in_proportions,start,end,emb); - this->embed_proportions_range_straight(this->embedded_counts,in_counts,start,end,emb); - } - void embed(const TFloat* __restrict__ in_proportions, const double* __restrict__ in_counts, unsigned int emb) { embed_range(in_proportions,in_counts,0,this->dm_stripes.n_samples,emb);} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) = 0; - }; - - template - class UnifracVawUnnormalizedWeightedTask : public UnifracVawTask { - public: - UnifracVawUnnormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - const TFloat * _sample_total_counts, - unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - }; - template - class UnifracVawNormalizedWeightedTask : public UnifracVawTask { - public: - UnifracVawNormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - const TFloat * _sample_total_counts, - unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - }; - template - class UnifracVawUnweightedTask : public UnifracVawTask { - public: - UnifracVawUnweightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - const TFloat * _sample_total_counts, - unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - }; - template - class UnifracVawGeneralizedTask : public UnifracVawTask { - public: - UnifracVawGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - const TFloat * _sample_total_counts, - unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - }; - -} - -#endif From 71a9ba5dab1a8691920bcd8d17c81442ee78809f Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 27 Jan 2025 12:33:00 +0200 Subject: [PATCH 06/48] Simplify source files and code --- R/unifrac_cpp/R_interface/rapi_test.R | 8 +-- R/unifrac_cpp/api.cpp | 37 ----------- R/unifrac_cpp/api.hpp | 23 ------- R/unifrac_cpp/biom_interface.hpp | 60 ------------------ .../{unifrac_internal.cpp => propstack.cpp} | 49 +++++---------- R/unifrac_cpp/propstack.hpp | 40 ++++++++++++ R/unifrac_cpp/su_R.cpp | 51 ++++++++++++++-- R/unifrac_cpp/tree.cpp | 16 ++--- R/unifrac_cpp/tree.hpp | 11 ++-- R/unifrac_cpp/{biom.cpp => tse.cpp} | 22 ++----- R/unifrac_cpp/{biom.hpp => tse.hpp} | 42 ++++--------- R/unifrac_cpp/unifrac.cpp | 61 ------------------- R/unifrac_cpp/unifrac.hpp | 28 --------- R/unifrac_cpp/unifrac_internal.hpp | 44 ------------- 14 files changed, 132 insertions(+), 360 deletions(-) delete mode 100644 R/unifrac_cpp/api.cpp delete mode 100644 R/unifrac_cpp/api.hpp delete mode 100644 R/unifrac_cpp/biom_interface.hpp rename R/unifrac_cpp/{unifrac_internal.cpp => propstack.cpp} (57%) create mode 100644 R/unifrac_cpp/propstack.hpp rename R/unifrac_cpp/{biom.cpp => tse.cpp} (74%) rename R/unifrac_cpp/{biom.hpp => tse.hpp} (63%) delete mode 100644 R/unifrac_cpp/unifrac.cpp delete mode 100644 R/unifrac_cpp/unifrac.hpp delete mode 100644 R/unifrac_cpp/unifrac_internal.hpp diff --git a/R/unifrac_cpp/R_interface/rapi_test.R b/R/unifrac_cpp/R_interface/rapi_test.R index 823f9e953..2b6e56b5a 100644 --- a/R/unifrac_cpp/R_interface/rapi_test.R +++ b/R/unifrac_cpp/R_interface/rapi_test.R @@ -4,14 +4,14 @@ library(miaSim) library(ape) library(picante) -source = "R/unifrac_cpp/su_R_s.cpp" +source = "R/unifrac_cpp/su_R.cpp" sourceCpp(source) data(GlobalPatterns, package = "mia") data(esophagus, package = "mia") data(HintikkaXOData, package = "mia") data(Tengeler2020, package = "mia") # This dataset produces divergent values for some reason - Presumably something to do with the tree being unrooted -tse <- Tengeler2020 +tse <- GlobalPatterns rowTree(tse) <- ape::reorder.phylo(rowTree(tse), "cladewise") ts1 <- rowTree(tse) @@ -24,8 +24,8 @@ newick <- readChar(fname, file.info(fname)$size) y <- rowTree_to_bp(ts1) x <- newick_to_bp(newick) -faith <- faith_pd(tse, is.rooted(rowTree(tse))) -x <- estimateDiversity(tse) +faith <- faith_cpp(tse) +x <- estimateDiversity(tse, index="faith") faith2 <- colData(x)$faith faith - faith2 diff --git a/R/unifrac_cpp/api.cpp b/R/unifrac_cpp/api.cpp deleted file mode 100644 index 21f550f6f..000000000 --- a/R/unifrac_cpp/api.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "api.hpp" -#include "biom.hpp" -#include "tree.hpp" -#include "unifrac.hpp" - -#include -#include -#include -#include -#include - -#include -#include - -#include - -using namespace su; -using namespace std; - -std::vector faith_pd_one_off(const Rcpp::S4 & treeSE, bool isRooted){ - - // Check that tree and table are non-empty and match before calling the c++ code - // shear the tree (to contain only the obs in the table?) - Also should be done before the call? - - std::cout << "Start\n"; - su::BPTree tree = su::BPTree(treeSE, isRooted); - std::cout << "Tree ok\n"; - su::tse table = su::tse(treeSE); - std::cout << "Table ok\n"; - - std::vector results = su::faith_pd(table, tree); - std::cout << "Results ok\n"; - - // compute faithpd - return results; - //return std::vector(); -} \ No newline at end of file diff --git a/R/unifrac_cpp/api.hpp b/R/unifrac_cpp/api.hpp deleted file mode 100644 index 528d52109..000000000 --- a/R/unifrac_cpp/api.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifdef __cplusplus -#include -#include -#define EXTERN extern "C" - -#else -#include -#define EXTERN -#endif - -/* compute Faith PD - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * result the resulting vector of computed Faith PD values - * - * faith_pd_one_off returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * table_empty : the table does not have any entries - */ -std::vector faith_pd_one_off(const Rcpp::S4 & treeSE, bool rooted); \ No newline at end of file diff --git a/R/unifrac_cpp/biom_interface.hpp b/R/unifrac_cpp/biom_interface.hpp deleted file mode 100644 index 73e09e0bd..000000000 --- a/R/unifrac_cpp/biom_interface.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2021-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - - -#ifndef _UNIFRAC_BIOM_INTERFACE_H -#define _UNIFRAC_BIOM_INTERFACE_H - -#include -#include - -#include - -//Faith calculations mainly need n_samples, get_obs_data and sample_counts -//sample_counts - OK -//n_samples - OK -//get_obs_data - -namespace su { - class tse_interface { - public: - // cache the IDs contained within the table - std::vector sample_ids; - std::vector obs_ids; - - uint32_t n_samples; // the number of samples - uint32_t n_obs; // the number of observations - std::vector sample_counts; // Counts summed per sample - - /* default constructor - * - * Automatically create the needed objects. - * All other initialization happens in children constructors. - */ - tse_interface() {} - - /* default destructor - * - * Automatically destroy the objects. - * All other cleanup must have been performed by the children constructors. - */ - virtual ~tse_interface() {} - - /* get a dense vector of observation data - * - * @param id The observation ID to fetch - * @param out An allocated array of at least size n_samples. - * Values of an index position [0, n_samples) which do not - * have data will be zero'd. - */ - virtual std::vector get_obs_data(const std::string &id) const = 0; - }; -} - -#endif /* _UNIFRAC_BIOOM_INTERFACE_H */ diff --git a/R/unifrac_cpp/unifrac_internal.cpp b/R/unifrac_cpp/propstack.cpp similarity index 57% rename from R/unifrac_cpp/unifrac_internal.cpp rename to R/unifrac_cpp/propstack.cpp index 1ae4e61fd..7b748bad0 100644 --- a/R/unifrac_cpp/unifrac_internal.cpp +++ b/R/unifrac_cpp/propstack.cpp @@ -8,8 +8,8 @@ */ #include "tree.hpp" -#include "biom_interface.hpp" -#include "unifrac_internal.hpp" +#include "tse.hpp" +#include "propstack.hpp" #include #include @@ -23,8 +23,7 @@ using namespace su; -template -PropStack::PropStack(uint32_t vecsize) +PropStack::PropStack(uint32_t vecsize) : prop_stack() , prop_map() , defaultsize(vecsize) @@ -32,47 +31,36 @@ PropStack::PropStack(uint32_t vecsize) prop_map.reserve(1000); } -template -PropStack::~PropStack() { +PropStack::~PropStack() { } -template -std::vector PropStack::get(uint32_t i) { +std::vector PropStack::get(uint32_t i) { if(prop_map.count(i) > 0){ return prop_map.at(i); } else { - return(std::vector()); + return(std::vector()); } } -template -void PropStack::clear(uint32_t i) { - prop_map[i] = std::vector(); +void PropStack::clear(uint32_t i) { + prop_map[i] = std::vector(); } -template -void PropStack::update(uint32_t node, std::vector vec) { +void PropStack::update(uint32_t node, std::vector vec) { prop_map[node] = vec; } -// make sure they get instantiated -template class su::PropStack; -template class su::PropStack; - - -template -std::vector su::set_proportions(const BPTree &tree, +std::vector su::set_proportions(const BPTree &tree, uint32_t node, - const tse_interface &table, - PropStack &ps, + const tse &table, + PropStack &ps, bool normalize) { - std::vector props = std::vector(); + std::vector props = std::vector(); if(tree.isleaf(node)) { std::string leaf = tree.names[node]; props = table.get_obs_data(leaf); // Here we basically just need the row for the specified node - //std::cout << "l " << props[0] << " " << props[1] << " " << props[2] << "\n"; if (normalize) { //#pragma omp parallel for schedule(static) for(unsigned int i = 0; i < table.n_samples; i++) { @@ -90,7 +78,7 @@ std::vector su::set_proportions(const BPTree &tree, } while(current <= right && current != 0) { - std::vector vec = ps.get(current); // pull from prop map + std::vector vec = ps.get(current); // pull from prop map ps.clear(current); // remove from prop map, place back on stack //#pragma omp parallel for schedule(static) for(unsigned int i = 0; i < table.n_samples; i++) @@ -104,11 +92,4 @@ std::vector su::set_proportions(const BPTree &tree, } ps.update(node, props); return(props); -} - -// make sure they get instantiated -template std::vector su::set_proportions(const BPTree &tree, - uint32_t node, - const tse_interface &table, - PropStack &ps, - bool normalize); +} \ No newline at end of file diff --git a/R/unifrac_cpp/propstack.hpp b/R/unifrac_cpp/propstack.hpp new file mode 100644 index 000000000..42fc46e4a --- /dev/null +++ b/R/unifrac_cpp/propstack.hpp @@ -0,0 +1,40 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#ifndef __FAITH_PROPSTACK +#define __FAITH_PROPSTACK 1 + +#include +#include +#include + +#include "tse.hpp" + +namespace su { + + class PropStack { + private: + std::stack> prop_stack; + std::unordered_map> prop_map; + uint32_t defaultsize; + public: + PropStack(uint32_t vecsize); + virtual ~PropStack(); + void clear(uint32_t i); + void update(uint32_t i, std::vector vec); + std::vector get(uint32_t i); + }; + + std::vector set_proportions(const BPTree &tree, uint32_t node, + const tse &table, + PropStack &ps, + bool normalize = true); +} + +#endif /* __FAITH_PROPSTACK */ diff --git a/R/unifrac_cpp/su_R.cpp b/R/unifrac_cpp/su_R.cpp index c65f06f61..9337057a9 100644 --- a/R/unifrac_cpp/su_R.cpp +++ b/R/unifrac_cpp/su_R.cpp @@ -3,13 +3,56 @@ #include -#include "api.hpp" +#include "tse.hpp" #include "tree.hpp" +#include "propstack.hpp" + +/* Access the C++ implementation of the fast Faith's PD algorithm from R + * + * treeSE an R TreeSummarizedExperiment object. + * faith the resulting vector of computed Faith PD values + * + * This functions makes several assumptions about treeSE: + * + * - It must contain a non-empty counts assay and a non-empty RowTree + * - The RowTree must be sorted in cladewise order + * - The RowTree must be rooted - Unrooted trees can be passed without error, but don't produce correct results + * + // Check that tree and table are non-empty and match before calling the c++ code + // shear the tree (to contain only the obs in the table?) - Also should be done before the call? + // Assure that tree does not contain ids that are not in table + * + */ // [[Rcpp::export]] -Rcpp::NumericVector faith_pd(const Rcpp::S4 & treeSE, bool isRooted){ +Rcpp::NumericVector faith_cpp(const Rcpp::S4 & treeSE){ + + su::BPTree tree = su::BPTree(treeSE); + su::tse table = su::tse(treeSE); + + su::PropStack propstack(table.n_samples); - std::vector results = faith_pd_one_off(treeSE, isRooted); + uint32_t node; + std::vector node_proportions; + double length; + + std::vector results = std::vector(table.n_samples, 0.0); + + // for node in postorderselect + for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { + node = tree.postorderselect(k); + + // get branch length + length = tree.lengths[node]; + + // get node proportions and set intermediate scores + node_proportions = set_proportions(tree, node, table, propstack); + + for (unsigned int sample = 0; sample < table.n_samples; sample++){ + // calculate contribution of node to score + results[sample] += (node_proportions[sample] > 0) * length; + } + } Rcpp::NumericVector faith = Rcpp::NumericVector(results.size()); @@ -17,5 +60,5 @@ Rcpp::NumericVector faith_pd(const Rcpp::S4 & treeSE, bool isRooted){ faith[i] = results[i]; } - return(faith); + return faith; } diff --git a/R/unifrac_cpp/tree.cpp b/R/unifrac_cpp/tree.cpp index 059631c21..b020428ea 100644 --- a/R/unifrac_cpp/tree.cpp +++ b/R/unifrac_cpp/tree.cpp @@ -7,8 +7,7 @@ using namespace su; -BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names, bool rooted) { - isRooted = rooted; +BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names) { structure = input_structure; lengths = input_lengths; @@ -28,9 +27,7 @@ BPTree::BPTree(std::vector input_structure, std::vector input_leng index_and_cache(); } -BPTree::BPTree(const Rcpp::S4 & treeSE, bool rooted) { - - isRooted = rooted; +BPTree::BPTree(const Rcpp::S4 & treeSE) { //Initialize vectors openclose = std::vector(); @@ -47,8 +44,6 @@ BPTree::BPTree(const Rcpp::S4 & treeSE, bool rooted) { const Rcpp::List & rowTree = treeSE.slot("rowTree"); rowTree_to_bp(rowTree); //Also sets the size of nparens - std::cout << "BP ok\n"; - //Resize vectors // resize is correct here as we are not performing a push_back openclose.resize(nparens); @@ -61,14 +56,11 @@ BPTree::BPTree(const Rcpp::S4 & treeSE, bool rooted) { //Builds a vector that lets us find the corresponding indices for each true/false pair structure_to_openclose(); - std::cout << "structure ok\n"; //Get metadata rowTree_to_metadata(rowTree); - std::cout << "metadata ok\n"; //Finalize - index_and_cache(); // This causes a crash for some reason - std::cout << "cache ok\n"; + index_and_cache(); } @@ -101,7 +93,7 @@ BPTree BPTree::mask(std::vector topology_mask, std::vector in_leng } } - return BPTree(new_structure, new_lengths, new_names, isRooted); + return BPTree(new_structure, new_lengths, new_names); } std::unordered_set BPTree::get_tip_names() { diff --git a/R/unifrac_cpp/tree.hpp b/R/unifrac_cpp/tree.hpp index 189036244..a470604b1 100644 --- a/R/unifrac_cpp/tree.hpp +++ b/R/unifrac_cpp/tree.hpp @@ -7,8 +7,8 @@ * See LICENSE file for more details */ -#ifndef __UNIFRAC_TREE_H -#define __UNIFRAC_TREE_H 1 +#ifndef __FAITH_TREE_H +#define __FAITH_TREE_H 1 #include #include @@ -34,13 +34,13 @@ namespace su { * @param input_lengths A vector of double of the branch lengths * @param input_names A vector of str of the vertex names */ - BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names, bool rooted); + BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names); /* constructor from a TreeSummarizedExperiment * * @param treeSE An R treeSE object */ - BPTree(const Rcpp::S4 & treeSE, bool rooted); + BPTree(const Rcpp::S4 & treeSE); ~BPTree(); @@ -122,7 +122,6 @@ namespace su { std::vector select_0_index; // cache of select 0 std::vector select_1_index; // cache of select 1 std::vector excess; - bool isRooted; // Is the tree rooted or not? void index_and_cache(); // construct the select caches void rowTree_to_bp(const Rcpp::List & rowTree); // convert ape tree structure to boolean structure @@ -138,5 +137,5 @@ namespace su { }; } -#endif /* UNIFRAC_TREE_H */ +#endif /* __FAITH_TREE_H */ diff --git a/R/unifrac_cpp/biom.cpp b/R/unifrac_cpp/tse.cpp similarity index 74% rename from R/unifrac_cpp/biom.cpp rename to R/unifrac_cpp/tse.cpp index 38e8fd978..8dd4dd91c 100644 --- a/R/unifrac_cpp/biom.cpp +++ b/R/unifrac_cpp/tse.cpp @@ -12,7 +12,7 @@ #include #include -#include "biom.hpp" +#include "tse.hpp" #include @@ -63,10 +63,9 @@ void tse::create_id_index(std::vector &ids, } } -//Basically just gets the row for the specified id -template -std::vector tse::get_obs_data_TT(const std::string &id, TFloat t) const { - std::vector out = std::vector(); + +std::vector tse::get_obs_data(const std::string &id) const { + std::vector out = std::vector(); uint32_t idx = obs_id_index.at(id); for(unsigned int i = 0; i < n_samples; i++) { out.push_back(assay(idx, i)); @@ -74,19 +73,6 @@ std::vector tse::get_obs_data_TT(const std::string &id, TFloat t) const return out; } -std::vector tse::get_obs_data(const std::string &id) const { - double t = 0.0; - return(tse::get_obs_data_TT(id, t)); -} - - -//Returns a pointer-based array - can perhaps be changed to simply referring to the R object's internal storage? -//What exactly does this array contain? It contains n_samples elements which are doubles. -//I'm fairly sure that it just sums the counts over samples. Basically just get a column sum. -//std::vector uses move semantics so it shouldn't affect memory usage too much -//the R representation is inherently 'dense' so we can just iterate over the columns -//Might be useful to store? - std::vector tse::get_sample_counts() { std::vector sample_counts = std::vector(); diff --git a/R/unifrac_cpp/biom.hpp b/R/unifrac_cpp/tse.hpp similarity index 63% rename from R/unifrac_cpp/biom.hpp rename to R/unifrac_cpp/tse.hpp index 1e257f2d1..e3076a3d8 100644 --- a/R/unifrac_cpp/biom.hpp +++ b/R/unifrac_cpp/tse.hpp @@ -7,20 +7,25 @@ * See LICENSE file for more details */ - -#ifndef _UNIFRAC_BIOM_H -#define _UNIFRAC_BIOM_H +#ifndef __FAITH_TSE_H +#define __FAITH_TSE_H 1 #include #include -#include "biom_interface.hpp" - #include namespace su { - class tse : public tse_interface { + class tse { public: + // cache the IDs contained within the table + std::vector sample_ids; + std::vector obs_ids; + + uint32_t n_samples; // the number of samples + uint32_t n_obs; // the number of observations + std::vector sample_counts; // Counts summed per sample + /* default constructor * * @param treeSE An R TreeSummarizedExperiment object @@ -31,7 +36,7 @@ namespace su { * * Temporary arrays are freed */ - virtual ~tse(); + ~tse(); /* get a dense vector of observation data * @@ -52,23 +57,6 @@ namespace su { */ std::unordered_map obs_id_index; std::unordered_map sample_id_index; - - /* load ids from an axis - * - * @param path The dataset path to the ID dataset to load - * @param ids The variable representing the IDs to load into - */ - void load_ids(const char *path, std::vector &ids); - - /* load the index pointer for an axis - * - * @param path The dataset path to the index pointer to load - * @param indptr The vector to load the data into - */ - void load_indptr(const char *path, std::vector &indptr); - - /* count the number of nonzero values and set nnz */ - void set_nnz(); /* create an index mapping an ID to its corresponding index * position. @@ -78,12 +66,8 @@ namespace su { */ void create_id_index(std::vector &ids, std::unordered_map &map); - - // templatized version - template std::vector get_obs_data_TT(const std::string &id, TFloat t) const; - }; } -#endif /* _UNIFRAC_BIOM_H */ +#endif /* __FAITH_TSE_H */ diff --git a/R/unifrac_cpp/unifrac.cpp b/R/unifrac_cpp/unifrac.cpp deleted file mode 100644 index fba907972..000000000 --- a/R/unifrac_cpp/unifrac.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include "tree.hpp" -#include "biom_interface.hpp" -#include "unifrac.hpp" -#include "unifrac_internal.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -using namespace su; - -// Computes Faith's PD for the samples in `table` over the phylogenetic -// tree given by `tree`. -// Assure that tree does not contain ids that are not in table -std::vector su::faith_pd(tse_interface &table, - BPTree &tree) { - PropStack propstack(table.n_samples); // construction seems to go okay - - - uint32_t node; - std::vector node_proportions; - double length; - - std::vector results = std::vector(table.n_samples, 0.0); - - // for node in postorderselect - for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { - node = tree.postorderselect(k); - - // get branch length - length = tree.lengths[node]; - - // get node proportions and set intermediate scores - - node_proportions = set_proportions(tree, node, table, propstack); - - for (unsigned int sample = 0; sample < table.n_samples; sample++){ - // calculate contribution of node to score - results[sample] += (node_proportions[sample] > 0) * length; - //std::cout << node_proportions[sample] << " " << (node_proportions[sample] > 0) * length << " " << results[sample] << " - "; - } - } - return results; - //return(std::vector()); -} diff --git a/R/unifrac_cpp/unifrac.hpp b/R/unifrac_cpp/unifrac.hpp deleted file mode 100644 index 242b84e7d..000000000 --- a/R/unifrac_cpp/unifrac.hpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include -#include -#include -#include -#include - -#include - -#ifndef __UNIFRAC - -#include "biom_interface.hpp" -#include "tree.hpp" - - namespace su { - std::vector faith_pd(tse_interface &table, su::BPTree &tree); - } - -#define __UNIFRAC 1 -#endif diff --git a/R/unifrac_cpp/unifrac_internal.hpp b/R/unifrac_cpp/unifrac_internal.hpp deleted file mode 100644 index 1a68597c2..000000000 --- a/R/unifrac_cpp/unifrac_internal.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#ifndef __UNIFRAC_INTERNAL -#define __UNIFRAC_INTERNAL 1 - -#include -#include -#include - -#include "biom_interface.hpp" -#include "unifrac.hpp" - -namespace su { - - template - class PropStack { - private: - std::stack> prop_stack; - std::unordered_map> prop_map; - uint32_t defaultsize; - public: - PropStack(uint32_t vecsize); - virtual ~PropStack(); - void clear(uint32_t i); - void update(uint32_t i, std::vector vec); - std::vector get(uint32_t i); - }; - - template - std::vector set_proportions(const BPTree &tree, uint32_t node, - const tse_interface &table, - PropStack &ps, - bool normalize = true); - -} - -#endif From ff0702ff182dbb9e89b3557d1a8c9ecac8e7ee30 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 3 Feb 2025 04:26:20 +0200 Subject: [PATCH 07/48] Modify folder structure to fit Rcpp guidelines --- R/unifrac_cpp/Makefile | 196 -------- R/unifrac_cpp/R_interface/README.html | 441 ------------------ R/unifrac_cpp/R_interface/README.md | 39 -- R/unifrac_cpp/R_interface/test.biom | Bin 33800 -> 0 bytes R/unifrac_cpp/R_interface/test.tre | 1 - .../R_interface/rapi_test.R | 2 +- R/unifrac_cpp/su_R.cpp => src/faith_R.cpp | 0 {R/unifrac_cpp => src}/propstack.cpp | 0 {R/unifrac_cpp => src}/propstack.hpp | 0 {R/unifrac_cpp => src}/tree.cpp | 0 {R/unifrac_cpp => src}/tree.hpp | 0 {R/unifrac_cpp => src}/tse.cpp | 0 {R/unifrac_cpp => src}/tse.hpp | 0 13 files changed, 1 insertion(+), 678 deletions(-) delete mode 100644 R/unifrac_cpp/Makefile delete mode 100644 R/unifrac_cpp/R_interface/README.html delete mode 100644 R/unifrac_cpp/R_interface/README.md delete mode 100644 R/unifrac_cpp/R_interface/test.biom delete mode 100644 R/unifrac_cpp/R_interface/test.tre rename {R/unifrac_cpp => src}/R_interface/rapi_test.R (97%) rename R/unifrac_cpp/su_R.cpp => src/faith_R.cpp (100%) rename {R/unifrac_cpp => src}/propstack.cpp (100%) rename {R/unifrac_cpp => src}/propstack.hpp (100%) rename {R/unifrac_cpp => src}/tree.cpp (100%) rename {R/unifrac_cpp => src}/tree.hpp (100%) rename {R/unifrac_cpp => src}/tse.cpp (100%) rename {R/unifrac_cpp => src}/tse.hpp (100%) diff --git a/R/unifrac_cpp/Makefile b/R/unifrac_cpp/Makefile deleted file mode 100644 index 0ecac3ec8..000000000 --- a/R/unifrac_cpp/Makefile +++ /dev/null @@ -1,196 +0,0 @@ -H5CXX := h5c++ - -PLATFORM := $(shell uname -s) -COMPILER := $(shell ($(H5CXX) -v 2>&1) | tr A-Z a-z ) - -ifdef DEBUG - OPT = -O0 -DDEBUG=1 --debug -g -ggdb -else - ifneq (,$(findstring gcc,$(COMPILER))) - OPT = -O4 - TGTFLAGS = -fwhole-program - else - OPT = -O3 - endif -endif - -ifeq ($(PREFIX),) - PREFIX := $(CONDA_PREFIX) -endif - -ifeq ($(PLATFORM),Darwin) - AVX2 := $(shell sysctl -a | grep -c AVX2) - LDDFLAGS = -dynamiclib -install_name @rpath/libssu.so -else - AVX2 := $(shell grep "^flags" /proc/cpuinfo | head -n 1 | grep -c avx2) - LDDFLAGS = -shared -endif - -EXEFLAGS = - -MPFLAG = -fopenmp - -LDDFLAGS += $(MPFLAG) -CPPFLAGS += $(MPFLAG) - -ifeq ($(PERFORMING_CONDA_BUILD),True) - CPPFLAGS += -mtune=generic -else - CPPFLAGS += -mfma -march=native -endif - -CPPFLAGS += -Wextra -Wno-unused-parameter - -ifeq ($(PLATFORM),Darwin) - BLASLIB=-llapacke -lcblas -else - BLASLIB=-lcblas -endif - - -LDDFLAGS += -L$(CONDA_PREFIX)/lib -CPPFLAGS += -Wall -std=c++11 -pedantic -I. $(OPT) -fPIC -L$(CONDA_PREFIX)/lib - -ifeq ($(PLATFORM),Darwin) - LDDFLAGS += -Wl,-rpath,$(PREFIX)/lib -else - LDDFLAGS += -Wl,-rpath-link,$(PREFIX)/lib -endif -BASE_LDDFLAGS = $(LDDFLAGS) - -R_LDFLAGS = -llz4 $(BLASLIB) - -UFCMP_LIBS=libssu_cpu.so -UFCMP_LINK=-lssu_cpu -ifdef ACC_CXX - # Tell the generic code we will be building the ACC code, too - CPPFLAGS += -DUNIFRAC_ENABLE_ACC=1 - - UFCMP_LIBS+= libssu_acc.so - UFCMP_LINK+= -lssu_acc - - R_LDFLAGS += -lssu_acc - - ACC_CPPFLAGS += -mp -acc - ACC_CPPFLAGS += -Wall -std=c++11 -pedantic -I. -fPIC -L$(CONDA_PREFIX)/lib - - ifdef DEBUG - ACC_OPT = -g - else - ACC_OPT = -fast - endif - ACC_CPPFLAGS += $(ACC_OPT) - - ifeq ($(PERFORMING_CONDA_BUILD),True) - ACC_CPPFLAGS += -ta=tesla:ccall - else - ACC_CPPFLAGS += -ta=tesla - endif - # optional info - ACC_CPPFLAGS += -Minfo=accel - - # use the GNU OMP library to avoid conflicts - ACC_LDDFLAGS = -shared -mp -acc -Wl,-rpath-link,$(PREFIX)/lib -L$(CONDA_PREFIX)/lib -lgomp -Bstatic_pgi - - ifeq ($(PERFORMING_CONDA_BUILD),True) - ACC_CPPFLAGS += -tp=px - endif -endif - -ifeq ($(PLATFORM),Darwin) - TEST_DEPS = -lssu -else - TEST_DEPS = -lssu -lssu_internal -endif - -test_su: test_su.cpp libssu.so - $(H5CXX) $(CPPFLAGS) $(EXEFLAGS) test_su.cpp -o test_su $(TEST_DEPS) -lpthread - -test_ska: test_ska.cpp libssu.so - $(H5CXX) $(CPPFLAGS) $(EXEFLAGS) test_ska.cpp -o test_ska $(TEST_DEPS) -lpthread - -test_api: test_api.cpp libssu.so - $(H5CXX) $(CPPFLAGS) $(EXEFLAGS) test_api.cpp -o test_api $(TEST_DEPS) -lpthread - -test: test_su test_ska test_api - # test = (su,ska,api) - -ssu: su.cpp libssu.so - $(H5CXX) $(CPPFLAGS) $(EXEFLAGS) su.cpp -o ssu -lssu -lpthread - cp ssu ${PREFIX}/bin/ - -faithpd: faithpd.cpp libssu.so - $(H5CXX) $(CPPFLAGS) $(EXEFLAGS) faithpd.cpp -o faithpd -lssu -lpthread - cp faithpd ${PREFIX}/bin/ - -main: ssu faithpd - # main == (ssu,faithpd) - -rapi_test: main - mkdir -p ~/.R - if [ -e ~/.R/Makevars ] ; \ - then \ - echo "WARNING: OVERWRITING ~/.R/Makevars" ; \ - echo "The original Makevars file has been copied to ~/.R/Makevars" ;\ - cp ~/.R/Makevars Makevars-original ; \ - fi; - echo CXX1X=h5c++ > ~/.R/Makevars - echo CXX=h5c++ >> ~/.R/Makevars - echo CC=h5c++ >> ~/.R/Makevars - echo LDFLAGS=$(R_LDFLAGS) >> ~/.R/Makevars - Rscript R_interface/rapi_test.R - -ifeq ($(PLATFORM),Darwin) - -# We never use ACC under MacOS, so keep it simple - -libssu.so: tree.o biom.o unifrac.o unifrac_internal.o unifrac_cmp_cpu.o cmd.o skbio_alt.o api.o - $(H5CXX) $(LDDFLAGS) -o libssu.so tree.o biom.o unifrac.o unifrac_internal.o unifrac_cmp_cpu.o cmd.o skbio_alt.o api.o -lc -lhdf5_cpp -llz4 $(BLASLIB) - cp libssu.so ${PREFIX}/lib/ - -else - -libssu.so: biom.o unifrac.o cmd.o api.o $(UFCMP_LIBS) libssu_internal.so - $(H5CXX) $(LDDFLAGS) -o libssu.so biom.o unifrac.o cmd.o api.o $(UFCMP_LINK) -lssu_internal -lc -llz4 - cp libssu.so ${PREFIX}/lib/ - -libssu_internal.so: tree.o skbio_alt.o unifrac_internal.o - $(H5CXX) $(LDDFLAGS) -o libssu_internal.so tree.o skbio_alt.o unifrac_internal.o -lc -lhdf5_cpp $(BLASLIB) - cp libssu_internal.so ${PREFIX}/lib/ - -libssu_cpu.so: unifrac_cmp_cpu.o libssu_internal.so - $(CXX) $(BASE_LDDFLAGS) -o libssu_cpu.so unifrac_cmp_cpu.o -lssu_internal -lc - cp libssu_cpu.so ${PREFIX}/lib/ - -libssu_acc.so: unifrac_cmp_acc.o libssu_internal.so - $(ACC_CXX) $(ACC_LDDFLAGS) -o libssu_acc.so unifrac_cmp_acc.o -lssu_internal -lc - cp libssu_acc.so ${PREFIX}/lib/ - -endif - -api: libssu.so - # api == libssu.so - -capi_test: api - gcc -std=c99 capi_test.c -lssu -L${PREFIX}/lib -Wl,-rpath,${PREFIX}/lib -o capi_test - export LD_LIBRARY_PATH="${PREFIX}/lib":"./capi_test" - -api.o: api.cpp api.hpp unifrac.hpp skbio_alt.hpp biom.hpp tree.hpp - $(H5CXX) $(CPPFLAGS) api.cpp -c -o api.o -fPIC - -unifrac.o: unifrac.cpp unifrac.hpp unifrac_internal.hpp unifrac_cmp.hpp biom_interface.hpp tree.hpp - $(CXX) $(CPPFLAGS) -c $< -o $@ - -unifrac_cmp_cpu.o: unifrac_cmp.cpp unifrac_cmp.hpp unifrac_internal.hpp unifrac.hpp unifrac_task.cpp unifrac_task.hpp biom_interface.hpp tree.hpp - $(CXX) $(CPPFLAGS) -Wno-unknown-pragmas -c $< -o $@ - -unifrac_cmp_acc.o: unifrac_cmp.cpp unifrac_cmp.hpp unifrac_internal.hpp unifrac.hpp unifrac_task.cpp unifrac_task.hpp biom_interface.hpp tree.hpp - $(ACC_CXX) $(ACC_CPPFLAGS) -c $< -o $@ - - -%.o: %.cpp %.hpp - $(H5CXX) $(CPPFLAGS) -c $< -o $@ - -clean: - -rm -f *.o *.so ssu faithpd test_su test_api test_ska - diff --git a/R/unifrac_cpp/R_interface/README.html b/R/unifrac_cpp/R_interface/README.html deleted file mode 100644 index d83388128..000000000 --- a/R/unifrac_cpp/R_interface/README.html +++ /dev/null @@ -1,441 +0,0 @@ - - - - - - - - - - - - - -README - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - -
-

R interface for Strided State Unifrac

-

This provides an R interface for Unweighted Unifrac. This interface -works using R’s Rcpp library. To load this, in R use -library(Rcpp) and sourceCpp("su_R.cpp"). The -Unifrac method takes in three arguments: a file path to an HDF5 -formatted BIOM table, a filepath to a newick formatted tree file, and -the number of threads to be used It is expected that the observations -described in the BIOM table correspond to a subset of the tips of the -input tree. The method returns a list containing an int -n_samples, denoting the number of samples in the table, a -boolean is_upper_triangle, denoting whether -Unifrac generated a square matrix and if it has returned the upper -triangle , an int cf_size, denoting the size -of the condensed form of the matrix, and c_form, an array -representation of the condensed form of the matrix, obtained by taking -the upper triangle.

-
> library(Rcpp)
-> sourceCpp("su_R.cpp")
-> table = "../test.biom"
-> tree = "../test.tre"
-> nthreads = 2
-> unif = unifrac(table, tree, nthreads)
-> unif
-$n_samples
-[1] 6
-
-$is_sqaure
-[1] TRUE
-
-$cf_size
-[1] 15
-
-$c_form
- [1] 0.2000000 0.5714286 0.6000000 0.5000000 0.2000000 0.4285714 0.6666667
- [8] 0.6000000 0.3333333 0.7142857 0.8571429 0.4285714 0.3333333 0.4000000
-[15] 0.6000000
-
-
- - - - -
- - - - - - - - - - - - - - - diff --git a/R/unifrac_cpp/R_interface/README.md b/R/unifrac_cpp/R_interface/README.md deleted file mode 100644 index 3fb7e2d51..000000000 --- a/R/unifrac_cpp/R_interface/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# R interface for Strided State Unifrac - -This provides an R interface for Unweighted Unifrac. This interface works using -R's Rcpp library. To load this, in R use `library(Rcpp)` and -`sourceCpp("su_R.cpp")`. The Unifrac method takes in three arguments: a file -path to an HDF5 formatted BIOM table, a filepath to a newick formatted tree -file, and the number of threads to be used It is expected that the observations -described in the BIOM table correspond to a subset of the tips of the input -tree. The method returns a list containing an `int` `n_samples`, denoting the -number of samples in the table, a `boolean` `is_upper_triangle`, denoting -whether Unifrac generated a square matrix and if it has returned the upper -triangle , an `int` `cf_size`, denoting the size of the condensed form of the -matrix, and `c_form`, an array representation of the condensed form of the -matrix, obtained by taking the upper triangle. - -```R -> library(Rcpp) -> sourceCpp("su_R.cpp") -> table = "../test.biom" -> tree = "../test.tre" -> nthreads = 2 -> unif = unifrac(table, tree, nthreads) -> unif -$n_samples -[1] 6 - -$is_sqaure -[1] TRUE - -$cf_size -[1] 15 - -$c_form - [1] 0.2000000 0.5714286 0.6000000 0.5000000 0.2000000 0.4285714 0.6666667 - [8] 0.6000000 0.3333333 0.7142857 0.8571429 0.4285714 0.3333333 0.4000000 -[15] 0.6000000 - -``` - diff --git a/R/unifrac_cpp/R_interface/test.biom b/R/unifrac_cpp/R_interface/test.biom deleted file mode 100644 index b3c019bf8515f8e05176fac398e0cf114281bc82..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 33800 zcmeI5O-vg{6oB8_2HZdbsj8Ixc3Y(>J(yr{g2N#pzqo2dXhS1%N$dp!mH;DTBB!E} zQL2hYtrV$J6;(Z?Kap}o>Lu!lG^vyxD&^1vm2yCB#i@rxttu*2N@sT7TjIs*B?(S2 zek0-8nR)YO=9?dew=?GB$dO~~*KJt`()s=1gF5q(J|3d;f2d+8OzBa4(9oYk|GGy# zXs`}sT0O?sp?=+~A~Gn~{qE7DAS6jY`Irr&sDSiPJp1KAAae9zmuzVdHdjag4vrrT zLedzEr_ROFiR6eGH5MBl9g545rCgW&{=Kj=})GHW9h(nYRHUs z)wfBAQN85UtN|0Z&{r#->jCwhi>Jm+v~{8lsu}HO0h*>-OWk4jiT`3f*PHbmh>yfm zv2?sIV4RnV+}d3wWT>7#@w}*SgQ>4K6*oIS&?l5GL*JH1^i|~Ot2Em^HW)KIWV!8? zI;5*qp3u=`K51fQtr5gVM&38WOZ1_9E&7@uFPYJLN8@k?3z(B2gmYUr2e&CX(xXEB zSuO!!8-;Tl97#A1<3WI(KAV`kKFJRK8LC`Rc6spR-*U@?~FvrYmIe#hYx$4yze zAPPtMuD*Vf?~B*POewd4W`qhZM%0HiR5rV ztx=81)BuRtrW{Tah-cI<0-B$4-b>fq6buELS_94dx`TV$f_vIREsZU~a7$~eN~d{D zxH>w{p6Gt>Y>4#Q;`dOPe3WmY_SoW^)y9j~QUH_>5+5q(q*?utSVNS{jo$<7^TzLm z4SC~R#HyZ8Pc#0u1IX1ML}`F9DYRnvfeA1HCcp%k025#WOn?b60Vco%m;k`Z&J#y8 z@Sv>_;0pc$PIdshJ&WvAnRgUDlDmrGnUm#7)yW=sceCPgf2;D{?1A@pDLLX?o;1t1 z)pmQJjwtHTPE2IT_LmsxOZ3LaK=`BS)S~^)vLjLk7hkOdK4?I>8=dNb=Jy9fxZ8t)5l=a#Dyhsj z=~8r~e!zNP6RL*LLo}pTh*+8K@hY{|pqmf)I!8RqbPf9A{X_DN3fWctsMqW9%0i#j35|Cn?dD`;_aQtRKBECjSkGBEmEPF`YPB6?e$T(M*5|FaajO z1eieS5>We4ueDK`-d?yZv=0q86_%s@WoBPtxwVp-Inx`3$=ZLSmk!?64&7R)>DpiQ zMfj6V*-N*6xjC2~zY>DU4{ExFn308JdVi&{N$;;Pb~U_m`=g&O-fo|$-wM8O;dQA; zIG?dspZ6JO^!X|Sv@ao$IrF=Sj?6{D+o)FtV-uaB{ZohLzx(RJuaosK5cZrlYoKkc zYWi^ot{s^DtmD?-)$NH54WBkXytxBrRpEU?Nu_bD;pZPfJ06+1yltxX>)6ocZQ$Dp zI+pD*PRX)I^m)y=pjUz?3auh(0E^f3{*K@N`sTsEcNgvrPV5Nn-(Ba0&;RiJxmLQ8 z+rb2w025#WOkmjr9^VdCC#*E$T(mH>C>xS)mbC)*rwE=-lcpRpPvx;c^_b-q>%++o z^{PXGXC3TNRs$UJaWoTP0!)AjFaah|x&++zPjv5qe6Ki}U!?uhRGY#mi?X&KOW)r; zpZycr)$Bc;we6pXlhpuw1`aa;Ccp%k025#WB~QR@|3vpZ$)|HrPWQ`Kzz%iiu%ar9 zq}PUd>`>449Mo$LJ$=QteN;He4N@1wT&6jn+5&^tz9IoeNfC-c?0k{1V{XYoui+CTUepq3YMOoXA)v|xWui0GE z(f`@@PsGV;z>*vs!33B96JP>NfC-d30k{1V-A^T--h&qLK1$Cy1yvSFX2a+H9(1Qe zN5A}LZUS+#DxRCnAtt~Cm;e)C0!&~H5O6wAYCjL>cAlTlldgfm@Dxq3pC_erboaBl mO6go?S8ntZhn&W*{TwNe-zP>~78#w!?=+r6CI~A%e*XvBCayjJ diff --git a/R/unifrac_cpp/R_interface/test.tre b/R/unifrac_cpp/R_interface/test.tre deleted file mode 100644 index 401910e9c..000000000 --- a/R/unifrac_cpp/R_interface/test.tre +++ /dev/null @@ -1 +0,0 @@ -((((((((((((((((((t866:0.8313991029,t1687:0.3733365205):0.6214818014,(t3269:0.6072595785,t1686:0.7529063553):0.8813435675):0.2431555497,(((t2720:0.1146772795,t3354:0.0820990426):0.6279464895,(t1508:0.6595000362,t3157:0.5381378131):0.6036559318):0.9752331681,(t2936:0.2551624496,t3000:0.05830493267):0.2189676561):0.6016189293):0.6093351725,(t1409:0.06985254632,(((t3454:0.8042654931,(t4167:0.2065694619,t4059:0.3304703967):0.09274322307):0.1545934496,(t1963:0.8274634348,t4111:0.9223411726):0.1472488912):0.950080327,(t3893:0.9364278358,(t1832:0.9313541986,t3662:0.3542085034):0.9250311363):0.8204278164):0.2623061107):0.8409996089):0.05529688462,(t333:0.4158636159,t1767:0.5100084862):0.190012262):0.6363199549,((t937:0.1614484172,t4631:0.7026415281):0.9756822931,t3887:0.001032893546):0.2980521361):0.6100336898,t907:0.3821411079):0.1475626735,(((t1562:0.6701928617,(t3716:0.2011611089,t1016:0.3637938015):0.4790978113):0.6990956133,(((((t1107:0.6637271889,t2840:0.7656728218):0.517867452,t3661:0.4304010128):0.06343328184,((t3192:0.9852911846,t4126:0.3187654694):0.5443374345,t2336:0.8467219928):0.5904103168):0.1341467793,(((t2414:0.203484318,(t412:0.8888965137,((t1212:0.2144512064,(t4862:0.4041996542,t4283:0.03360790829):0.5145890177):0.708200976,t1790:0.9912529374):0.3069289338):0.7088423646):0.2650691359,t2795:0.3816543671):0.1875626091,((t2757:0.2675529977,t2965:0.4615569487):0.1818874718,((t4621:0.7309411902,t2619:0.849376152):0.987313323,t4410:0.3897798315):0.7727495972):0.4278311483):0.8209839279):0.005038926611,(((t675:0.1684947007,t1676:0.5381485247):0.7439194887,t159:0.4282397088):0.06210065517,t1678:0.4105465328):0.5742849279):0.3442878388):0.7197340357,(((t3296:0.320776033,t1552:0.7741130814):0.8737377529,t1738:0.1963442313):0.9038650303,((t2940:0.4459364363,(((t402:0.5717571357,t591:0.9301284561):0.636806218,((t3490:0.5776866795,t1304:0.4995453237):0.6047905236,t3083:0.7596928983):0.3777288443):0.3528932505,(t2372:0.8104584259,t4045:0.357436178):0.5526586915):0.3186718693):0.3538908418,((((t1876:0.8001238538,t277:0.9216230945):0.9068796132,((t758:0.6737462839,t886:0.1913464598):0.5354478974,(t1289:0.5047061227,t3972:0.7230309916):0.9546566226):0.7594838168):0.1208872946,t1267:0.03093142342):0.7116731715,(((t2896:0.6969843917,(t1879:0.8709465866,t1347:0.8908235182):0.1870362924):0.2875384493,t4303:0.5906736872):0.3181333772,(t1430:0.09074050607,((t2559:0.7986080786,t2581:0.2758466483):0.9652930978,t1368:0.1860396883):0.5633136057):0.7996817871):0.2368269465):0.1223834648):0.9785036482):0.1878155204):0.5875749625):0.6865005223,((((t930:0.1181095932,(t107:0.8333898354,(t3892:0.9600333602,t4747:0.4963327439):0.6568270572):0.7843931997):0.07698448002,t4362:0.3409487531):0.4231897073,((((((t529:0.04959728243,(t640:0.2147065429,t3690:0.2545317535):0.1329500256):0.7216223071,((t1673:0.6350189031,t3293:0.6712232393):0.6078203127,t4114:0.6816496283):0.7059944186):0.81562798,((t1752:0.1870343112,t1934:0.9348407013):0.4338724373,(((t4911:0.3117885669,t1768:0.9695757565):0.2889108541,t550:0.2876443232):0.9507387944,t3871:0.6463846464):0.9826903841):0.4953189236):0.3031298388,(t3993:0.8383750333,t3406:0.1060100815):0.8212537798):0.5591524122,(t2948:0.6562415836,(t2734:0.8999172021,(t1930:0.9211484026,t216:0.3518829397):0.2120475902):0.9303359622):0.3419920085):0.3668603834,(t1333:0.9122814275,t4649:0.6358980944):0.6920417589):0.6566554087):0.643499925,((((t1412:0.4486003867,t755:0.7555005001):0.3969241746,t1656:0.7176111068):0.6536936334,t4807:0.3639599544):0.118114522,(((t2120:0.9193203296,t1639:0.7436536453):0.483570592,((t3770:0.7877304046,t4979:0.06589412759):0.08375591203,(t4736:0.3814637288,t2052:0.3728892156):0.2851516092):0.0205797716):0.02622013958,t1084:0.9509191096):0.405228748):0.561063454):0.2318455605):0.1753108837,(((((t2646:0.12860857,(t1622:0.3871864763,(t267:0.8130197593,t913:0.6790412655):0.6844102743):0.05539356847):0.7421562918,((t1883:0.7993078497,((t3238:0.08534995257,t3603:0.1386850916):0.6240059605,(t3570:0.3704532592,t3492:0.4279712306):0.2013301745):0.02492270293):0.4694976134,t2392:0.6688378071):0.2137457884):0.2944884764,((((t1013:0.3968843073,((t4864:0.9482195915,t2061:0.3587614582):0.123466908,(t3766:0.612816578,t2902:0.08410423715):0.7703144071):0.4174562509):0.7675790566,(t1163:0.7610452443,(t2815:0.1097113073,(t2884:0.1855741069,t910:0.9298445894):0.06679133931):0.1308139546):0.8947804321):0.547407127,((((t3512:0.3123587975,t538:0.1997674562):0.0274069747,t1816:0.7843595201):0.518524423,(t495:0.2170114561,t1073:0.3725154614):0.1091845315):0.7535025117,((t897:0.8108963056,(t1137:0.693028131,t3033:0.518422103):0.4107569826):0.6008314493,t373:0.1386215745):0.9438396601):0.2902637431):0.5673053069,((((t1193:0.7558353972,t4176:0.3539376841):0.9614218306,((t4720:0.1573807234,t2818:0.8950932315):0.2238170567,t4291:0.3897347897):0.8315451082):0.9570792422,((t1601:0.3854548566,t2367:0.501024422):0.5245058786,(t1293:0.5080356814,t1278:0.1580390946):0.2233934382):0.9534459224):0.08009390417,(((t3761:0.3336408893,t2426:0.3739440746):0.03781672195,(((t4264:0.6076085074,(t3159:0.2337467058,((t4580:0.05722609977,t4074:0.1294833701):0.01331580174,(t660:0.107948028,t4994:0.3645382558):0.7614466671):0.8255050587):0.2813613035):0.07591580786,((t444:0.4889106292,((t848:0.8255068781,t2916:0.8064365082):0.9674706035,t4376:0.1939471222):0.1076038913):0.4793788702,t3356:0.2319602058):0.9083877718):0.05033111176,(t2276:0.05197685934,t620:0.5648543704):0.5495417307):0.9923970785):0.5722151687,(t2495:0.5662884507,((t3402:0.447677698,t4573:0.5410352189):0.229000591,((((t1222:0.758369239,t3167:0.3261838197):0.9416625514,t1659:0.04069067468):0.354748891,(t1484:0.5904282497,t4031:0.2863436346):0.02740597283):0.06279171817,t3090:0.2235296504):0.6806243469):0.4830548868):0.2040736931):0.3763294795):0.2272899386):0.3996402575):0.3156907097,(((t1349:0.4669708272,((t3318:0.9977994349,(t4585:0.007838176563,(t677:0.3869352438,(t4316:0.08979598852,t2193:0.530110654):0.6284318732):0.4792206499):0.5654797545):0.2588597636,(t4909:0.3334113173,t2011:0.8704887505):0.3976832477):0.1914889184):0.4892883177,((((t2464:0.002572481288,t3482:0.2588424592):0.04849005351,((t4233:0.2146493508,t3351:0.442112681):0.9735390993,t1900:0.899863557):0.8962532862):0.9084300497,(t4976:0.7234419258,(t4137:0.08894893457,t284:0.3383265315):0.2182320838):0.5241291963):0.1267592236,(t3338:0.09958937974,(t2081:0.09586508386,t4513:0.1785695285):0.03012143169):0.1738754734):0.6673604408):0.08750341623,((t2742:0.1556912579,(t4020:0.2850743537,t4359:0.9686899327):0.08356065536):0.3017442834,(t2097:0.02868690901,t3850:0.5237277283):0.9745898312):0.650786048):0.7844161796):0.172100046,(((t3899:0.8545834734,(((t1969:0.2020547255,t4950:0.8966408703):0.4309893181,(t1584:0.4535946152,((t3526:0.9639999992,t449:0.658337642):0.08031757222,(t3100:0.6227267974,(((t4494:0.1360878113,t694:0.9735614178):0.2148821759,(t2201:0.3367159995,t2539:0.9978895185):0.2363615001):0.9136687769,((t3701:0.3443828623,t198:0.8449363422):0.5596893611,(t205:0.1922649972,t4414:0.6131308302):0.998525921):0.7875632562):0.9447442961):0.094211024):0.09039981849):0.0942905338):0.3552417455,((((t3674:0.3699161999,t4430:0.5583526609):0.3636858882,(t3680:0.3514062983,t3129:0.3560908763):0.77759952):0.8376044335,((t4809:0.7839681495,((t73:0.8458224731,t832:0.02920071315):0.5244866598,(t234:0.1105332496,t3436:0.6492459632):0.9089311964):0.143582796):0.8149665371,(t994:0.1810347207,t4717:0.3273448183):0.3870441106):0.6867058265):0.6309708101,(t238:0.2335639638,(((t2829:0.6940432303,((t2078:0.5121471782,t643:0.9560798111):0.2380270909,t4077:0.9250784921):0.09147981368):0.4799288791,t3989:0.5019952122):0.8400811804,((t588:0.2799502863,(t2762:0.8330721518,t3995:0.2293385617):0.8058388657):0.6619244178,((t2277:0.03030529013,(t1103:0.417617064,t1628:0.7531720959):0.2904342064):0.9922625371,t1640:0.4167473505):0.7044557438):0.7944476453):0.01753724366):0.7580058079):0.1679762842):0.4854042039):0.6624881688,((((t4488:0.04132898641,((t3362:0.5572877568,t2428:0.01979416958):0.1260109774,(t4244:0.2547936849,t895:0.727302857):0.9251294867):0.1607061664):0.6319261612,(t370:0.9452616326,t343:0.06567107118):0.7694829579):0.3651739305,(((t3928:0.5820202043,t684:0.8006292286):0.722828218,t1733:0.6531603646):0.1611002039,((t2364:0.6117900054,t3740:0.7533898307):0.4633000528,t1406:0.8537637237):0.2380282378):0.9738505706):0.1402269127,((t2286:0.8612355865,(t2915:0.4517645189,t3306:0.7157090202):0.09892280772):0.09952634852,t4156:0.5061485644):0.8300967051):0.4571315604):0.6464888712,((((t4620:0.3717226819,t2398:0.4639170691):0.9069901905,(t1695:0.4494498344,t208:0.9338418769):0.3048537313):0.5753543843,(((t1408:0.3348016278,t2050:0.6979630752):0.07184065925,t4349:0.4378902102):0.2745788111,(((t109:0.5284655394,t1647:0.6463780864):0.9625444652,t3034:0.8764540514):0.9901140176,(t1290:0.7389137307,(t3106:0.397667265,t394:0.6857739007):0.3048634599):0.1374744785):0.5048887343):0.2137722648):0.9844458397,((((t2055:0.6795427711,t1986:0.197292648):0.2835855929,(t1498:0.7202436905,(t2503:0.9803528474,t1747:0.0265964584):0.3603068111):0.1440980178):0.2559785696,(t37:0.8919375236,(t2715:0.0001679745037,(t4916:0.6175938013,(t1922:0.665739248,(t4121:0.02225081855,(t4116:0.6575521091,t1315:0.6776197732):0.2996522097):0.2809470573):0.2966181994):0.1982580761):0.2591595659):0.7671651486):0.5420052521,t1173:0.5751995419):0.5408020962):0.09271502146):0.694344946):0.8642061879):0.4587742444,(((((t3705:0.3077488469,(t2683:0.3824585932,(t1031:0.492785126,t3137:0.6769487069):0.3193018534):0.4485238849):0.5818818845,(t3988:0.3966336916,(t361:0.6868546931,(t1684:0.7659938822,t4279:0.08717331616):0.5659515623):0.195033377):0.6608112429):0.6918420675,t4089:0.1229582443):0.5935022221,((((((t1759:0.05596134486,(t1499:0.7477170415,t2058:0.06373910606):0.08335241559):0.5618415982,(((t3092:0.4642444279,t122:0.3389575589):0.1902088546,(t3165:0.9979357789,(t1777:0.8572698389,t3686:0.3864898013):0.4624512338):0.2065343531):0.1222046658,(((t1467:0.2027243823,t785:0.5290673098):0.5791875266,(t1192:0.8304317927,t4285:0.868538467):0.8903516214):0.2709914381,t3784:0.6495769273):0.6268899313):0.4878179457):0.8749882865,((t499:0.3672033937,t2018:0.5924020063):0.8508136154,t250:0.0178107298):0.280602918):0.1264096648,t3560:0.344473344):0.1474100943,((((((((t2756:0.4914784722,(t3875:0.6713702285,t4329:0.1033865751):0.04412020883):0.6977141672,(t3691:0.1388739347,t1593:0.05767298979):0.2015471144):0.6463584441,((t4977:0.4073140486,t3702:0.05452403403):0.6306369812,(t1887:0.1061515368,(t3759:0.7181896823,t4882:0.5734710088):0.7374663292):0.1986337157):0.6505447451):0.2866082962,t2526:0.2340005359):0.3219974504,(t13:0.2337360389,t1791:0.2469362153):0.6399987563):0.05970709957,((t1064:0.3444667293,t4800:0.9214540147):0.528797467,(t3472:0.8128262968,t4327:0.5139588483):0.1445924458):0.7723366928):0.03411449934,((t4710:0.7008015877,(t1960:0.082305711,(t340:0.1880501874,t3439:0.1643805823):0.3643843364):0.2740489526):0.8135308253,((((t3398:0.2251789223,((t1445:0.4711492194,t4705:0.6156857519):0.6195273409,t485:0.8202841245):0.7617390025):0.9760007358,t679:0.2885267076):0.1918063064,t4452:0.3304648919):0.5278047847,(t145:0.8493148971,t1903:0.6771793943):0.5363263367):0.1391534698):0.307604121):0.5899123165,((((t4995:0.6808952286,(t2171:0.4406946807,t1848:0.3042994367):0.1491642012):0.4726482916,((t2925:0.6971553112,t725:0.1424836561):0.9414467306,(t2:0.04259154852,t2673:0.804576179):0.1907947834):0.7031442164):0.02356514777,((((t300:0.8495152174,((t1911:0.3079968672,t221:0.7993741685):0.8231502238,t4735:0.03782425635):0.3011443706):0.7581174325,(((t1183:0.2796791503,t479:0.703756982):0.3783575273,((t4253:0.7978001726,(t2290:0.2940325185,t4441:0.5233497315):0.4365729503):0.9907537305,((t4300:0.9818675399,t2282:0.2242055268):0.2348890863,(t4778:0.02383773611,t4899:0.4475373442):0.6738085677):0.3906179508):0.8571994142):0.1506352832,(t4975:0.09785921429,((t768:0.7420771497,t1965:0.7000401868):0.4112272328,t2003:0.07539929613):0.7877485934):0.8536505855):0.6284328497):0.4422431237,t4388:0.9619839406):0.06630207878,t4939:0.8316900609):0.8284955923):0.6957386476,((t126:0.2745769685,t318:0.515695011):0.8739172786,(t1806:0.4223237687,(t4235:0.6324440965,(t3558:0.3856438636,t100:0.1793875324):0.1103117014):0.6897586144):0.9797722392):0.07411500579):0.9114833896):0.1744418826):0.12724513,(((((t3575:0.2915715522,t2959:0.6054766681):0.2958707309,(t341:0.1462142987,(t2872:0.06567132124,(t4787:0.6459316753,t3186:0.3659973063):0.1336052534):0.4502089962):0.6560140238):0.7582268999,(t4984:0.5793036509,(((t3254:0.9246715868,t1862:0.4961994721):0.03015597421,(t4075:0.2329804187,t4443:0.01785692992):0.6949531741):0.752783861,((t2032:0.00182481599,t3081:0.2864647491):0.1948995111,(t1642:0.1777490822,(t4802:0.6750068944,t4822:0.9309473911):0.224436945):0.2201781396):0.1557234703):0.4853850524):0.8829760635):0.6089727767,(((((t3936:0.7378455061,(t2897:0.04230677406,t3949:0.1348307354):0.05313787027):0.3296164989,(((t3846:0.6774868004,t1854:0.8158837501):0.9599367611,(t138:0.8174523709,t3594:0.4482949905):0.665435472):0.6561711624,(t2941:0.9655892011,(t450:0.3526771103,t1975:0.8324833733):0.5348272631):0.5999133568):0.6406183194):0.6497490988,t1146:0.3918073028):0.8136717316,(((t1196:0.589256949,(t2107:0.9461472598,(t1078:0.08970693359,t2101:0.07596769161):0.5243451649):0.6314494654):0.7436189905,(((t1968:0.4095283882,t515:0.3888608604):0.1900956177,(t3762:0.4513915717,(t4458:0.05377129768,t2264:0.320200067):0.644646083):0.363355296):0.635781954,t272:0.844973563):0.1486973958):0.6539557155,((t2591:0.3961518875,t3425:0.4591378972):0.3509312924,t3772:0.7132560285):0.5538575212):0.7374948005):0.5583165744,(((t3437:0.5564842799,(t46:0.6903763306,t4938:0.0871099371):0.8569924438):0.01571567683,(((t4370:0.05613637832,t3307:0.2048978412):0.119604972,t3120:0.2856470875):0.7677832902,(t3407:0.4573363201,t1322:0.3836230936):0.1984996686):0.1137238124):0.6174768275,(t4730:0.03508606879,t2523:0.1745297287):0.7856323458):0.0273360936):0.1670195637):0.6524346559,(((t3667:0.9109485524,(t1152:0.6157433027,t1939:0.1478762547):0.4132024939):0.4341737363,(t945:0.09898446617,t4841:0.752896589):0.8002180101):0.8601645206,((t4478:0.544979098,(t4165:0.9513458982,t3528:0.274565479):0.6006933185):0.7104016929,t4635:0.8616838527):0.4258347338):0.161475023):0.3792145243):0.8458303383):0.1501436951,(((t2056:0.06283738953,t130:0.7353540463):0.3818885721,(t3431:0.4171808963,(t3289:0.6297804855,t812:0.6145251342):0.3355520647):0.3989941175):0.4776869144,t1610:0.5099943201):0.2747247163):0.5808562574):0.1173578743,(((((((((t633:0.7878978646,t4257:0.2668918124):0.058420585,t3858:0.3574693177):0.3867737106,(((t713:0.6144745413,t225:0.7666423055):0.9468104623,t3854:0.4185951184):0.8965026038,(((t1046:0.03403013782,(t4872:0.7471951458,t2751:0.3780142374):0.9674805063):0.4954787777,(t1620:0.3943154635,(t1835:0.3586856071,t766:0.1593085309):0.4414879608):0.6448796524):0.7341502274,(t321:0.06573951617,t78:0.9645420606):0.9772567479):0.1642606161):0.5015719032):0.009362907847,t843:0.9306513865):0.9098140819,t4133:0.1726960964):0.1627949611,((((((t3506:0.1555005824,(t2291:0.2415448872,t2218:0.6229850096):0.9834204547):0.0766422525,(((t2740:0.4675714979,t3267:0.5807356378):0.07195233577,t1839:0.3615350355):0.6904575126,t2152:0.0539963597):0.7748350685):0.5079670688,(t3105:0.762835742,t2725:0.783796466):0.9584548178):0.4012050428,((((((((t1295:0.4164196772,t2954:0.6073149303):0.2107647646,t2483:0.2546401017):0.5298662703,((t4601:0.4726096373,t2845:0.9699278455):0.1594873061,(t628:0.7959083684,t472:0.5038546089):0.2468013354):0.2405225963):0.2042652881,t2719:0.3846707568):0.9881216274,(t263:0.2444275522,(t1540:0.002669940703,(t2890:0.4910230506,t3113:0.1967816416):0.2833629383):0.6500252585):0.9340893866):0.4419223468,((t2604:0.4997490097,t2689:0.1908188728):0.397136529,((t1230:0.1887639058,t2606:0.6170728437):0.6400903785,t4779:0.05593576096):0.2662141183):0.04496151092):0.4585990051,t1598:0.4488763234):0.6067683478,((t1386:0.1602443401,t2467:0.2520359186):0.07220733934,(t2764:0.7036789996,t231:0.1407988446):0.6025765829):0.04527508747):0.4518896716):0.440780648,(((t4427:0.7291558969,t392:0.9098402502):0.324533311,(((t241:0.876763883,t617:0.7776813521):0.4155768962,(((t2014:0.2481024927,t966:0.5110307785):0.9683095682,(t797:0.006647384958,t3194:0.5751418984):0.7965638384):0.6731909767,((t1383:0.08988061198,t4905:0.3317841429):0.4784023084,t4354:0.4192292877):0.130420354):0.4951545659):0.6413437445,(t1254:0.8138528576,(t585:0.9455490054,(t2662:0.3696457266,t547:0.1125562578):0.9929754266):0.1361916685):0.5261483763):0.4947226157):0.6115640856,((((t561:0.3359035361,t4271:0.7219727971):0.6940529414,t2105:0.5294406756):0.796080841,(t2287:0.03226945712,t2982:0.5769619986):0.3669758472):0.7853436787,(t4773:0.6322223386,t976:0.6932521139):0.9681658773):0.1409364128):0.6987744232):0.03979036654,((((t1929:0.830397408,((t1051:0.02677565673,(((t3498:0.6135061702,t224:0.1955287277):0.6820306573,(t4776:0.1175722748,t1251:0.7412909265):0.2617493835):0.3223988065,t3377:0.4271865219):0.04042899003):0.6740744531,t1572:0.6555063031):0.9782287439):0.1330191679,((t1757:0.7890949878,t1039:0.573943957):0.5220484147,((t4961:0.8228227056,(t2732:0.02673610114,t1188:0.4030509922):0.5501612416):0.03512003901,t557:0.6656008158):0.9630228325):0.02071409463):0.03710257704,((t4058:0.8863248506,(((t4144:0.08660087571,(t3256:0.1293852923,t125:0.4784358384):0.1715465183):0.3329215932,t445:0.7716247349):0.2207148937,t69:0.5560288329):0.5100735258):0.7043115939,(t3136:0.5859558817,t133:0.8939665779):0.7172426854):0.5837200184):0.8319789858,t3069:0.5247035872):0.8014201785):0.1257841513):0.8516854437,(((((((((t3721:0.2272730847,t2524:0.7267203361):0.2073178284,t426:0.0979973576):0.2254999974,t958:0.755704436):0.3981743976,((t291:0.4075606614,t3639:0.8241939354):0.8926953345,(t2251:0.5661321448,t2157:0.7918906126):0.1668497066):0.548846822):0.9142585853,(t2184:0.07892514626,((t842:0.09095380595,(t2309:0.2447845724,t3066:0.3357016074):0.1155967922):0.6148679897,(t4920:0.067015148,t3388:0.6636415964):0.03361500497):0.5691323599):0.4181748333):0.4819492891,((((((t1844:0.8891954913,t44:0.1263200638):0.2889231648,t689:0.6517695789):0.9710279307,((t3043:0.3420520991,t4734:0.1014613379):0.08970208839,t1928:0.1404719288):0.2330005949):0.5202469786,t3527:0.3073822088):0.7455507996,((((t2873:0.993842721,t38:0.8716940666):0.9702959431,t4451:0.811977949):0.3911699778,t4219:0.8243935166):0.8491760052,((t1723:0.6181446293,(t3320:0.4218083268,t2378:0.5709121886):0.5222427058):0.6756005385,((t3117:0.4757901349,(t1667:0.03000710253,t1563:0.2898409474):0.7336020495):0.8879907588,(t1795:0.05369648803,t3545:0.5110886015):0.03172167391):0.5453401254):0.3470024748):0.2373066347):0.1232472439,((t2045:0.09022754454,((((((t2221:0.06582522835,t559:0.3750349162):0.9823703724,t2857:0.91887657):0.3647457191,t4769:0.4148848066):0.2218487649,t2599:0.2610909671):0.117852998,(((t3623:0.2997155234,(t2384:0.2523896666,((t4418:0.3204169911,t4210:0.6035194506):0.1483694732,t2862:0.1070660686):0.382917634):0.1145242173):0.8360710598,t831:0.6974025108):0.9243301519,((t1145:0.1953724711,(((t2462:0.02957250248,t2420:0.7349497643):0.4452858581,t4450:0.5839691546):0.833532796,t4169:0.975084123):0.6798284068):0.9351414097,(((t3315:0.5507207906,t3260:0.01705354918):0.7518011779,t518:0.9714504066):0.5806051695,(t1564:0.7447547193,t4506:0.5968745956):0.2476783139):0.4340931242):0.4297363679):0.7803913141):0.6385204501,((((t2904:0.7438715228,t3027:0.7617073336):0.2069750784,t2889:0.1843298285):0.4033136379,t4479:0.7930720742):0.2364758458,(t313:0.8746632556,(((t1462:0.01935528778,(((t4791:0.8398026014,(t2301:0.3197807434,t1038:0.4504582621):0.8727893296):0.8012714216,(t1551:0.227017323,t3554:0.8384136984):0.7671152134):0.636139391,t2369:0.3707443958):0.607876146):0.9290237429,((t4206:0.8721011363,(t229:0.5312422623,((t288:0.7717724796,t552:0.2046030276):0.4665413366,(t4193:0.6593752354,t3097:0.06107479008):0.4428279179):0.2099730223):0.9599901461):0.4818946796,t1997:0.4558941461):0.1404252206):0.8787578936,t3216:0.716959503):0.1899293421):0.2787984787):0.9139637328):0.8343013593):0.6285247961,(((t1028:0.9913741208,t88:0.7136295764):0.165553801,t1565:0.7906226674):0.06138058635,(t2731:0.9538947833,t641:0.4709784789):0.8748916266):0.02064642939):0.5488434534):0.516574932):0.5289227734,((((t1755:0.9561335959,(t3279:0.7558561666,t43:0.4442185273):0.2835182364):0.7017092011,((t1186:0.2810068065,t302:0.8027740314):0.5610312901,(((t4574:0.890135447,(t3265:0.2827178964,(t1623:0.9504110247,t1030:0.7499874001):0.8045510368):0.2839150925):0.6179945848,(t3904:0.762395394,t3763:0.8478405366):0.9658707301):0.2542268585,(t4927:0.7666204537,t2354:0.8009463164):0.20428839):0.918566271):0.3655805727):0.8375776128,t1797:0.6570838718):0.796043894,(((((t154:0.8041998569,t2016:0.5771751257):0.2804442178,t4055:0.6514532836):0.7740242362,t4161:0.2860580245):0.2508791196,t2860:0.1996267657):0.7764814005,((t1424:0.378427458,t4587:0.9449776406):0.5118783186,((t4857:0.5625643381,(t2643:0.7123449885,t489:0.7666595853):0.03049568855):0.1289974691,(t49:0.6853816502,(t1391:0.7640693565,t532:0.3948962637):0.809085659):0.3425395396):0.9466729695):0.8294895785):0.04192236811):0.04168931209):0.7047592879,(((((((t22:0.1062093936,((t1608:0.5267388825,t651:0.1217423705):0.5899810842,((t4287:0.5227646604,t2521:0.5014049015):0.8743285942,t1241:0.9668636306):0.4575376154):0.03729532915):0.2942101231,(t1444:0.4691511907,(t4198:0.3791185436,(t1334:0.8677457774,t1740:0.6761152355):0.8855947256):0.2441808309):0.5557023743):0.9297091982,(((t2994:0.07313275663,t3835:0.590317568):0.6060474706,t4380:0.03807822103):0.7266947783,t1214:0.1094867818):0.5525156616):0.3165107092,((((t1689:0.8687409447,t482:0.2549425911):0.5812358698,(t303:0.4390058708,t3353:0.5086138793):0.3762504952):0.01645651134,(t4571:0.04501585197,(t4575:0.2743993851,((t3727:0.6485891575,t3872:0.6394018608):0.08933446277,(t2164:0.4854876618,t4368:0.9923337493):0.6518029273):0.2402326344):0.7225597147):0.1010865043):0.5105761525,(t1875:0.06032745005,t816:0.5409573084):0.008607164258):0.2561405229):0.3016850099,((((t1523:0.8558590831,t4840:0.294845476):0.2742795113,((t2859:0.7109305528,t616:0.1376374094):0.2209257833,((t4358:0.7087082041,(t1416:0.1961699151,t4523:0.2965116177):0.8787157112):0.8280309597,(t3902:0.05308877141,t3852:0.4151797926):0.235782797):0.7114830553):0.9277588653):0.7383752821,t4561:0.2057104243):0.03686085646,t1609:0.9181670593):0.1685556176):0.05538826296,t2031:0.7053138402):0.7046458928,(((t3226:0.0102605545,t2117:0.3897832988):0.6513349356,t3401:0.5885010974):0.8343021218,(((((t4934:0.1817306485,(t2123:0.7340407188,t2454:0.6104232282):0.1886806476):0.8746837541,t4986:0.2951512481):0.4614188874,t2195:0.2262022588):0.06382826786,((t3501:0.9004146492,t3446:0.5758411724):0.9606548897,(t347:0.3655740756,(t3622:0.758681197,t3665:0.9648439176):0.5645550238):0.4364375959):0.9828783802):0.4975256296,(((t2803:0.2527835162,t898:0.3529181536):0.1843255248,(t27:0.6882137144,t4046:0.02794211241):0.7162194757):0.4623265967,(((t4609:0.814070601,((t4062:0.1450867835,t4828:0.09026327729):0.4442945814,(t4771:0.737855226,t4404:0.05066533503):0.006584957242):0.5666926212):0.302958742,(((t1585:0.564349792,t664:0.09179626498):0.966538725,t2147:0.5980658231):0.8023969457,t2508:0.06598238577):0.691382552):0.442074684,t3294:0.64118041):0.6807374794):0.7692557098):0.7764729986):0.442435506):0.8189105245):0.9433969869,((t4929:0.04752131505,t2012:0.9867965637):0.9948739554,t2455:0.04473616625):0.1120181805):0.1489457684):0.6889146112,(((((t476:0.2264062176,t4684:0.2312254407):0.3889585189,t1644:0.7865779568):0.9279815371,t4113:0.5871377555):0.8840677089,t3847:0.4483252489):0.2803470888,(((t2512:0.6852147239,(t35:0.06016547373,t908:0.6581263512):0.5731073944):0.385760973,t3849:0.333136999):0.6110870452,(t534:0.8452889619,(t3517:0.4495121788,t2119:0.4179212325):0.663919671):0.6861887071):0.2076667706):0.7435675948):0.4227561087,((t3788:0.3898302007,((t1119:0.4965937783,t3589:0.3181228347):0.9094769817,((t256:0.3326093024,t2085:0.1543390448):0.7657084488,t2248:0.6890798693):0.588582275):0.5533623886):0.5621604042,(((((t709:0.03085775184,(t2111:0.2596664443,t2810:0.6147328482):0.995723933):0.3582475041,t4889:0.9339648089):0.5444564556,((t3832:0.783544641,t2767:0.104425156):0.4558330271,(t986:0.6722481819,(t4871:0.1085624942,t1604:0.7791883154):0.4663334307):0.395669759):0.3671582295):0.3892007871,(t86:0.6587422066,(((t2113:0.9756236989,(t4881:0.7471210759,t3655:0.3530885589):0.469051074):0.824929679,((t1921:0.3051331437,(t4021:0.9797260971,t2208:0.8523167975):0.4366853214):0.5225139929,t3580:0.0625502828):0.3908204229):0.3321972652,t2066:0.4542768572):0.8504227449):0.9353928568):0.07003547181,((((t3838:0.7098855111,(t4554:0.8891642259,t2288:0.3118351218):0.3736097808):0.161281171,t927:0.3298595808):0.3784930583,(t781:0.6679566738,t849:0.6259508296):0.1130717548):0.2727484729,t4065:0.9912141578):0.1440196547):0.2147689259):0.7043501181):0.07248589955):0.530497049,(((((((t3769:0.6466846738,t672:0.1033081454):0.6266500417,(((t955:0.2273717627,(t3352:0.5579966898,t1339:0.01964822924):0.3156101445):0.3591132183,((t1745:0.1167644225,t632:0.5405399653):0.6658276576,t3050:0.76286367):0.7346331722):0.5164513744,((t1878:0.4324891842,(t4087:0.3476188192,t2584:0.7207941094):0.6210414676):0.2790375431,((t1996:0.1042214141,t3344:0.5070581199):0.2879082335,t4645:0.8558622005):0.4922160259):0.03505943157):0.6815868346):0.4029777001,((((t460:0.4704633558,t172:0.2671403964):0.686816779,(t2323:0.1825801868,t1010:0.1803271479):0.6091568777):0.2921220942,(((t1724:0.3313620081,t4464:0.007205237402):0.6120055735,(t1894:0.3802170276,(t1420:0.2573057683,t1697:0.7206129674):0.9375002088):0.1827242058):0.1120939164,((t2794:0.4763097398,t85:0.1901003367):0.08367978805,(t2781:0.5407154397,t3262:0.04304612358):0.2294769997):0.8126289344):0.08153182222):0.3462535993,(((t3823:0.5676665765,t3619:0.7623456235):0.05221901299,(((t875:0.8342969231,t4510:0.01280949032):0.4805617358,((t3237:0.868317923,t2516:0.3559935484):0.2391033964,(t96:0.2634183168,t3078:0.5413043841):0.4104156578):0.6433246587):0.7133132869,(((t916:0.4961097147,t1710:0.5212233118):0.3681945752,(t4060:0.4648038673,(t494:0.4682004587,t4558:0.2630725261):0.9225172596):0.8050615776):0.4532664379,(t4079:0.1588851346,t2602:0.09221257851):0.4393938517):0.03791397042):0.9239859581):0.2933065468,(t3022:0.6358551804,(t729:0.2333274339,t3212:0.1765346471):0.6913841155):0.148070453):0.357521517):0.1009253073):0.1452931252,((((t2548:0.2236656782,(t815:0.8213350857,((t1055:0.8640718691,t4634:0.6664643853):0.7207097509,t2589:0.2309196405):0.4839087222):0.6354335675):0.837207197,(t1926:0.3605173982,t4132:0.8602008261):0.2074110394):0.7975981371,(((t4650:0.6114310657,t1480:0.5927743833):0.7775010511,t1815:0.7387591815):0.093524531,((((t3952:0.3417194458,t3025:0.3344225893):0.3593524352,t1458:0.9698847521):0.6445904365,t4591:0.5833607835):0.8164989168,(t462:0.1087539014,t2621:0.00215459778):0.5499014652):0.5343262032):0.2478236798):0.3348515588,(t4627:0.4438257637,(t3544:0.4862496899,t3644:0.3581603325):0.6639149655):0.798017679):0.06999673764):0.5962453596,(((t1022:0.3090051059,((t1052:0.3328659611,t2115:0.3883270295):0.9024150395,t3579:0.1267078514):0.3432275653):0.5559913013,t90:0.4690397023):0.2694137404,(((t2452:0.2666485338,t4151:0.8441270906):0.1241979003,t3658:0.1196388621):0.2930851004,t1613:0.8959609733):0.9758957163):0.3256739243):0.318873889,((((((t959:0.3091299231,t3548:0.6373040627):0.877055404,t2240:0.9956788805):0.05453642411,((((t1110:0.4505025849,t3478:0.9588692153):0.9506132605,(t1262:0.7979903871,t1798:0.8417904859):0.8500949503):0.03177290736,(t3616:0.8490903568,t4107:0.7680919052):0.01018177648):0.1415458149,(t2280:0.03998941439,t4367:0.738323306):0.3047480215):0.4852210497):0.4157199706,t3568:0.01037793071):0.7840320186,(((t786:0.07067953167,t4476:0.2343864425):0.07798373513,((t1366:0.9256579953,(t3280:0.9086080925,t1168:0.00243516732):0.9665693233):0.9192321084,t3394:0.2588717935):0.07120910799):0.7155589776,t583:0.6690537904):0.2810324449):0.5849530844,((t3933:0.6101319897,(((t2788:0.637666493,((((t2432:0.1340591561,t312:0.9343538997):0.903231292,(t4100:0.1527151116,t1677:0.2576732035):0.5319194936):0.6589867629,(t4550:0.559303558,t144:0.0886226478):0.1319019839):0.8748646763,(t1561:0.9570078254,(t2179:0.4851890197,t1477:0.6949043816):0.8570922611):0.8941170226):0.1368041006):0.6804844884,t4378:0.008004796691):0.9268674203,((((((((t3696:0.4309293772,t2573:0.8052610301):0.7855452653,(t868:0.8255533013,t233:0.8835596314):0.3080080952):0.2007256106,(t2789:0.5761511396,t1800:0.8347072047):0.428101057):0.005547654582,t227:0.8791453352):0.4778623141,t4974:0.6944202664):0.7070723181,(t3749:0.3686779568,(t1276:0.3139098885,t4949:0.5140190755):0.0808090663):0.7302102458):0.2413145641,(((t1091:0.3533034031,(t3310:0.1232876047,t2116:0.628836398):0.4255135322):0.007262541912,(t905:0.7864620169,(t2060:0.5953573496,(t4565:0.7373510567,t4296:0.7121330944):0.5411335668):0.229283927):0.6448725243):0.3898677453,t1247:0.005315080052):0.5133132506):0.3200960087,((t4624:0.2680101108,t4426:0.8724194732):0.3928657519,t2448:0.8980655884):0.8232885108):0.6005251962):0.492095921):0.0661570658,((((((((t1892:0.07490668516,t1300:0.7155320027):0.6556291892,t556:0.5938144561):0.9076140674,t1495:0.8580564891):0.1105535126,(t1553:0.6643014061,(t3084:0.1838265697,t114:0.9746948618):0.6744990337):0.2293225429):0.3844142871,t175:0.04850835819):0.4151965023,t933:0.563647782):0.2607754976,((t4508:0.7912175111,((t3634:0.1404989895,t952:0.1977472259):0.04340601573,(t680:0.9767462285,(t2898:0.9942323398,(t4667:0.008750366513,t3204:0.06591424509):0.1737183398):0.0230799003):0.6838204332):0.06915117893):0.1759667194,(t4805:0.3751401543,t1421:0.5349217146):0.08212707122):0.01960918121):0.9615102094,((t1818:0.02239792328,t4493:0.8646344917):0.02888329653,(t1422:0.5962255253,(t332:0.5765972435,t3947:0.9994824568):0.3736898096):0.725945405):0.5272825065):0.9426176234):0.462855007):0.4692642924):0.6231946275,((((((t4584:0.3446172159,t3324:0.2740713169):0.6993737789,((t4416:0.7008554828,(t745:0.7204409756,t4466:0.03135086969):0.6014024185):0.4866102792,t2289:0.1004670695):0.5513202175):0.259507034,(t1784:0.4614843968,((t401:0.3483010333,t1318:0.1402908785):0.3985952279,(t4764:0.8497869105,t663:0.05895327055):0.05699259927):0.2789120059):0.5578124062):0.4990431448,(t4197:0.2207745444,t1190:0.05646117101):0.3738585685):0.7549968797,((((t1701:0.3322706043,(((t1905:0.3732423827,(t757:0.4035894233,((t3242:0.7552078962,t2403:0.5936345106):0.1508447032,(t4178:0.61984832,t3728:0.7685682925):0.3778034118):0.7662743039):0.2142664462):0.117607801,((t1023:0.9588375415,t4861:0.1600330039):0.2320436148,(t3035:0.8584500449,t3125:0.6009808923):0.441564067):0.8285019302):0.6594157638,((t3217:0.487779933,t4642:0.6032790754):0.9606279701,t438:0.6125991954):0.8090999648):0.8741976963):0.5877119387,t3193:0.3671189328):0.2184245391,t3979:0.1792570727):0.3983644147,(((t1165:0.7263523617,t890:0.2832219426):0.07846978703,t4454:0.5662037719):0.1181382251,t47:0.9923386106):0.5804408917):0.220135994):0.007112539373,((t2868:0.2712685561,t3937:0.275737674):0.1641175735,(t4056:0.768098864,((t42:0.2365621892,t3567:0.6706970748):0.1907827624,t4983:0.8764561741):0.7494066874):0.4942596904):0.5996063228):0.1864717037):0.862709363):0.2909562169,(((((t4664:0.9304566951,t1417:0.05257266574):0.9538545979,t715:0.7549683796):0.2979818108,(((t639:0.7151184981,t3234:0.547814091):0.8006778057,(t790:0.01991344942,t4124:0.256110417):0.537971291):0.2506390414,((((t4552:0.5155060624,t4201:0.8163021519):0.3725403999,(t414:0.4584031648,t3991:0.8911658113):0.4593221471):0.7854233689,(t2841:0.4291381338,t292:0.8226222498):0.9112290922):0.9897106818,(t3476:0.4077175015,t1132:0.6122271449):0.8860064833):0.01477308688):0.5107039129):0.888281065,(((t4854:0.1313291453,t526:0.07222869736):0.2509720894,t670:0.4472576221):0.766625968,(((t1793:0.2389741635,(((t4863:0.9820780463,((t4926:0.9223933427,t1884:0.3346033227):0.05232663313,t4001:0.1193057916):0.8076331932):0.7739556783,t3549:0.6004829931):0.8882187577,(((t2239:0.3772761128,t2250:0.2095319733):0.1052006851,t1240:0.6904255881):0.6335157263,t603:0.5203551303):0.9040458892):0.1352078444):0.8636435727,((t143:0.8836502237,t4280:0.3276579389):0.01877145306,t775:0.5463361186):0.05106935976):0.9104602209,(((t734:0.5162792953,t4704:0.7455891052):0.1655608963,(t3857:0.9605378576,t4953:0.9289893166):0.1238029888):0.8203120271,((t2296:0.7620274387,t2969:0.766865572):0.4544611631,((t1011:0.3972623609,t1225:0.05121889338):0.9187192614,(t3898:0.9651806743,((t3403:0.8659662877,t2570:0.08471553889):0.323113448,t4566:0.2656651421):0.4838454106):0.1887069202):0.7914218155):0.2909786284):0.6412001203):0.3802099496):0.3064949738):0.880862786,((((t3341:0.3332408047,t2515:0.1978242584):0.8614687233,(t1461:0.861720603,t3843:0.9288190815):0.3957614873):0.69364743,((t4812:0.7317738635,t3652:0.3375414284):0.565154887,t2544:0.7843509533):0.4330095451):0.7054571616,(((t2541:0.4933483594,((((t3923:0.4715760476,t2755:0.8865578156):0.2502611224,(t119:0.3522350364,t2989:0.1754289723):0.6067595079):0.2132098943,t975:0.212742205):0.2965624079,(t1775:0.3828190726,t665:0.05458779749):0.005589244189):0.2234592594):0.6484751042,(((t375:0.8568919525,t2041:0.7244076359):0.4808880824,((t3774:0.7240769605,(t150:0.3972217536,t3011:0.7937627106):0.9788834793):0.4871354501,t4080:0.988321912):0.3351018999):0.5312720148,((t383:0.01207726309,(t3499:0.2568179797,(t299:0.8896732756,t385:0.2647393481):0.667279324):0.3519880599):0.1533683233,(((t467:0.418780687,t2498:0.6713116074):0.5403868537,t1035:0.2980335103):0.06631625933,(t106:0.5664194273,(t204:0.8044231657,t1527:0.3726060388):0.8450954028):0.09663877636):0.2212472481):0.6314126034):0.9628856818):0.499548621,(((((t3434:0.02921271767,(t661:0.6966851011,t3641:0.342011869):0.8244890231):0.5888657626,((t508:0.4760097784,(t1681:0.2457318238,t4181:0.9818324908):0.6673686649):0.9187927989,((t3020:0.1534836716,(t4135:0.1878414021,t596:0.764049435):0.2463693663):0.0377906668,(((t355:0.830097822,t683:0.631418912):0.7811487305,t4278:0.08528722147):0.2779802149,(t1863:0.1677124577,(t4683:0.6648355403,(t3429:0.4877056612,t3323:0.3100777254):0.02398575284):0.6693547082):0.8219353205):0.462421733):0.8788085955):0.9803359851):0.4286797429,t1942:0.7418841412):0.6818343839,((t475:0.8252504314,(t2655:0.9207634099,((t4729:0.1058409377,t3681:0.647373857):0.5800706963,((t4958:0.6524147743,t2040:0.5286924397):0.2672304942,(((t95:0.1119787451,t3608:0.8565463028):0.8246072433,(t2847:0.5987734403,t4444:0.9840675676):0.2915324722):0.392334224,t4148:0.8568997411):0.6183971874):0.9127077053):0.5213467709):0.7230928952):0.1333552203,(((((t2232:0.2205803443,(t3557:0.2978168651,t1506:0.3850140183):0.06996091548):0.6449486064,(t431:0.6946327062,(t1324:0.1068797028,t4391:0.292209873):0.004252291517):0.1702999678):0.8252241763,(t2062:0.9994597856,t2175:0.213354528):0.74027936):0.3237326844,((t1544:0.5699626633,(t1460:0.4113790395,t899:0.08304075035):0.54801297):0.1936414354,t3798:0.4422166443):0.06655659946):0.2460256666,((t2391:0.8475717674,t4343:0.3584974871):0.342210504,t1050:0.8479827428):0.1725088675):0.8290164387):0.634862249):0.4533321522,((((t4281:0.5646155793,(t2067:0.5218651907,(t653:0.7031459841,(t1104:0.9791517481,t2919:0.5508994241):0.7258873556):0.2585143552):0.8625433333):0.5460079182,((t2441:0.8395343409,t2244:0.1548981154):0.9797443727,(t4162:0.2197723729,t2792:0.9625409972):0.3465276305):0.8702985502):0.7059335867,(((t853:0.8620653683,t4390:0.03419136116):0.9327281343,(t1361:0.2163218269,t1491:0.5646170527):0.6218528966):0.7057344399,(((t567:0.8550386804,t791:0.2068807057):0.7368793134,((t637:0.08879817254,(t1638:0.3268671932,t3064:0.4647028318):0.5596295425):0.6141556352,t473:0.798001904):0.473791338):0.4576970944,((t4782:0.3626284255,t4323:0.1962700405):0.8224465291,(((t3657:0.9684312907,t3861:0.1475586966):0.5796871684,t1967:0.01208710275):0.5235993129,t1469:0.8825382099):0.03749012272):0.5428821018):0.5300997873):0.1519905326):0.8845694123,(((t5000:0.6542834463,t4596:0.3955448049):0.1582832257,((t2502:0.1293041811,(t4757:0.5799618098,t4341:0.6409835743):0.4716792221):0.2549489588,t4796:0.2944373179):0.7562957068):0.4390988136,((((t1177:0.7042823255,t405:0.6745223184):0.3263553947,((t2043:0.7949909673,t4933:0.9306182039):0.5296911064,((t1889:0.7580958125,t4804:0.3421752078):0.1580273192,t4112:0.7942998982):0.9547303247):0.4788878018):0.1389126801,t4463:0.8718663051):0.7158308181,((t2362:0.1835035582,t753:0.6578063713):0.7754279808,t497:0.3557105674):0.1888224029):0.5325981814):0.3214960583):0.1916291704):0.7138409382):0.9680554771):0.5993425802):0.7299016058):0.4045579233,((t4353:0.1107426004,(t2706:0.1409204477,t1158:0.494018099):0.1074948939):0.3680671498,((((((t2471:0.477228865,((((t190:0.6279092438,t1833:0.4570953285):0.1860131605,t1653:0.5785431438):0.4694334727,((t4139:0.3854422728,t4799:0.8529077165):0.5567523644,t599:0.4512025889):0.6665724323):0.7083741869,(t3756:0.8943691009,t1423:0.009874930605):0.5635304069):0.460964622):0.6377704144,(t2681:0.6104491751,(t3057:0.05157451704,t1943:0.2220654311):0.4718299431):0.3266741713):0.3740884187,(((t1870:0.04568355624,(t3139:0.5958024331,(t578:0.3606470337,t1651:0.1003955645):0.6571315897):0.7115604032):0.3396621484,(((t4109:0.737479209,(((t1338:0.7933644499,t2687:0.8810002108):0.37037879,t2578:0.6125737969):0.2917059367,(t3225:0.02518131561,t3175:0.04199181893):0.2581833464):0.5016670155):0.8108378814,(t4449:0.6662035922,t4537:0.5077769589):0.5667670546):0.06295551523,((t4660:0.6646387281,t2071:0.6016214364):0.3256749455,t80:0.7274544344):0.2224477578):0.6685227843):0.9571340487,(t3366:0.2648028,(((t1273:0.7439932788,(((t3218:0.448566528,t326:0.2594898166):0.6261468036,t4205:0.4008740869):0.266143691,(t1449:0.315638005,t2625:0.6102447689):0.1348877717):0.2770228032):0.9484447937,((((((t1648:0.0710146015,(t1244:0.8875683583,t4277:0.9055026341):0.9926929721):0.9354329193,(t3848:0.4621008357,t2233:0.8691533813):0.6097526776):0.9697218987,((t3571:0.3002396666,((t4421:0.7226957385,(((t248:0.6331399546,t3469:0.5242302737):0.5434815402,t2039:0.8367426426):0.7528521668,t4073:0.4847606472):0.9986144644):0.4991449337,(t2555:0.5412526568,t3782:0.4691321114):0.1567828935):0.3965436046):0.5747563448,(t2155:0.2956574522,t3276:0.562233102):0.4276773022):0.9195994914):0.2654775479,((((t4029:0.8716457572,t4551:0.5098797474):0.1592948956,t3625:0.8528882326):0.9010232496,t3706:0.8191902256):0.348420091,((t2506:0.5884979342,t3419:0.4183517862):0.4663748727,t4839:0.4952596903):0.05719872564):0.965208919):0.07442370267,t4869:0.3335710757):0.2371746358,t3537:0.384530978):0.2411411754):0.3683480818,((t3442:0.4070422184,((t3971:0.04083079286,t4697:0.2762315597):0.3199129221,(t1403:0.1763692936,(t3371:0.1556898048,t2249:0.6660126662):0.1512764152):0.803617235):0.473045357):0.9722921816,t991:0.603489609):0.4174242041):0.7283605086):0.5506962622):0.09124358324):0.9464069572,(((((t2400:0.9705576329,((t2104:0.4208654726,t2335:0.4269797488):0.9772462484,((t4952:0.8789142785,t2436:0.2013784337):0.801069631,(t4877:0.3199565767,(t2671:0.1940777826,t3420:0.3889506406):0.8898991027):0.2412282587):0.7203557712):0.8552576851):0.7778617404,((((t1896:0.6637700479,t1754:0.9356380405):0.8179022677,t919:0.9822418476):0.5630755189,(t1787:0.8135371506,t4669:0.01248314418):0.2223929283):0.02853894513,((((t859:0.3312765246,(t2126:0.6147523939,t3775:0.7615870694):0.6444736742):0.2489153333,t3427:0.9375454322):0.2014391206,(t4028:0.3361945648,(t1144:0.5410954524,t3736:0.1249863813):0.3851700076):0.15557622):0.5414525308,(t2793:0.6484910289,(t3360:0.8046605496,t876:0.3311111818):0.4460008682):0.5736540004):0.1743675778):0.8788767331):0.7930357086,(((t990:0.6368215128,((t2545:0.8928038324,t295:0.07223393559):0.6814632406,t730:0.8002198073):0.3362920096):0.8674865395,(t2427:0.01845167112,(((t1486:0.9996755584,t1252:0.2028291838):0.9960522573,t2042:0.04575532489):0.5237805296,(t1972:0.4059354018,t935:0.6507278581):0.8355227534):0.1522516466):0.4286695353):0.5273843405,((t4255:0.4543393997,(((t4420:0.2971841071,t545:0.3332000007):0.01865118812,(t94:0.7683817795,t3692:0.8107102367):0.5053617253):0.6078841612,t2787:0.8389514058):0.6196948041):0.1145944379,t3026:0.5628551843):0.1219715753):0.5296312575):0.9698127317,t4187:0.8519778538):0.07975409017,((t1351:0.3331482511,((((t4072:0.5070701337,t1471:0.3583125344):0.6869560846,t399:0.5557919878):0.3168950486,t432:0.6126366067):0.7207304907,((t4781:0.6262775937,t1314:0.7919575688):0.8411403566,(t4763:0.5874064958,t1649:0.6437614192):0.8352220107):0.02254280518):0.3803870066):0.7237869536,((((t3202:0.4554702342,(t3423:0.4873491582,t3169:0.2562238786):0.4213889998):0.6376321344,((t2224:0.7993498789,t825:0.3101734177):0.9971639239,(t2779:0.7520597123,t4047:0.4295145557):0.5429032641):0.3204396437):0.09376283642,(((t2204:0.868632175,t2907:0.0110831738):0.169708889,((t605:0.9111692719,t358:0.567261819):0.05806265236,t2482:0.6795250291):0.7371255949):0.6812536654,((t3836:0.3588431552,t316:0.1484292119):0.04866353865,((t1456:0.4573692996,(t3685:0.4049420615,(t1732:0.03175419802,t63:0.7370199659):0.04544956447):0.4321082784):0.4295368057,(t513:0.718055706,t4810:0.7073802983):0.2262096833):0.8606561651):0.8932628231):0.5201998306):0.7852239942,(((t1256:0.9519475286,t4925:0.302369585):0.2312221755,t2229:0.9986906303):0.6951793965,(t3428:0.9815349209,(t3122:0.9759477731,t2349:0.1864664955):0.696283869):0.9521299626):0.7049105288):0.6249511011):0.3216903589):0.8987392392):0.2801516475,(((((t4496:0.4463092897,t752:0.3158174364):0.867041653,t4775:0.7598986188):0.8731350999,(t3168:0.178622514,t1313:0.3941091052):0.3326715976):0.2848390217,t3072:0.3147836681):0.446998256,((((t3792:0.8984242238,t1058:0.1500867321):0.7425107306,((t780:0.05250215554,t4097:0.2417020369):0.7772540979,t289:0.1198037756):0.2988460825):0.119683774,t1758:0.2234687479):0.6866154983,(((t1857:0.1844939264,(t498:0.3224776275,t2642:0.8830989553):0.4936092405):0.3093467366,(t1200:0.9315602698,t4465:0.06757676019):0.1046388031):0.8706031714,t655:0.01064087218):0.3891302701):0.1128686606):0.4859765146):0.835902421,((((t2279:0.9717465041,t3886:0.4193502069):0.06555824843,(((t1605:0.7753756978,t1730:0.4917756831):0.5128283896,t2998:0.8065010493):0.7042934997,((((t3982:0.9509645812,t1510:0.3448941733):0.829875448,t718:0.0721514686):0.07321393304,(((t3895:0.7212343053,t4436:0.4256257957):0.4148385255,t3919:0.8034451739):0.7761058277,(t4874:0.2431940178,t2654:0.4809309542):0.4536632861):0.1276760825):0.8771003252,t826:0.9029375031):0.5793730048):0.3272070892):0.8232686038,((t574:0.6935549122,t4431:0.3369199848):0.6660736643,(t2632:0.7172376872,(t3284:0.2890637992,(t3760:0.610869026,t2661:0.3828362881):0.9132228284):0.6985175717):0.2757551523):0.305714546):0.9497868454,(((t3281:0.7757596613,(t1041:0.927387987,(t21:0.4005575341,t4104:0.8487283036):0.09940630058):0.1363720093):0.9446371587,((t1980:0.9734664827,t2615:0.5329524074):0.943363965,t3946:0.4273997399):0.4193501268):0.5490211681,((t3397:0.2102524454,t4616:0.03972084937):0.3606578496,t4088:0.5346075103):0.5938322914):0.6471076494):0.4116916063):0.5551993193):0.5818428893):0.4733014414,(((t56:0.4686750816,(t4171:0.3328116143,((((t67:0.9993218847,t58:0.3014979505):0.2153624303,t4801:0.858343744):0.1397100568,t2424:0.5627536532):0.3531578565,(t4487:0.120865172,t4896:0.9840020852):0.722391112):0.584701688):0.08989368798):0.3315123618,(t1762:0.9500000593,(t4136:0.7692101826,(((t3585:0.1911480248,(t1325:0.001689209836,t2255:0.8488115035):0.9699812687):0.4517220745,((t1841:0.7408196907,t1873:0.1788257684):0.8215829337,((t2877:0.01625809073,t298:0.3340431126):0.5077751833,(t1587:0.6160635191,t1063:0.4995452394):0.6997666017):0.4765272953):0.8427055916):0.7527390639,((t2694:0.630414933,(t2865:0.3308303428,t3804:0.4585468143):0.5944606748):0.9974351265,t3929:0.9625774261):0.6486500097):0.8115323854):0.09416588559):0.8292487813):0.5469847133,(((((((((t2992:0.5683556173,t4259:0.3949927753):0.7805599114,t1175:0.04091445007):0.7643867584,((t3821:0.02768375655,t3006:0.1913647617):0.4203181083,t3475:0.9063835873):0.08342644386):0.9979251395,(t4433:0.5105338693,((t1521:0.3995661624,t4803:0.2436534266):0.2399714079,t4991:0.9149852279):0.5945335242):0.4999193475):0.03605821752,t3650:0.4262857866):0.3821511313,(((((t4728:0.8763825349,t2640:0.8909392343):0.8235181617,t3939:0.06163756945):0.7023730883,(t3670:0.8489563302,t45:0.8891752213):0.9062200298):0.7284130137,(t4214:0.6071781716,(t255:0.5471948399,t3329:0.07519407175):0.4449009895):0.4435284184):0.9209371316,((t2214:0.864842118,t1959:0.1228758341):0.3623794119,t1991:0.3001923957):0.8381461618):0.5013161055):0.03621138702,(t4845:0.3836973961,(((((((t264:0.1797379714,t2920:0.06323192385):0.9472994932,(t178:0.8134361336,t3555:0.4632465078):0.5426755052):0.2946241021,(((t1479:0.2834749671,t2206:0.1258117519):0.7001927397,t285:0.8636411829):0.3024032288,((t4602:0.4241112387,(t307:0.1070842601,t3141:0.4348691227):0.6733420957):0.0474547029,t940:0.6841482823):0.4393613203):0.8206952848):0.08065264951,((((((t2743:0.1904597802,t631:0.3801400191):0.4537953029,(t2708:0.8371253381,t2886:0.004927282687):0.7707951961):0.3148066546,(t2626:0.745920066,(t2370:0.9215266744,(t4516:0.2544125484,t4012:0.9781097556):0.4198086429):0.5622178058):0.7167179745):0.1032030664,(t702:0.763994524,t3503:0.1553396252):0.431600013):0.4082977704,t1661:0.3882870362):0.5601166503,((((t1040:0.499051779,t2421:0.9045748545):0.4754377627,((t4185:0.6205980275,t4640:0.047475409):0.4838943111,t3601:0.7463765482):0.9137264625):0.8396863625,((t2738:0.7547164068,t3981:0.6234005406):0.1647720113,(t2832:0.8119530433,t4411:0.6729323159):0.4709452786):0.1100061161):0.8806596005,(t2950:0.2572114377,((t2558:0.7671095899,t525:0.8266024466):0.7879298865,(t3277:0.07909034076,t348:0.1474553389):0.2294729133):0.473996164):0.03797697159):0.344315456):0.5689797066):0.3359961072,((((t3451:0.07902769349,t3001:0.3875313066):0.5004862808,((t3860:0.844207234,((t3249:0.7483110789,t1195:0.6853311474):0.3935448329,t4613:0.4835625468):0.8231310782):0.8616584544,t600:0.2031736805):0.6895686598):0.7588504981,(t148:0.09319290449,t4586:0.3463730731):0.9267917315):0.2067353737,t2595:0.6939287838):0.6630084696):0.3517247469,((((t1358:0.2051085916,t1142:0.2535161141):0.2356792837,((((t194:0.1963996682,t2990:0.8241801145):0.5477449715,((t2215:0.8922409483,(t359:0.2819857213,(t1475:0.7335701424,t4572:0.2278105463):0.9821699557):0.9990439347):0.7445889062,((t3130:0.932181898,(t1003:0.4691574103,t3114:0.9672080709):0.06537476392):0.5752563705,((t4502:0.5084266593,t4366:0.403658591):0.2622919441,t3286:0.0406470811):0.5171034969):0.3138342754):0.5583873198):0.4776753755,t447:0.6068464008):0.06063132593,((t2957:0.2039489527,t3729:0.4641009655):0.6983884296,(t65:0.5302408147,t2418:0.2530338576):0.7596179815):0.1529862811):0.1936920679):0.3463638735,((t1694:0.789749603,(t410:0.8857452367,t2577:0.204111401):0.2676870029):0.5145677184,(((t1259:0.6243996972,t2191:0.1395968294):0.1414316138,t3051:0.9098383149):0.6104961869,((t2746:0.9354026457,t2804:0.925767374):0.1337771374,t3056:0.02450954658):0.5843916459):0.7518114641):0.1798938902):0.423137611,((t3533:0.9757340511,(t72:0.01517983759,t382:0.5286995219):0.4901269246):0.5750538178,((t1150:0.7687767656,((t366:0.6142901366,(t4511:0.6463828427,t2479:0.7865037511):0.3651600857):0.9743084444,(t1141:0.4077304071,(t4:0.7421213654,t3024:0.4245137293):0.4052562108):0.6585915205):0.1432655065):0.3668315208,(t943:0.914228024,t346:0.7226156169):0.229525069):0.2749038998):0.640280901):0.5096835378):0.3947588347,((t3578:0.6678591974,t1539:0.02688177186):0.8730778901,t2997:0.5442402717):0.03954109619):0.7367512153):0.3130241984):0.9557205813,(((t934:0.08885042346,((t4943:0.7681824565,t4666:0.7884132648):0.430543829,((t2567:0.7613236192,t4647:0.8441665513):0.8276139072,(((t345:0.485455778,t882:0.04423073493):0.3159170146,((t3987:0.8490459814,t3224:0.6555161292):0.4186672282,t1387:0.767411242):0.07535287458):0.3255701531,(t3088:0.1736841297,t4687:0.1593686373):0.0143699469):0.06147860666):0.5972477924):0.6454618159):0.3578858965,(((t3399:0.6922279608,(t1227:0.07279453869,t3994:0.1977977045):0.9250212964):0.6493315734,(t3614:0.9628730402,(t477:0.4931149457,t2153:0.6466388721):0.278322249):0.5116253912):0.5139884478,t3709:0.2226705174):0.7410245829):0.1560418631,((((t4709:0.01547131012,t1124:0.7757657599):0.6755204608,(((t4741:0.1844353909,t1712:0.2946070931):0.3033433531,(t903:0.2345348301,t1951:0.6515262525):0.02518605581):0.1926513275,(t323:0.8523304395,t62:0.08897882816):0.5348312724):0.7108556966):0.7834853258,(t1048:0.04900486208,t1149:0.9334965798):0.4003091659):0.2707316291,(((((t707:0.5182382076,(t20:0.7646884294,t2109:0.1980254753):0.4493309145):0.7037261934,t4177:0.5034425973):0.5418781326,((t4355:0.8896170026,t4814:0.8395685484):0.8667432563,t4247:0.9097422885):0.4744975835):0.139687004,((((t197:0.7592019648,t1827:0.5059122564):0.6455280827,t4243:0.3151332475):0.7940043439,(((t2015:0.9685657995,t188:0.5177651295):0.9996646901,(t4674:0.06534428243,t3868:0.1392651426):0.7662261429):0.4299562648,t4049:0.7989851125):0.7915994257):0.4046109545,((((((t2910:0.2856124884,t3998:0.4131231478):0.3555277931,(t2473:0.007324372884,(t3230:0.5106419667,t3108:0.674084597):0.1768954021):0.5667304955):0.5571086986,t997:0.6583933989):0.1964697065,((t1786:0.9011179721,((t516:0.7434655449,((t4887:0.6546093866,t2154:0.09627265506):0.7461024655,t630:0.007537527243):0.9425626635):0.9086317783,(((t89:0.7400260819,t334:0.6808794844):0.3554692685,(t764:0.003195799189,((t2723:0.6121856633,t2373:0.3929359764):0.2481850327,t1702:0.3423850278):0.237114317):0.4051361245):0.09202447906,t2122:0.7053843862):0.05282155401):0.07046277798):0.249602237,t912:0.03067528619):0.07358160033):0.6664595972,(t1066:0.7569250409,(t1305:0.4392006425,t2583:0.1490234821):0.9629560602):0.8277951095):0.9652523915,(t2446:0.6310957014,(((t4340:0.7684437605,t2322:0.6377697303):0.1788574064,(((t4530:0.8596319782,t1779:0.2722145903):0.3804833062,t560:0.6932128028):0.9835307521,((t4752:0.2737035069,t4982:0.09819985554):0.6147408606,t634:0.9432918408):0.4080122488):0.3075683757):0.3831631818,(t4422:0.9733662456,(t380:0.2677505114,t158:0.3671286749):0.9330587434):0.6576022322):0.4208260037):0.3222646452):0.9280143622):0.1717364278):0.4577977227,((t1000:0.5509319867,t711:0.07187527837):0.9277023722,(((t2861:0.3340854507,t4130:0.4591609144):0.1053898169,(t2262:0.1149932325,t4693:0.2332712759):0.2090749755):0.9027315977,(t217:0.9636097224,(t2928:0.5128834809,t1096:0.1159280853):0.1984668444):0.7048599408):0.6615817256):0.3667517549):0.03848553356):0.5243290702):0.8572676014):0.6570304416,((((((t1490:0.6932693394,t4813:0.9867736525):0.8887978443,((t1114:0.5809965285,t2037:0.9336156386):0.04703436582,(t4792:0.7670502604,t1033:0.2572370144):0.2232324991):0.576036897):0.2489234114,((t1090:0.6681642598,t1861:0.7592822921):0.5409648407,t4700:0.6785265177):0.05193778384):0.2892193224,((((t1910:0.3160585102,t4923:0.04208368249):0.04808597988,t2314:0.9112550071):0.4262476773,((t1718:0.9596122832,t1410:0.6763408249):0.7451057229,t3584:0.3915688666):0.9577314954):0.8923585019,(t1317:0.9794028152,(((t3409:0.9630143095,t1543:0.1187219149):0.2824911382,t2676:0.3542244716):0.4642618545,t1672:0.1254665174):0.8097363436):0.5053133694):0.2160847241):0.9988232758,((((((((t3546:0.6127355194,t2852:0.08237288939):0.4432123553,t4853:0.6185161674):0.638206308,(((t2029:0.5011937697,t1898:0.4220412152):0.4177122805,t3518:0.6557646575):0.5064452945,t2590:0.6847039601):0.02987745591):0.8142041245,t2608:0.6898910287):0.1739860389,((t252:0.5513642298,t1556:0.4358791208):0.475605414,(t1748:0.06122780219,(t4118:0.7664316038,t1077:0.6857726299):0.7227829446):0.9400820734):0.9176695209):0.2050848741,((t995:0.3843363277,t1044:0.5294049557):0.03525020485,(t2682:0.4704738713,t836:0.9912010645):0.002895758953):0.6543571826):0.3725051805,t2460:0.5794375308):0.8704791195,(((t1700:0.01301623811,t1451:0.1506979994):0.7094372038,t814:0.8712445896):0.7178737819,t11:0.873640296):0.7535745106):0.7136196867):0.7952476251,((((t1092:0.3115337621,t4213:0.8566735301):0.3055560649,t1961:0.6281866122):0.1655700796,(((t918:0.1747456768,(t2102:0.9304132541,t834:0.6489289068):0.5543183968):0.4556021083,t2892:0.07734527602):0.8816009036,(t4751:0.2212550647,t4491:0.03246501926):0.1210206342):0.5378542719):0.7218904784,((t1831:0.7088573559,(t3039:0.2050984807,t706:0.7538442865):0.9405935137):0.1461368694,((t909:0.0614246903,((t4294:0.7530829513,t1202:0.3808521081):0.03547592345,t1284:0.7092968402):0.9835932779):0.1483054191,(((t3542:0.5183108682,t884:0.01049827249):0.8388102937,(t4849:0.441805477,((t2798:0.05319192004,((t1987:0.9769116782,t1021:0.690025053):0.2116704141,t3038:0.9827959649):0.09709312557):0.4772203767,t3626:0.4265433326):0.01943905489):0.9944709251):0.6617160912,(t3417:0.331042337,t213:0.8466180656):0.3969420043):0.7329561443):0.87057892):0.4604139021):0.4813063571):0.7990057706):0.6415172855):0.6678200508):0.9383532861,(((((((((((((((((t2962:0.7262138692,t3922:0.8583351264):0.7161026634,(t336:0.7422789659,(t102:0.74303062,t4434:0.8937873503):0.1427770783):0.02465950488):0.9909834159,((t1509:0.1205776036,((t3561:0.934114797,t2456:0.61937402):0.2097503061,((t2828:0.8452374968,t2924:0.3701922887):0.4136367294,((t1882:0.9889265688,(t2065:0.4537541952,t2079:0.7174955611):0.8356350283):0.9128047386,t999:0.2084656192):0.07642332604):0.5967560955):0.8393798631):0.667187131,t1811:0.680862278):0.8853018477):0.4850144743,t1920:0.7151014295):0.2848867627,((((((((t3326:0.5353890422,t3800:0.3645006984):0.2478848353,t2796:0.6690276565):0.7904291577,(t1102:0.00570087065,(t4971:0.3237898755,t3602:0.990154071):0.4400586628):0.550433977):0.5442886134,t1330:0.5556930776):0.6117003686,(t1319:0.4745831352,t1282:0.5457444051):0.7737812246):0.1190525598,(t2124:0.07137664082,t1316:0.9582177997):0.1256239337):0.5778165422,(t4954:0.6034199249,((t3450:0.2082994911,t4386:0.2360969407):0.5511637467,(t257:0.3070545059,(t827:0.4293418007,(t678:0.5941290427,(t4336:0.9044811127,t2246:0.7943649262):0.9168136851):0.1483359498):0.594744635):0.2570373758):0.2839867861):0.7414448969):0.4724285717,(t4007:0.1859351087,t1332:0.3810958802):0.2426472993):0.492919263):0.07921248046,((t2871:0.661856913,t3029:0.9956787496):0.614281669,(t3913:0.3594356859,((t2500:0.6831667242,t4149:0.6431639136):0.5516189961,(t3961:0.8166393847,t3855:0.2546768542):0.4254611351):0.02258943021):0.658599227):0.2355531412):0.472023821,(((t604:0.452792451,t2148:0.8389070204):0.3583359655,t4405:0.4204529207):0.659835666,(t4352:0.8666897214,(t3553:0.5559384008,(t3819:0.08926707553,t484:0.7154611591):0.404522259):0.05359331914):0.3632904361):0.3948235237):0.5181460318,(((t1874:0.1355053952,(t964:0.7533758597,t686:0.9844604735):0.183650122):0.7069136717,t1696:0.4060957308):0.05870199832,t2808:0.8396733731):0.866898672):0.1832010448,(((t1232:0.08006161731,(t2701:0.8607525362,((t1637:0.6824927994,t954:0.7245764658):0.8207018448,t3551:0.7166925969):0.2768487402):0.4727787913):0.5248614741,((t2613:0.3654149936,((t2648:0.1286852288,((((t3259:0.5070074962,t1664:0.9151285114):0.2723700923,t353:0.07942079986):0.8792005605,t1893:0.1564026419):0.1172787147,t3586:0.31520587):0.4191328976):0.4746167548,((t4382:0.668854695,((t4266:0.6942115454,t4759:0.9251362102):0.1276626752,t463:0.5920489666):0.06901807222):0.8891850021,t1917:0.9551467802):0.7135965317):0.6093363722):0.3366824749,(((t1927:0.1652037762,(t3010:0.7443171921,(t4774:0.05417106254,t2605:0.3119480584):0.6296363084):0.6548634963):0.3114851988,((t2530:0.5482004059,t239:0.9850254925):0.7656948289,(t3973:0.4502616085,t222:0.3419384907):0.1624722034):0.6988789586):0.2249992865,(t4125:0.4914448904,((t1067:0.6593904477,t610:0.3369132897):0.9780704966,t1596:0.237529248):0.3897448229):0.6126249847):0.9516359342):0.4012620279):0.4237613117,((((((t1118:0.3783206646,t2108:0.7804531916):0.9224953155,(t384:0.6595169653,(t2089:0.03323150403,t3395:0.4163023452):0.709629087):0.6090891897):0.2694969382,(t1580:0.003898847848,t4179:0.9587801204):0.8697528949):0.7935501849,((t645:0.001531203976,t4092:0.8965186332):0.6515337583,t2222:0.9239231932):0.370236512):0.810224117,(((t652:0.4089869517,(t4833:0.5379008662,t4460:0.9477532159):0.6428246435):0.8888959237,((t2850:0.9468437084,t4555:0.773919052):0.172683717,t944:0.631241061):0.6874550991):0.3906486121,((t4732:0.6644497479,(t1356:0.828384636,t602:0.7148917774):0.03186064656):0.2766948789,(t3257:0.0134123133,t3378:0.02960873791):0.5065867337):0.9603377432):0.7573593766):0.3788352862,((((t1121:0.8400318578,(t1727:0.2598242431,t723:0.5820624123):0.902386379):0.5028996905,t977:0.8005011298):0.3318134772,(t4322:0.269030743,t1753:0.5229965521):0.8806119692):0.797340415,(t1680:0.5867746982,(t1617:0.7191714298,t1949:0.6596775411):0.7487695187):0.4226935252):0.8843422038):0.3909669449):0.5417169568):0.5114382424,((((((t1567:0.1844823088,t3599:0.9039775273):0.4835802736,(t548:0.03273510071,t4767:0.02672401746):0.3378277521):0.3577588,(((t424:0.08024800848,t4745:0.3147766502):3.427267075e-06,(t2007:0.911275578,(t127:0.6719994016,t3707:0.6370994784):0.8615536501):0.08571552136):0.608506822,((t963:0.8954207071,t4004:0.1710815181):0.7203015303,(t2082:0.9753508174,t3123:0.635486145):0.6631354715):0.7841217713):0.6585065254):0.8198231205,((((t1428:0.9879597069,t4520:0.9947917201):0.02603928093,(t3178:0.7134850451,(t2961:0.2175829681,((t4918:0.3267469991,t75:0.806449634):0.9262028558,((t3587:0.2930043011,t2903:0.9516623588):0.4193006081,(((((t389:0.3402687062,t969:0.5061908925):0.3697433439,t308:0.4861857144):0.5600478824,(((t4122:0.3142874816,t4808:0.9828627198):0.2153276552,t1413:0.6939961817):0.1179870605,t3358:0.8280851066):0.3168632302):0.7464490433,(t3176:0.8260539107,t563:0.8996708137):0.9911272344):0.3439247569,t53:0.7261300127):0.7146047754):0.2551055667):0.6233245106):0.9600828567):0.3450316775):0.5784728285,(((t1181:0.7961972628,t3096:0.2175641307):0.0989612001,(t1880:0.4179928249,t3261:0.3475408361):0.2301712115):0.1828998581,t4536:0.8773570918):0.3965672867):0.5203566179,(((((t296:0.4962522339,(t3642:0.4497568575,t2899:0.8129758721):0.3301557752):0.2441319616,(((t3841:0.5856999112,t2538:0.2656358804):0.7877510739,(t2917:0.7939528148,t4232:0.6278766962):0.1803098298):0.05187540012,((t505:0.257752409,(t335:0.8851803045,t2076:0.1084350201):0.7093391018):0.04393044533,t1999:0.9639111937):0.2097887732):0.7898364442):0.3349640444,t2983:0.9945538465):0.3651028003,t873:0.4657309745):0.2077373657,t750:0.06851134798):0.05651681428):0.02990587684):0.568570222,(t408:0.8290724331,t4618:0.3050913746):0.974504231):0.8530586013,(((((t1766:0.5785169625,((t2444:0.1964719016,(t1662:0.2404880368,t4648:0.2487684961):0.2832237349):0.1989366617,t854:0.6079415688):0.9256657488):0.7512660285,(t4360:0.9631684893,(t1629:0.3435779579,((t3595:0.2225473409,t32:0.8150938721):0.7657266038,t4820:0.3309884691):0.5334544):0.8536345856):0.8840599447):0.5045049391,(t4967:0.03526027827,t4393:0.03504588688):0.2806280362):0.3065179011,((t2182:0.3809294947,(t4762:0.889537808,t2327:0.4205300524):0.9629419369):0.9261945221,(((t1742:0.6586132899,t1:0.2439815288):0.4315093237,t1466:0.7518830958):0.8714073719,t4490:0.7333032934):0.5044749463):0.3279135083):0.2239434922,((t4685:0.5753700216,(t3609:0.6175053166,t214:0.5655104215):0.6993877119):0.6711862364,(((t3889:0.7590449103,t4357:0.6798005202):0.3475025736,((((t3301:0.2741813918,t2312:0.8178372607):0.1876533707,t3827:0.1749639947):0.4963001544,((t4173:0.824564669,t3635:0.7417331673):0.4160173705,t4743:0.05443758378):0.5306879554):0.7629743188,t128:0.5647710063):0.2857035312):0.1444029463,((((t3345:0.334779449,t4270:0.1058641498):0.2625456776,(t4725:0.5735976107,((t742:0.3297777141,((t4057:0.2443992328,t1140:0.1056225006):0.3865051235,t2880:0.4282674969):0.4648150541):0.5310326878,(t4707:0.5831282181,t4230:0.0363113198):0.5300266079):0.3989459372):0.2982925221):0.9989453731,(((t509:0.4984098547,t275:0.1400559484):0.7930502705,(t3486:0.05330740102,t3464:0.6512927921):0.6183371663):0.132246179,(((t4600:0.4332387527,t3596:0.1654799755):0.1100643626,t61:0.06889371714):0.9352715786,t3724:0.4764409203):0.3504803083):0.8416362784):0.3266389561,(((((t1655:0.3338650414,t103:0.8236319672):0.9929849051,t4597:0.7570586994):0.9568472649,((t2434:0.2504198209,t3839:0.7122473572):0.7889866326,t1250:0.5060904166):0.8359238699):0.7995930989,(t3372:0.4739475425,t2639:0.1556260188):0.1401465081):0.004005366005,(t4240:0.4544786301,(((t3411:0.5287708368,(t3748:0.8777483352,t4050:0.799628072):0.6812146788):0.4289193612,t3890:0.1908239357):0.4363218157,(t212:0.5147207784,(t4267:0.1932296476,(t4880:0.6118474281,t996:0.8419796473):0.6801335325):0.6654939319):0.233021626):0.1669444793):0.3458538419):0.01115980605):0.9437996801):0.7577234146):0.6465956764):0.6471493801):0.5583716554):0.7520835288,((((t4084:0.7113987007,t4628:0.2698982207):0.1747126551,t3278:0.8913992182):0.6786976752,(((t2528:0.4973286414,t2230:0.1247115331):0.2626977852,t2991:0.2733335379):0.1595955738,t2474:0.6849966929):0.4124169736):0.5120637666,((t280:0.281440116,(t240:0.6452526185,t3448:0.8434458328):0.9662224487):0.00692372513,((((((((((t4504:0.3090105972,t1302:0.7995785354):0.7904413466,t3005:0.5781695005):0.2197984329,(t2383:0.1823836593,(t4719:0.6322632323,t801:0.477838997):0.1493833216):0.8419143513):0.9379060357,t1170:0.6712732548):0.9263909035,t4643:0.4157575618):0.4467396941,t1984:0.7724634516):0.2890088405,t1817:0.3480296144):0.3469312706,((t4722:0.7305953342,t4249:0.6763996407):0.9420758062,t4843:0.8965585527):0.5335896797):0.01741760666,(t4835:0.8365049786,t1792:0.5501905924):0.2631628548):0.3988401878,((((t16:0.6834248656,t3463:0.3617509233):0.8271554811,((t3997:0.196211996,t93:0.1453959302):0.439272328,t3246:0.8807860925):0.6056077126):0.4260622188,(((t1957:0.2868695825,t1174:0.2309518459):0.2613003873,((t839:0.7853680588,t2727:0.9920329151):0.11601342,(t3283:0.2833106688,((t1288:0.7023590382,t2931:0.6141899168):0.8169669176,(t3956:0.9682801757,t2492:0.08359932806):0.3023653438):0.405985462):0.7040732002):0.7462452657):0.2885388869,((t594:0.8468178732,t2088:0.4128975002):0.8556786396,t783:0.4798944306):0.6337389685):0.7739790755):0.1640336085,(t1194:0.2368415657,t1162:0.9484809532):0.1104401278):0.4034753377):0.1689618062):0.1142551003):0.09222533344):0.1329308471,(t2333:0.007825135719,(t186:0.4937455179,(t4794:0.06020901701,t4344:0.7841492801):0.2287012585):0.2842475143):0.2768523567):0.5299976587,(((((t2036:0.4405954094,(((t166:0.4513484389,t577:0.9506866713):0.06507066754,(t3822:0.9107432063,t2680:0.5976844975):0.980326389):0.01877636719,t4870:0.8322180631):0.6260457765):0.8190441623,((((t4282:0.179717391,t3710:0.8906076108):0.9608471682,(((t901:0.7763043579,t2942:0.6476838898):0.7039241516,((t4172:0.4447395764,t2199:0.1728869642):0.9063360847,(t1337:0.785604873,t3443:0.1969642164):0.04977966822):0.1300333319):0.5817552099,t4063:0.8917331693):0.6318063152):0.7867649812,(((t4915:0.5278157443,(t1006:0.4145518111,(t2433:0.2928249028,t2496:0.4507393125):0.9773183155):0.7569449318):0.1637509586,((t4401:0.2987637655,t2980:0.3483995374):0.0258481456,t108:0.6204921142):0.173995943):0.1980753115,(t4314:0.143935276,t512:0.7094376604):0.02067261352):0.5105723352):0.4587925454,((((((t4000:0.3878473961,t1955:0.9217705303):0.2358263177,t453:0.06399881095):0.5610117144,t2754:0.6788395494):0.9287288031,(((((t3127:0.892684886,(t1865:0.3275158557,(t1924:0.7168268529,(t4533:0.2141097956,(t2069:0.6426789684,t3426:0.3880048452):0.01012898097):0.1743045854):0.6319946114):0.6271233344):0.2919093622,((t1340:0.4825708452,(t2394:0.3374720199,t2143:0.9809464659):0.8276730704):0.3819046619,((t2001:0.1712828206,(t1459:0.5572053397,((t1992:0.9427959051,t200:0.8078974748):0.2230089966,(t1253:0.2885210831,t4644:0.9449514681):0.4743002446):0.2081758261):0.6107601211):0.7672076034,(((t4373:0.4224551204,t1245:0.486102323):0.9678493661,((t4387:0.6867055648,t2657:0.1356155784):0.3089737417,t1209:0.9855114771):0.02427270915):0.8964985707,t3384:0.6749291152):0.3100609556):0.5930570129):0.9383101412):0.2625636309,(t236:0.1012587028,(t2684:7.105595432e-05,t510:0.7905280346):0.8733937622):0.8222653079):0.5823246415,(t1344:0.9254428018,((((((((t3118:0.1040853884,t3457:0.6976789827):0.2812245579,t1105:0.1523652102):0.5247684622,t1716:0.9184710039):0.4347472272,t1956:0.139966673):0.03697826923,t794:0.4474921427):0.3198264765,(t3968:0.01561442623,(t3697:0.03309637238,t48:0.2392675329):0.8297406249):0.1214160703):0.2677891972,((t2697:0.7228317864,(t4272:0.6955559249,t3052:0.1212348503):0.510660446):0.8216106254,t273:0.04460524395):0.285070807):0.09515914181,((t1370:0.8247850279,(t4483:0.6413038895,t3247:0.6255824168):0.7946476543):0.2671518866,(t2562:0.4926836079,t3258:0.7683827931):0.5191153679):0.8641255347):0.434407952):0.7386618992):0.1003132293,((t3754:0.1528475615,(t1437:0.2521358016,(t3273:0.4569258506,t4160:0.4420627088):0.9036253279):0.3137168845):0.9214993231,t622:0.1753153994):0.6875868337):0.9079641439):0.8159785327,t2894:0.8683465328):0.3144849443,((((t3500:0.2174643609,(t429:0.9429278637,t2475:0.09661523276):0.6467378302):0.4284414572,(((t1788:0.6325364669,(t2259:0.5643249038,t4663:0.8346780883):0.098181471):0.8661162907,t2048:0.9545170122):0.702393227,(t142:0.4987407257,t2238:0.9545663558):0.9399609426):0.7665764655):0.2638789965,((t4110:0.9450869875,t1134:0.5860108251):0.7755987796,((t1823:0.4399406891,t2034:0.07218393381):0.3540829711,(t1185:0.215702889,(t2635:0.03400844848,t2729:0.7401463999):0.3975910922):0.0471498901):0.3823134245):0.8467795982):0.453222641,(((t4471:0.5070170932,t1260:0.9572457285):0.3352085319,t3198:0.9076079237):0.2212412681,(((t2493:0.07784569566,(t4987:0.4467527189,t396:0.1993106157):0.9489312868):0.1130697392,(t1711:0.2374455857,(t4689:0.8031826077,t4134:0.8644662038):0.1860905394):0.884776301):0.4594913532,((t1535:0.6797542591,((t2299:0.1952631101,t3467:0.9276725044):0.6601518267,((t4694:0.7917074778,t3941:0.1939830135):0.793794516,((t276:0.5535281657,(t443:0.163111296,t2638:0.2404224884):0.9953644106):0.8714757748,t601:0.9922141591):0.8936089175):0.3500113913):0.3907055203):0.9470289228,((((t411:0.8892813064,(t1731:0.1137634676,t427:0.7484523198):0.7715137694):0.05781302624,(t2771:0.3454575201,(((t84:0.1340223446,t1998:0.3526036164):0.6273540517,(t2955:0.09404921532,((((t2717:0.9525596115,(t2093:0.7015653446,t3219:0.06627323548):0.7692494278):0.7032709303,(t3487:0.06485897745,t270:0.06158223702):0.4189626474):0.1120371916,t2761:0.9931760672):0.3868831932,(t1201:0.6044189965,t1329:0.5739250337):0.1408602663):0.9467953723):0.4454013335):0.8951859141,(((((t3505:0.2784547431,t4094:0.3499765312):0.9306254296,t3191:0.4602360178):0.6705352091,(t3381:0.8893911785,t33:0.05029384769):0.1421050171):0.8701545307,t3864:0.5503185035):0.1820352292,t3227:0.1809635703):0.4946118339):0.5582480875):0.9797163103):0.3181473331,(((t2669:0.7735680772,t2823:0.3494082035):0.02417387511,t3519:0.1478207435):0.8570490258,t1047:0.04962586565):0.6861325419):0.7532274844,((t4978:0.4982785615,t3773:0.6828479297):0.2767159804,(t3495:0.6784780018,(t1131:0.6584981559,(t2934:0.6086825014,t4423:0.04016133258):0.1421606252):0.5449224976):0.04323421023):0.4579206966):0.7816028884):0.2307484918):0.9516916894):0.4918339697):0.9123380941):0.869848578):0.2785552375):0.1004747536,((((((((t3491:0.4335138886,t4748:0.1400322514):0.2525995702,t4002:0.09436890576):0.02209823276,t3817:0.6024864793):0.3975430417,(t167:0.1525500917,t3515:0.0578564601):0.1135589082):0.4366631401,t2356:0.612857216):0.03312625643,(t4157:0.7319163128,(t2951:0.8195105705,t3744:0.9934666695):0.002051943215):0.9257125922):0.1120324801,((((t523:0.3683013392,t2777:0.9105640552):0.4414764908,t2406:0.4191308771):0.45298629,t3343:0.1453825799):0.1481226557,(t2730:0.7202222662,(t793:0.511303504,t1157:0.1419341005):0.6160562474):0.6027775933):0.9096826667):0.3534672654,((((t3653:0.2392133458,(t4547:0.8674878811,(t3529:0.5105062064,t3410:0.7225932346):0.5938546609):0.9204834765):0.3003669134,t1345:0.09367364761):0.3876323588,((t1855:0.9383467061,t893:0.9251530587):0.08135259617,t4688:0.8466076404):0.1899984921):0.7431564101,(t2399:0.9017488202,(t2057:0.7648492239,t4221:0.654516336):0.7387904963):0.7905760899):0.9576236228):0.2468244559):0.6560781917,(((((t1220:0.02463638806,(t962:0.1923351241,t696:0.5418487466):0.1910762563):0.9249072359,t4527:0.9588721867):0.2319914654,((t3444:0.5096210332,t2028:0.07436235575):0.03401017957,t1582:0.8321982867):0.4663031856):0.2608651796,(t2693:0.6563487633,(t2553:0.6295327423,(t1429:0.1556336989,((t2247:0.09405928664,t2149:0.2680600684):0.3460829179,t3101:0.8610302578):0.471151703):0.04652113724):0.727698961):0.4072797264):0.7943767714,((t237:0.7817081171,t3228:0.8807131613):0.4931400535,(((((((t3741:0.10361787,t4174:0.9064539389):0.2694355219,t2140:0.5610431139):0.5672331613,(t1323:0.5220925177,t1371:0.9451466422):0.2540856332):0.3813403731,t1513:0.2345292515):0.5498658898,(t4184:0.7509594737,((((t1098:0.9460167964,t1112:0.5436120615):0.8802956603,(t787:0.6914048451,t3539:0.5194457916):0.03716892167):0.1108253072,t169:0.01069202903):0.8384362198,((t4738:0.8546245962,t1812:0.5348012799):0.6461196237,t762:0.1153883126):0.2637670825):0.862475428):0.1050547645):0.4176039489,((t2586:0.7499921029,((t1500:0.02044343459,t2027:0.3348222179):0.3237804268,((t3689:0.6078839707,(t2853:0.6352343571,t3757:0.2663663013):0.2798020944):0.7826251891,t3266:0.6487117496):0.6497461733):0.3159767149):0.6204324234,(((t2970:0.7001024887,t3470:0.3267781525):0.4574851508,t1516:0.9740719567):0.4813881828,(t4592:0.981659618,(t3562:0.9862664249,t2513:0.2776642111):0.5735258013):0.9303701306):0.7627139403):0.8158748376):0.4813487413,((t3799:0.6550234298,((t2265:0.5691744401,(((t1012:0.3639946983,t4032:0.2513800713):0.05077923788,t642:0.111116068):0.332600015,t3008:0.7463258731):0.8327168939):0.0596749594,(t2059:0.4885736173,t3031:0.3480907534):0.426270128):0.3755060467):0.03681151336,(t1950:0.2029797365,((t3305:0.5627737804,(t4714:0.6405647639,t4908:0.2053580165):0.4354477008):0.559139661,(t703:0.7935215533,t1208:0.6252977592):0.04521982861):0.2336645124):0.06484406511):0.4471674964):0.385971332):0.6591193636):0.7686109957):0.1250434432,((((t3061:0.3440663607,t4042:0.3157220664):0.675226545,(t59:0.628866812,t1089:0.4707864656):0.4955711076):0.1480921553,(((t861:0.6624049346,t2443:0.224396925):0.2847800031,t1283:0.05358729139):0.7775096421,(t3104:0.2491828208,(t2842:0.3309956335,t1611:0.6659108354):0.2361551465):0.5764183176):0.8832583958):0.5620367536,(t746:0.3837163656,(t2412:0.6583311439,t2269:0.7888609043):0.4374579969):0.9452234805):0.5297878124):0.6996258383):0.3415587887,((((((t3718:0.5074331015,t2091:0.9762871319):0.1811176061,t2469:0.1494129056):0.8782151802,(((t4495:0.9885551857,(t4654:0.6719454667,(t4538:0.8634280069,t3969:0.8553444003):0.8008243486):0.6771141468):0.4766978489,((((t2908:0.5818098409,(t4858:0.4159208869,t4474:0.05818697251):0.7745204403):0.9324367635,((t1558:0.4736906223,t4044:0.665242705):0.2250373163,t2389:0.7295064945):0.3089176943):0.2673065483,(t4696:0.1253566248,(t18:0.5562083421,((t180:0.8087127898,((t1147:0.4080226736,(t1615:0.1755405967,t4203:0.5146258022):0.1837007273):0.5720820408,t3174:0.1923566561):0.7669331504):0.08475451125,t4756:0.881315602):0.5069478815):0.4342382322):0.7497944215):0.5907712085,((t4237:0.05716132326,t3576:0.7684986677):0.08016940788,t1749:0.7509441033):0.3090163285):0.2411447875):0.3455243844,(((((t3364:0.34792628,t1129:0.5038216433):0.5441117203,(t1860:0.7143098817,(t949:0.06636277982,t1734:0.351874568):0.9854714014):0.4995358372):0.03393022786,t2063:0.05341036804):0.9451750594,((t2210:0.7825913473,t1744:0.6990753382):0.4731194037,((t3023:0.1679361237,(t4780:0.421774203,t2307:0.6262942585):0.2137286193):0.3505443342,((t4637:0.2698253188,(t2722:0.3325449296,t2816:0.546037917):0.3756804399):0.4088706858,t3091:0.4856662734):0.6109957881):0.9863933926):0.9387679112):0.6389107239,(((((t4981:0.8478533158,((t1487:0.5215197825,t4661:0.7316549162):0.2389087351,t3552:0.9588676286):0.5110373397):0.1714820098,(((t1018:0.8430333142,t1840:0.01911786478):0.042871841,(t4753:0.7676874339,t1548:0.08773567271):0.8203522854):0.2735698586,t829:0.5314179112):0.9354918224):0.6734886642,((t3013:0.01223673136,(t2630:0.6517426269,t3621:0.5579286371):0.1238641946):0.8461402038,(t3563:0.03705857205,t2192:0.3473468565):0.7967079529):0.6902407985):0.8339096378,(t1377:0.89918656,((t1231:0.4139542195,(t2741:0.4605228261,t39:0.3945815491):0.05059540644):0.5795721228,t31:0.3798495962):0.6751602513):0.1613921041):0.2757078975,(t4468:0.06827956578,t1328:0.7412680518):0.709372744):0.9975278385):0.1847563968):0.7529182183):0.6685144533,(t3391:0.8265393139,(((((t3303:0.02300456353,t2791:0.644107152):0.9647951617,t415:0.555972304):0.3688408323,(t2745:0.4287488409,((t3930:0.9095302937,t1912:0.9525530094):0.5336323192,t2345:0.3841391159):0.7811806232):0.5653265091):0.2391818666,(t3611:0.231048631,(((t81:0.1912265182,t2835:0.1174680332):0.3640010895,t1111:0.6997918475):0.4389217969,((t1271:0.7103070596,(t4196:0.2140415381,t3593:0.7422047108):0.1657404907):0.90213965,((t441:0.8982148496,t3370:0.8179885081):0.8231075644,((t3151:0.4392819994,t4127:0.5257334935):0.8790064119,(t2132:0.8195359597,t210:0.5704154421):0.8011257984):0.3606936478):0.6137814664):0.5840160763):0.6357191191):0.9892603077):0.890853995,((t4676:0.6922533268,t923:0.2501822175):0.9310205204,t984:0.8551313698):0.7925177705):0.5400493077):0.168681616):0.488126782,(((t4389:0.3458877853,(t4457:0.7923225379,t1632:0.7491949643):0.9259447595):0.4408739002,t1008:0.1913873628):0.842608558,((((t3693:0.3857462199,(t3879:0.7568280797,t3145:0.9899931664):0.7439075795):0.2408439193,((t501:0.9941636659,t1660:0.2161669778):0.8060132959,t350:0.8570286797):0.6125214973):0.4928399154,((t2876:0.6508947839,(t3959:0.8732843283,t4525:0.9837955022):0.6316757964):0.6923866039,(t51:0.8951506307,t569:0.2466581692):0.8383802888):0.9068373493):0.5322197783,((t3974:0.4531708441,(((t1802:0.2676001659,(t4413:0.8194835559,t271:0.5992415494):0.1719379958):0.2776597468,t2325:0.2783769357):0.09932349063,t2574:0.9280326176):0.2778331882):0.6306165769,((t4394:0.5079495979,t4010:0.5211637116):0.8248848014,t481:0.745801439):0.003628396662):0.3672056266):0.1515814501):0.725278693):0.8877521958,((t2480:0.05927129113,((t4755:0.3830011629,(t1138:0.9597058615,(t4686:0.7104720634,t4706:0.2416778356):0.9450298094):0.3729218752):0.8104782165,(((t3065:0.7320153033,t2491:0.09692441858):0.7762306281,((((t2139:0.04145612777,(t99:0.4728451278,t3297:0.8104012054):0.5694008139):0.2352781035,(((t3943:0.7746164536,t4048:0.4115498853):0.3138031936,t3248:0.7205174111):0.6524397761,t521:0.76257078):0.8780235185):0.4367877501,((t3646:0.5080549561,t1042:0.9520941379):0.1351560694,((t2627:0.8308105904,t1557:0.367066717):0.3223976509,t1970:0.6444319007):0.8107497683):0.3334985899):0.3917275881,(t835:0.25590939,((t258:0.1984209584,(t2353:0.5758404543,t4342:0.9567582933):0.4421464668):0.7643002984,(t4345:0.05998407467,t2620:0.9994047221):0.2421732962):0.7181127602):0.1561494695):0.2364479394):0.1835280061,t2996:0.1042222406):0.3432798772):0.2137563182):0.594169474,((((((((t3876:0.4912681824,t2425:0.8930247265):0.1938128436,t4143:0.7781412113):0.4296156678,t3566:0.1208547582):0.03143707709,(((t452:0.06286405702,(t4937:0.005857448326,t1706:0.08070565574):0.09601403191):0.6065935327,((t695:0.6426736352,t3739:0.01534578111):0.1614832615,((t613:0.6584661799,t1169:0.9065664147):0.5477463342,t646:0.002258092165):0.7034766371):0.6349377504):0.7388612218,((t2647:0.2590813586,((t614:0.2705418812,t4377:0.9876070665):0.09514414286,t3173:0.9704576891):0.932730657):0.3506622994,(t3473:0.2270562372,(t3985:0.2409176808,(t362:0.1109369814,t3042:0.08021445014):0.4879289952):0.7511877683):0.6035878267):0.6341009482):0.5159756786):0.06471874192,((((t2900:0.8801715737,t3342:0.2295453155):0.5562548172,(t496:0.01051674131,((t4096:0.9140247838,t3781:0.2054861542):0.2853377257,(t3932:0.9571212442,t1532:0.5884533862):0.4747523509):0.03889815626):0.865502869):0.3659548238,((t3099:0.7026090627,t4726:0.5867526701):0.2676256995,(t2481:0.6185686886,(t4293:0.0931499924,t1364:0.6392615368):0.7653348001):0.3459384607):0.454394215):0.13702334,((((t981:0.5379725944,((t1489:0.703229161,t2172:0.6340708009):0.2020587332,t3331:0.6049142927):0.3474886045):0.7989596224,t1095:0.01419937564):0.9042303436,(((t155:0.5791752611,(t4746:0.5270857823,t1352:0.7720973161):0.6022324318):0.8390010761,(t3335:0.849161898,(t4605:0.6174221081,t1583:0.6836379617):0.668287229):0.7570138166):0.5464033855,(t118:0.7986642586,t4024:0.6063636034):0.9490999258):0.8020704146):0.06805515499,(t1226:0.1323996638,(((t1279:0.1436497921,(t101:0.1362354541,(t2227:0.9049487426,t2696:0.09585445956):0.07034533867):0.4185555575):0.4856618694,t1086:0.1113172413):0.1319267359,(((t4679:0.2609236098,t2597:0.9943507127):0.1028425321,t2784:0.06530996389):0.4709627007,t2258:0.7218955809):0.3901349038):0.2361333712):0.2148582321):0.1405034994):0.9984194201):0.5310845601,(t3752:0.1649888011,(t2933:0.3763812126,(((t1179:0.02401749091,t4330:0.6266003756):0.2760254466,t2700:0.9479821592):0.4909517767,t1354:0.3978155542):0.6772292887):0.5740000149):0.3634165446):0.2295289459,(((t2350:0.2958781703,(t147:0.9509503511,t2183:0.5479835363):0.9483288983):0.346765172,t286:0.5826510598):0.2440643532,((((t1618:0.1451511809,t4988:0.8338828273):0.4419743095,t2728:0.8145442226):0.2333698769,(t3556:0.1644302213,t468:0.5517940556):0.4053673719):0.1781886325,(t1128:0.02794138598,(t12:0.1076982603,t1176:0.5385575735):0.1209488767):0.8491805126):0.7731262511):0.05186388362):0.8715103425,(((((t3150:0.1637162974,(t1737:0.2143738649,t1236:0.3574736102):0.4401737268):0.2282431084,(t4305:0.6562011095,((t1294:0.2199306951,t2047:0.7256445154):0.1831500421,((t3231:0.5772739297,t3730:0.7509981773):0.7497914117,(t1285:0.9890013656,t2849:0.8784068828):0.5872603415):0.9109540004):0.6312700519):0.37108106):0.685623752,((t413:0.4824377436,t1164:0.3325604142):0.5884122383,((t3361:0.4001204052,(t1218:0.7332760252,(t1419:0.2719845292,t182:0.2631517912):0.2184543661):0.6036438742):0.5644708413,t245:0.2534804579):0.4855942002):0.5280457262):0.09601887665,(((t4402:0.9676652874,t2260:0.1953467079):0.8273638566,t66:0.08008735953):0.3203009174,t2334:0.1242184814):0.9871498325):0.2194295281,(((t4461:0.5394940658,(t425:0.7549582496,t3925:0.3060626234):0.07645645272):0.416362369,((t1171:0.4600801102,t1126:0.8090640323):0.3192197173,((t207:0.9006679945,t542:0.9207556369):0.4294410599,t3468:0.01797916531):0.210681879):0.8657100152):0.8573118509,((((t4827:0.7512464493,t524:0.4392646465):0.9863706282,(t4392:0.0353441725,(t4675:0.5735928619,t1229:0.7767823657):0.5324997245):0.9831927998):0.04009053577,(((t1533:0.6484861458,(((t2656:0.5628228772,t2901:0.5876230062):0.1013386501,t1127:0.189776608):0.1568047907,t1780:0.1946117289):0.728710887):0.208614113,t4356:0.8870425483):0.01077792165,(t2911:0.5290552003,(t4166:0.1420111237,(t3910:0.2031622138,t2360:0.0537654683):0.7086734292):0.6999085587):0.6530871815):0.8440217196):0.7041317283,(((t1101:0.4845554559,(t693:0.9521668649,(t2358:0.5665360778,t811:0.2816568827):0.2386645305):0.9701437727):0.5956941347,((t4892:0.2736582246,t1805:0.7756556629):0.9486975286,((((t1794:0.810251985,t915:0.7599247298):0.6739438279,(t2653:0.2679301894,t378:0.2808274301):0.2985125012):0.4846359501,((t4023:0.006146292435,t4534:0.415595189):0.3088320256,t4701:0.01817663573):0.3650710427):0.9421814401,((t1249:0.6625316611,t1888:0.5578232217):0.6212042996,t1770:0.51330514):0.1874328551):0.5932504237):0.7712079845):0.537988364,((t1268:0.8784598566,t1393:0.09459985723):0.8256800091,t823:0.6652170031):0.4114926523):0.4071907259):0.2386713631):0.9695122116):0.5738056689):0.5871799232):0.7020032196):0.5270228449):0.9141213698,((((((t1856:0.4554060134,t3435:0.7338342601):0.8591136402,t3496:0.2724766836):0.9792074515,(((t4131:0.4493588384,t1438:0.1424898391):0.0169189279,(t1346:0.4873448659,((t3564:0.8928217215,t2855:0.5168398467):0.712301655,(t3109:0.4267446159,t2429:0.6532436686):0.3163490046):0.4123994086):0.6535635055):0.1853193012,(t636:0.6641721975,t1482:0.6693586495):0.8676144062):0.9786236645):0.2879595733,((((t3853:0.04455169896,((t3348:0.8287277913,(((((((t3255:0.617343432,t727:0.2154106735):0.1012310933,t568:0.6011281984):0.6217951449,t1577:0.7009667819):0.6624103796,t4192:0.2826758164):0.9119206527,(t2437:0.1889628186,t606:0.9565844375):0.7889055966):0.1195758537,t1643:0.3837734144):0.459727319,(t1117:0.2698911282,t2618:0.5926116046):0.8330705573):0.1032648038):0.414168054,t1206:0.8492447692):0.7138323134):0.9698224976,(t2881:0.2783603433,t2666:0.09769286518):0.4015298686):0.5426371822,(t1597:0.5330211655,t2976:0.5537147741):0.9786706397):0.875983658,((((t3953:0.1110261881,t1575:0.1038666626):0.7970567138,t3373:0.9463112121):0.3151886098,(t2256:0.1085454128,(t293:0.494775947,(t4485:0.1427976664,t4598:0.733915071):0.7007157537):0.956928045):0.3789895251):0.6441339992,((t635:0.2422891704,t2867:0.09260289278):0.08963877847,(t1335:0.969954977,t2245:0.1232232768):0.5454997937):0.4868543586):0.9475524505):0.5203342964):0.07464472926,((((((t1307:0.7448745144,t1549:0.8074504638):0.04834123538,(t1439:0.0464704372,t1143:0.5817480646):0.4043240268):0.970479633,t3538:0.4797786004):0.5141336827,(t3055:0.4362680903,t699:0.9673258108):0.4752776776):0.9283996534,((t2702:0.7561031333,t3119:0.3125612405):0.1533818038,t1454:0.4214225169):0.305761968):0.0512400975,(t1606:0.1635172532,(t4819:0.7113592383,(t4859:0.514698758,t4403:0.1776181532):0.6947528103):0.9120983917):0.2337281129):0.6036511629):0.4587940099,(((t1571:0.2347469344,((((((t1885:0.4648549536,t2205:0.3423239004):0.7271133899,(((t2943:0.733401611,t3524:0.4820557779):0.8062739889,t2161:0.6277431529):0.5578693014,t2002:0.5098305431):0.9197717921):0.3901361739,t4103:0.521268836):0.897731225,(((t2893:0.9821053573,(t29:0.6714318432,t2527:0.1848702407):0.3035584206):0.2247526785,t4011:0.4433292681):0.7310939576,(((t3330:0.04318036791,((t8:0.08348144381,t3148:0.7158740116):0.09420429147,t3250:0.004358904436):0.5794243214):0.7331423769,(((t4333:0.4577489819,t3915:0.3274542151):0.3613349234,t3060:0.1739946923):0.2744481734,(t3778:0.2019108806,(t4783:0.1555155762,t2150:0.642218383):0.7774707614):0.2836122969):0.349599502):0.4391534119,t3810:0.57538392):0.3221794753):0.5986578595):0.8557625245,(((t451:0.1744205549,t2300:0.2394699859):0.9904182642,((t4256:0.1891314115,t1156:0.4642362955):0.1918831174,t3870:0.3518105119):0.02302495181):0.8487849045,(t4932:0.685475261,((t4588:0.4764426483,t756:0.8637069417):0.05062302598,(t433:0.6106292214,((t2305:0.1798949142,(t4826:0.6423080359,t650:0.542726764):0.9022707359):0.567137663,t1237:0.5889266245):0.4441852246):0.7752515059):0.3152752926):0.3289108421):0.004907134222):0.9948941171,((t2922:0.332996178,((t2178:0.4801534307,(t3659:0.3371630004,t987:0.4654632651):0.4621898753):0.6625878876,(((t3416:0.909577426,t247:0.6051569113):0.3323170394,((t195:0.6771594041,(t3110:0.2685421258,t2118:0.9539791781):0.3602705286):0.6442660717,(t36:0.8065310852,(t931:0.4053850835,t690:0.09342274955):0.9165674765):0.219859709):0.8406578354):0.5010006183,(t2759:0.766777148,t1483:0.5716585668):0.8970591493):0.0163023218):0.2714568165):0.178097826,((((((t772:0.07234980096,((t3970:0.2556516523,t1774:0.7473114163):0.7367113307,((t487:0.3291204767,(t1915:0.6737416417,t2375:0.2043516154):0.475276906):0.6535222342,t2457:0.08902432304):0.7175145592):0.5115761126):0.6376107286,(t1045:0.6093795286,((t1441:0.2389455924,t2594:0.5455076403):0.5832318075,t846:0.4566403269):0.6144008227):0.8709627425):0.001059436472,((t177:0.3155212936,t4964:0.6736288012):0.195589324,(((t2438:0.503904656,t4529:0.5337130895):0.5037660524,t4274:0.1082925878):0.1722046456,(t573:0.9777321906,(t4346:0.6133071729,t1820:0.1443039007):0.911746575):0.1935084583):0.4993587444):0.5274296873):0.6773992765,t3404:0.6927224961):0.7624552012,t469:0.2176305975):0.326663807,(t544:0.6165601171,(t4816:0.9676665168,t1488:0.5706706215):0.7112443757):0.9025811823):0.4540297256):0.3950548787):0.07468537451):0.3215730707,((((((((t3597:0.2778795164,(t4768:0.3640320725,t3365:0.01658568741):0.3654491922):0.7050910292,(((t3962:0.05825421191,t4901:0.2232397597):0.349669602,t4518:0.4222123367):0.8222987463,t3332:0.01361004892):0.8684816549):0.457746349,(t539:0.07964168349,t4301:0.8961682636):0.478665082):0.2231043547,((((t4469:0.7857592576,(t3184:0.7019001909,t805:0.2204649528):0.8314798789):0.8302122941,((t2064:0.4600917122,t4311:0.1571462958):0.4913400623,(t3751:0.5141741321,(t209:0.24431416,t581:0.8388730425):0.3050636274):0.5205267509):0.9991438405):0.6470655424,t1369:0.3032506888):0.593077288,t4930:0.01240562112):0.8656959101):0.5042608958,((t1389:0.9001046948,t3516:0.9083159235):0.524610267,(t3786:0.3953087821,(t951:0.8998011777,t582:0.5606955183):0.580449735):0.6469036601):0.4011064649):0.590336259,((t3390:0.8315982851,((t917:0.5988220992,t2895:0.304213068):0.63178022,t1024:0.4064287215):0.1356475712):0.2176654835,((t2782:0.8343843911,((t54:0.7095737844,((t2187:0.6702263686,(t4364:0.9226403139,t883:0.717911659):0.2681436751):0.7453207106,t961:0.9855663897):0.6642666061):0.7799918016,(t4505:0.2609919675,t3131:0.6107438086):0.9097181126):0.6873232839):0.5555929479,(t3485:0.8560693327,t3433:0.2038849513):0.08809814625):0.08785267221):0.734919569):0.3240610184,(((t1154:0.2515835371,t2739:0.3910329333):0.7558204362,((t2707:0.7294976402,(t134:0.4851843589,t3190:0.2765906989):0.5612602818):0.04161430965,(t2283:0.9866168618,t116:0.2518849722):0.1378967855):0.9956789967):0.7314204467,((((t799:0.07380788494,t3631:0.6021298333):0.07271333132,t4691:0.4143983892):0.125977909,((t4439:0.2216689812,t1834:0.2722222121):0.09730051388,t4372:0.6841965835):0.2620185714):0.6279848493,((((t3382:0.8331882693,t3319:0.5416606984):0.1421982232,((t4750:0.8010217831,t1331:0.8270491208):0.710337963,(((t3369:0.6017719919,t4903:0.7916275745):0.5099746892,(t1397:0.6708954154,t830:0.5889599104):0.8337304106):0.6273544645,t1242:0.3373929246):0.3176791298):0.6009680098):0.5922259644,(t3059:0.5643526746,(t110:0.04729210632,(t3135:0.05593224615,t2995:0.2531147534):0.9050108318):0.4461702101):0.9784323771):0.4462039103,t4568:0.5483387862):0.04373953515):0.9025261765):0.9756627248):0.1298916589,((((((t2622:0.48191215,t3018:0.7444168539):0.1396671874,t1255:0.9484746438):0.5644144597,((t4379:0.2571523439,t4472:0.9462082535):0.1031161232,t3867:0.06826604926):0.5223095182):0.9378964424,(((t4054:0.7025092912,t3049:0.3619596492):0.3767808862,t1274:0.3648959694):0.3768779512,(t892:0.3261858011,t4153:0.2276025296):0.481593007):0.9884560702):0.5641372495,t3380:0.420128898):0.9593211666,(((t176:0.7291272117,(t330:0.2285093896,t1434:0.8140266149):0.2461815048):0.807351124,((t55:0.4705730958,(t2660:0.07369258977,t4108:0.8956295552):0.2999411812):0.4361851562,t3291:0.705221473):0.5625782227):0.07907431177,(t3790:0.715761242,t2672:0.904522578):0.04355101241):0.07673324202):0.1443478838):0.03750915523):0.8366761184,(((((t3880:0.009498371743,(t4866:0.829416381,(t77:0.2253937863,t2572:0.443762416):0.2815836424):0.9043423578):0.5083251945,t1180:0.809090744):0.1056817512,(((t2167:0.2688501936,((t377:0.4173331284,t1433:0.4375145175):0.4050415305,(t2169:0.01456472883,t2726:0.3118140332):0.1912904955):0.2985302159):0.9331593327,(t3660:0.6137267263,t2776:0.7787270402):0.9159004963):0.472194792,t1704:0.5341559888):0.03054457088):0.5855971854,((t2404:0.536513949,((((t1440:0.09305174975,t1703:0.4803197079):0.9445625183,t1502:0.7432476904):0.5563949964,(t1015:0.8012833111,(t1224:0.9997541034,t4018:0.9197692787):0.3193182859):0.5101394474):0.01780749904,(((t1399:0.6819714301,t1743:0.9916607647):0.1207272294,t1219:0.4478181247):0.7536255666,t2518:0.5641920085):0.6884802077):0.8074946171):0.6158743305,(((t129:0.3399521485,t3711:0.7580136268):0.7707588433,(t751:0.7696820339,t1310:0.2450120503):0.7058143062):0.02370283753,t3288:0.053807355):0.05112718837):0.9079942876):0.8495567185,((t2675:0.7575813993,t2054:0.7147112186):0.8014610172,((t3931:0.1651508578,(t185:0.5115830449,t1178:0.4925294972):0.9022826573):0.1235455943,(t262:0.7000839368,t1257:0.5507976615):0.6789103413):0.09495306131):0.3843114113):0.5105817989):0.5146451734):0.2653607025):0.9136875945,(((t4500:0.8369823263,(((t2972:0.7158841167,t2780:0.284190929):0.987707369,t759:0.01065959362):0.9310221791,t1953:0.3465204393):0.925167802):0.05986686749,((((((t1148:0.02572913631,t3722:0.5591894875):0.07132194657,t4593:0.6294633904):0.6290661243,((((((t2338:0.549685539,t1069:0.06180818868):0.4545175545,(((t3021:0.8694861883,t4617:0.08560915734):0.3727767158,t4159:0.3482113048):0.05875081173,((t1993:0.3553766415,t1207:0.5449401096):0.7918334526,t4304:0.3689052828):0.8009147546):0.09462153586):0.5054381862,((((t219:0.3396998795,t4846:0.5325069129):0.414721299,(((t2836:0.04239083803,t4832:0.3427956109):0.3126660329,t2580:0.7316503034):0.03463238687,t767:0.3322009149):0.4818835603):0.1121127263,t3999:0.2678172956):0.3720021686,(t3046:0.3483302249,t4223:0.6817101769):0.8559712677):0.8345867596):0.9626000368,(((t3942:0.1884689068,(t4615:0.4740338752,(t1465:0.4349414709,t311:0.4843829516):0.6494075418):0.2483379648):0.3801422291,t1280:0.5939206914):0.3179876846,t1595:0.1686464569):0.3168983343):0.1343455976,(t782:0.07591431658,((((t4842:0.1188244692,t4482:0.2925068098):0.07987042773,((t1721:0.9066852052,t1299:0.47622155):0.3871370147,((t3865:0.09764161357,t265:0.4461827667):0.9143252866,((t2704:0.674680145,t4633:0.8897380566):0.1852220371,t4824:0.5553345073):0.1240355191):0.06238064193):0.332126034):0.8693814105,t2134:0.6806587656):0.8521507066,((((t929:0.8002988698,t360:0.8183032291):0.009845185792,t1985:0.9126624789):0.8641686465,(t3359:0.38005118,t4917:0.5200587453):0.1385276746):0.16935671,t374:0.233729441):0.8459879947):0.8638308253):0.8059760581):0.3825276541,((t2519:0.06507771066,((t1197:0.8521167859,(t2476:0.3362973782,t1519:0.01831251686):0.2568920124):0.9586757338,t1671:0.22752981):0.2835618311):0.4942988891,(t4155:0.1929033052,(t2863:0.5156508496,(t448:0.2467189515,t3181:0.4397426683):0.1473059917):0.3392687084):0.8834627275):0.9323639253):0.7571982117):0.146434682,((t828:0.1404300332,((((t4227:0.1090566274,((t3725:0.6568177217,t4760:0.2437198397):0.6209602875,((t2824:0.5036426929,t2532:0.577140965):0.9957042299,((t4594:0.3095171417,((t980:0.006717170589,(t113:0.1866171919,t1814:0.3099160981):0.4332461294):0.3120039708,(t3471:0.2818450702,(t1708:0.8126967642,t776:0.1572455452):0.7730102895):0.3881945733):0.9498954143):0.6194212488,t3620:0.9156248278):0.407178387):0.1257237506):0.8613045986):0.4482021052,(t174:0.4459921096,t2207:0.6330357408):0.4553039488):0.7233551547,t3917:0.8036865431):0.9706450701,((((t3196:0.1340210841,((t3668:0.08909374918,(t397:0.4095169359,((t64:0.0420454077,(t1581:0.5624408533,t2053:0.6458594268):0.4943832883):0.6234029923,t3098:0.4803616086):0.5210689649):0.01183941821):0.6345238178,t2888:0.2901899479):0.7297252673):0.414621393,t840:0.8789100484):0.3682488038,((t4606:0.7089055008,((t3211:0.8094896381,(t769:0.7427877439,(t2293:0.2129344461,t2170:0.2581404191):0.6707271731):0.6197249177):0.2246985422,(t2024:0.7475114388,t2402:0.8309086787):0.1947811497):0.1233669175):0.7182466106,t3076:0.6135416909):0.6066994325):0.4392030942,(((((t3532:0.7363093009,t4740:0.6918825521):0.5426328494,(t4477:0.340933698,((t2986:0.9069751448,t2644:0.3677296161):0.7604535611,t1538:0.04005089682):0.5092387351):0.3587896966):0.7300251429,(((((t3777:0.5113722524,(t2783:0.6496069378,t4374:0.9223218246):0.6801467591):0.4291814344,(t2724:0.1584493206,t4540:0.041419046):0.4929437148):0.6988639643,(t608:0.5963644218,(t4453:0.5223036893,(t1989:0.3655173599,(t3633:0.3952991699,t2664:0.6562355424):0.9897218072):0.1945227834):0.6095784998):0.9909649452):0.02491518413,((t2044:0.2806361623,((t400:0.3591469496,t4789:0.7693813005):0.003038908355,(t809:0.1642546973,t1579:0.5120665554):0.46266476):0.1571634088):0.4460494511,(((t3185:0.5098795667,(t2790:0.7727807839,t3896:0.0366510991):0.3252120435):0.2835461679,t4972:0.6557989831):0.7941733068,((t2953:0.6719913266,t120:0.9234735344):0.2988974361,t4226:0.748288108):0.0006843348965):0.406105007):0.4261288217):0.1521647731,((t3720:0.5797700428,((t3160:0.1059150773,t4509:0.3929493751):0.5921763538,(t1427:0.6143258237,t419:0.7646313587):0.8615653722):0.3002199887):0.8456335361,(t3203:0.7404808493,t3592:0.1301935143):0.2416811101):0.2492846239):0.3009291091):0.5577313779,(((t2324:0.6973792133,t1029:0.2406649673):0.9089854714,((t4212:0.6869469616,(t4907:0.6388354555,t511:0.7353082828):0.5581969724):0.4012417477,t1336:0.575420161):0.5371567395):0.07368898857,t1736:0.9952599034):0.03336502588):0.7481436345,(((t2281:0.07256022957,t3337:0.6702866182):0.6520519743,(t79:0.7701018879,t2202:0.6437998023):0.09681177209):0.1554714723,((t2837:0.4657903819,(((t3327:0.1342333052,t798:0.9098021449):0.06409947365,((t3456:0.5288823529,t4682:0.8587517438):0.340582605,t2096:0.4005519615):0.2754574022):0.2233461717,t2988:0.1913614455):0.2467669721):0.8861047777,(t2692:0.07277175086,((((t368:0.8347213021,t1692:0.8053751253):0.2060074015,t3133:0.3197813483):0.5429385421,(t1239:0.6283798432,t1172:0.6885980866):0.1762647002):0.1264597785,t2812:0.9299414023):0.6506422285):0.5663623146):0.2927152955):0.591703448):0.2231672609):0.4563438375):0.9006128618):0.5514444432,((((t844:0.5720536876,t1946:0.6459459441):0.4623575248,t1407:0.2443437532):0.3069188646,(((t743:0.3695427808,t668:0.5327903118):0.7709483253,t4435:0.7198622294):0.9679794593,(t230:0.7378858384,(t4115:0.3709415938,(t3497:0.6867466671,t1890:0.7687915664):0.4324218887):0.2445589292):0.08001586422):0.357253839):0.1187253331,((((((t2918:0.2836341027,t3742:0.9865829756):0.8742365201,(t1447:0.2723757359,t4629:0.825598248):0.594071619):0.7224193772,(t4245:0.525883226,(t2145:0.08124206588,t294:0.1318612602):0.4427436804):0.437288142):0.2044635166,((t4522:0.7843593648,t1056:0.3918762994):0.9108479111,t4578:0.0546629501):0.7616028071):0.9593883697,((((t4715:0.8633079221,(t1443:0.997992194,t4164:0.3304667904):0.316386963):0.1026790114,(t3140:0.7829170411,t3541:0.4355274951):0.6869436326):0.1847779641,(((t3911:0.1359237181,(t1836:0.7276318881,t4891:0.5542378575):0.4816342618):0.6672778998,t2458:0.06779606198):0.5070403109,(t390:0.5007870004,((t2929:0.4699226981,t117:0.6878304353):0.608078378,t2190:0.4522648994):0.7273431094):0.6967197198):0.7575234787):0.3502995032,(((t1990:0.8247298382,((t1108:0.4754039778,(t586:0.7978644543,t2587:0.5099183416):0.1292700137):0.9669083608,t1418:0.3218551388):0.1109355083):0.1727054426,((t904:0.6865722307,t3079:0.3358588398):0.7931844026,(((t2556:0.3654822772,t1895:0.9362867798):0.301499065,(t337:0.08531545266,t1099:0.9171976291):0.2881839571):0.9741957404,(t4098:0.7821968084,t3897:0.6990451512):0.9611030989):0.1725696758):0.8070128013):0.2237789028,(((t1654:0.5596507511,(t1343:0.3806105421,t4397:0.08181013237):0.9760157776):0.638342805,(t3209:0.8896464566,((t2068:0.9345775826,t3019:0.1845085828):0.9683633423,t2304:0.9180472342):0.03846808639):0.9712639223):0.4561148998,((t4636:0.9530295967,(t1198:0.236275549,(t3885:0.4316755361,t422:0.07642566622):0.8550094459):0.4796622358):0.0280244872,t1151:0.9451876087):0.4585058074):0.9793507231):0.5831834909):0.8233530656):0.2641680946,((t2636:0.224543985,t2321:0.4922982377):0.386644474,((((t4856:0.3644424952,t4350:0.2913451123):0.2318404017,(t4665:0.8624050699,t881:0.7815753685):0.8935352324):0.1867722482,((t3489:0.2088805153,t4931:0.402618679):0.8080928561,((t3647:0.8394976901,t2800:0.9883021594):0.7658070254,(t1899:0.7681623884,t1060:0.2660722579):0.2057109757):0.4731678287):0.3794782853):0.4154082837,((t4286:0.8137694835,t555:0.3828139727):0.07427591737,(t4873:0.6539772099,(t4708:0.6638824544,t2905:0.7580100039):0.8293501411):0.09703705669):0.6259755625):0.4635938986):0.9592518804):0.8922702463):0.148444203):0.9343399641):0.3050308751,(((((((t2699:0.8012237339,t748:0.2971138852):0.203262157,((t2710:0.2165209348,t92:0.4777823789):0.8662169997,(t3737:0.6127622721,t1159:0.7211965011):0.03905109409):0.03947924264):0.9639162105,t527:0.3113428475):0.7297065221,(t1845:0.4124485485,t1353:0.7987751556):0.132463662):0.4291678162,(((t4384:0.5566129005,t2679:0.08691043104):0.7908748419,t1501:0.9259745886):0.4439405294,t4670:0.1426056463):0.3734167689):0.02125661052,((((t3367:0.6992029927,t3600:0.9555784948):0.3446867869,t2882:0.6148914178):0.7194906557,(((t3244:0.3919181186,t1120:0.8059376527):0.7269515335,t1213:0.5385679661):0.6660441153,((t1476:0.4000402284,t1635:0.1271797798):0.9407927128,(((t503:0.4732806697,t1566:0.7955614415):0.9320799538,t3874:0.8879348787):0.2920600104,(t3992:0.8307674127,t3045:0.8317342969):0.975092506):0.02935594879):0.771670348):0.2577193296):0.2427873353,((((t2261:0.412747598,t3747:0.02048412757):0.2675705592,t4432:0.9606487877):0.377837684,(t3040:0.8763132908,t1918:0.6285465527):0.9045121861):0.1957078567,((t2714:0.09086301387,(t749:0.5729200717,t649:0.1088098537):0.9886608864):0.6518330537,((t4123:0.449544901,(t162:0.5558426406,t202:0.5067394613):0.7176455399):0.1929503132,(t2649:0.729774453,t2579:0.919938473):0.08137630462):0.1766410554):0.4118976456):0.1122404407):0.2474422581):0.5146801604,(t4957:0.436951132,((t3087:0.5753574779,((t4815:0.8729810186,t4417:0.7263636144):0.9240609789,t3926:0.7476318018):0.7443793921):0.9512390504,(t2542:0.02293844242,t3484:0.724450717):0.6332095882):0.05870819837):0.2594692421):0.8810820857):0.1495012052,((((t2252:0.937488304,((t2504:0.9867848356,(t2317:0.943743147,(t4888:0.4339599803,t681:0.5157751769):0.8072077632):0.6645195126):0.7800397463,((t1301:0.102808954,t1994:0.9588635338):0.3589918974,(t765:0.2438202074,t1869:0.6499144204):0.7221982933):0.4669387878):0.5752985966):0.5863504931,((((t215:0.5727722896,t1904:0.8428073172):0.9647256706,(t3651:0.8852861626,t3089:0.4912025551):0.7162498443):0.1018998672,(t3299:0.9717466864,((t4258:0.5321303899,(t2733:0.7834940294,t2624:0.5684673015):0.7836041322):0.3227238562,t3421:0.5157056013):0.2613937748):0.7485891818):0.9887640327,(t796:0.07456954941,(t3813:0.3176633122,t2365:0.7265721126):0.665169504):0.772024906):0.9593439985):0.7177256555,((t1359:0.4387016946,(t357:0.6234506357,t3455:0.8149569801):0.8773031712):0.04695808911,t4651:0.1699312807):0.4375447084):0.4826788681,(((((t3128:0.933858467,(t3643:0.8054314826,t598:0.114755739):0.5425080012):0.8853158632,t4189:0.197608998):0.489264078,(((((((((t2176:0.8866862964,t2243:0.349186938):0.9862535028,t3213:0.7484109309):0.1792391839,t442:0.5261107686):0.8789361974,t2549:0.2448483566):0.2558014186,((((t171:0.9020560577,t3387:0.08420457202):0.5089216139,t4310:0.6069775461):0.2443324302,(t261:0.02017428586,t4514:0.2950399467):0.4964784621):0.1228583416,t2517:0.8857710287):0.3793380805):0.7143672667,((t3935:0.8670415848,t819:0.4806494319):0.2669103718,((t1414:0.3823591999,t4852:0.2762575715):0.2110284897,(t3152:0.4858272923,t3292:0.8382191358):0.3459897891):0.9452719593):0.1013912975):0.4099577169,((t3313:0.4846970714,t344:0.4913905775):0.7632178282,t3408:0.9770317287):0.3806704285):0.5981262724,(t4893:0.7027347006,(t504:0.6147600787,t2834:0.2072892771):0.03294551815):0.284631381):0.9845412469,((t735:0.4133740352,t2887:0.01353460038):0.2697750623,(t2310:0.8320460271,t3502:0.238386709):0.5066075828):0.8053682547):0.03046746785):0.6776652182,(t1160:0.4759279299,(((t3534:0.03743700986,((t2159:0.5152246291,t4997:0.835828562):0.5466340648,((t1191:0.2370719807,t2514:0.9385933217):0.8111704974,t3311:0.4988263033):0.5932488206):0.2217038346):0.06951630092,(t121:0.9985843864,t3102:0.2786764631):0.4404369935):0.1021271741,((((t2932:0.4698367058,((t3453:0.5631490678,t1707:0.9427366545):0.8003241699,((t3809:0.987141568,t3856:0.4675174854):0.6096208931,t4811:0.3921885728):0.01340659382):0.2553897954):0.4692299028,((t2318:0.790962084,t1547:0.9971508456):0.5075981405,t2094:0.2794549004):0.3334353804):0.9001811063,(t869:0.7318383113,t2121:0.1157797934):0.08645348134):0.6470715425,(t2141:0.6988364281,t1287:0.2330084988):0.5787462727):0.8766039493):0.06377180503):0.7309218945):0.684688176,((t1435:0.4589002668,t2472:0.1006257879):0.3694287753,t4737:0.8384478185):0.8062062371):0.7337007218):0.8959746282):0.8670469248):0.5744035253,((((t1515:0.7314148745,((t857:0.6937058407,t4188:0.3531461365):0.7961301573,((t3479:0.6725545558,t879:0.333635187):0.7967187157,t2858:0.7669005336):0.7693218684):0.8914538585):0.3327511635,(((((((((t4260:0.1064754152,t998:0.2574049702):0.195351867,t1093:0.3095765836):0.4667647046,t1769:0.3787756967):0.6617973789,(((t2825:0.9686296114,t4425:0.9006350124):0.3569231806,t2623:0.3120621273):0.573712192,t3615:0.8227911168):0.8945066028):0.1887728923,(t4659:0.5387571999,(((t1234:0.6469551385,t2711:0.8976277004):0.9942138637,((t2600:0.05442509754,t4275:0.1009387353):0.06102215615,t1223:0.3689330923):0.6042885287):0.2318796492,(t2329:0.1417985647,t4900:0.8538807179):0.341736882):0.08622415038):0.2224209411):0.103392605,((((t1634:0.4097096338,t1935:0.7501708716):0.548618919,t701:0.6546011576):0.1871454117,t4910:0.9149184108):0.9415250302,t74:0.7760634047):0.2042345528):0.829071708,(((((t4526:0.8667801642,t3322:0.3327974682):0.3578662928,(t290:0.188197478,(t2678:0.501363337,t983:0.1387528805):0.4288190794):0.143288722):0.7430509485,(t2752:0.1769741932,t1526:0.5169562961):0.8769162884):0.3189639973,t3188:0.9392928444):0.1430957802,((t2736:0.6635686026,(t1525:0.5984657467,t4407:0.5212928792):0.6164866213):0.2275492107,((t437:0.2526989372,t535:0.9105512251):0.3766164875,t2004:0.9699793241):0.09454207146):0.1359472563):0.3433801532):0.998804736,((t2641:0.8065798981,(t1394:0.123316949,t2866:0.500028698):0.03182609659):0.1365225806,((t3990:0.9320991945,(((((t2241:0.9083765817,t4216:0.06096653733):0.5385114108,t4251:0.3897182744):0.8903810428,(t2637:0.856557284,t4844:0.8184581583):0.1662196741):0.5301939086,(t936:0.4075053111,(t3094:0.05678880122,t3210:0.7504768339):0.828516084):0.3805793433):0.994455982,(t3368:0.5961451025,t1355:0.25489164):0.368018769):0.2507202199):0.2045819678,t2343:0.08113888884):0.4123190469):0.1976829055):0.8548034646,((t4191:0.7787076291,(t562:0.9033107206,t223:0.120892674):0.2589873418):0.3255200025,(t3735:0.9830148052,t4744:0.7531497411):0.804399993):0.8374731264):0.2969495396):0.7432577119,((t2477:0.940511015,t4202:0.08635751321):0.8326536708,((t4641:0.947972625,((t2489:0.2104190751,t2049:0.2205914403):0.4576844366,t439:0.6949845278):0.5420915766):0.6452192722,(t220:0.361704899,((t309:0.5279205495,t4146:0.7172448467):0.6132897523,t1668:0.6722036903):0.6506566028):0.8454663011):0.8304784249):0.2191784678):0.1775699225,(((t4090:0.6610624751,(t4215:0.1174950518,t4066:0.3435489342):0.03925399296):0.02184642665,(((t773:0.7976296747,t571:0.8299787033):0.3189818368,t1027:0.4338484704):0.1923920785,(t2025:0.7595946065,((t4623:0.6131265915,t877:0.1377695815):0.9687020318,t2668:0.6022374979):0.2372806836):0.987158336):0.4156237831):0.4042463067,(((((t546:0.6415574434,t2703:0.7609884674):0.7671890552,t1682:0.04103107867):0.02491541416,t4968:0.1276262009):0.1602516724,(((t971:0.6840931494,t4589:0.1188302776):0.1964416706,(t1512:0.4864058353,t612:0.2189840165):0.1026609538):0.8504436119,(t1457:0.7207864879,t4867:0.269663167):0.08948907792):0.2246571155):0.4927230231,((t1233:0.7077212846,t1221:0.3091154352):0.8350778508,t1830:0.7694225563):0.4522555959):0.9972205139):0.8277567942):0.5453875163):0.9862852611):0.609162159,(((((((t1261:0.8384260379,((t982:0.5945289328,t4758:0.3562018974):0.2427385356,t249:0.4056981746):0.7873854758):0.663503859,(((((t2000:0.9372161333,(t492:0.02706951881,t3820:0.524569914):0.2986664067):0.1111951028,t4332:0.02516701957):0.5346297799,t1155:0.114448624):0.631635915,(((t3700:0.2104276384,t3085:0.6688902327):0.3719157665,(t1641:0.6124643874,(t3132:0.1862125634,t1713:0.953584472):0.3054009029):0.5287357476):0.8384651057,t778:0.2240596258):0.2584256351):0.4515862034,t168:0.26872435):0.8658130984):0.8653871731,((((t4248:0.06238186383,t3794:0.407754425):0.6435149044,((t3149:0.4210521155,t1106:0.06191094941):0.7227248035,(t3945:0.8332250125,t97:0.815079035):0.9505549623):0.6193752047):0.7521885494,t2114:0.5959006359):0.9367237729,(((t3649:0.3690777873,t3312:0.6809168311):0.467814554,((((t369:0.4171270917,t3581:0.09447962162):0.4209646427,t393:0.2813234788):0.5762603432,t2686:0.1352388605):0.1256183407,t153:0.3658573376):0.5721252512):0.4771786497,(((t4051:0.9766817172,((t363:0.922010578,t2690:0.7955336913):0.7220247625,t4005:0.1658760691):0.7942039482):0.8030758812,((t3240:0.3127969161,t1518:0.855935558):0.06681084377,t666:0.115505697):0.5532432431):0.2473491449,t3074:0.5111969933):0.7523526787):0.4829520467):0.7182613621):0.9667968703,((((t1166:0.9580510824,((t4320:0.3563710554,(((t1591:0.8031230553,t656:0.7498661198):0.7078606791,t2086:0.0478030527):0.3088571164,t3785:0.5887087022):0.4599189693):0.01474047452,((t430:0.08852729038,((t3009:0.9386205161,t3080:0.9559022344):0.02579208068,(t4658:0.796111753,t2582:0.5118807643):0.5194544471):0.9710456664):0.3613994175,(t671:0.8261138103,t3669:0.9812037053):0.1859337194):0.1932827758):0.6496146901):0.3893976216,((t722:0.9433857107,t625:0.6256659487):0.4065536226,t4963:0.2028079275):0.9715239722):0.6343217012,(((t1979:0.1706170156,(((((t1936:0.4509160114,t3920:0.9356836039):0.5483713523,t1396:0.3265940335):0.6742770716,t1379:0.04072640347):0.3895931081,((t973:0.1888134347,(t2557:0.2812815872,t2535:0.08426499134):0.6898595081):0.337251103,(t3053:0.6475397204,t4855:0.6543969095):0.5885763231):0.08010148373):0.5287671254,t2974:0.1998683193):0.1185810096):0.3795859911,((t988:0.1751254715,(t3028:0.359916352,t3957:0.8019847393):0.503323273):0.8285343531,(t3894:0.03016225714,t1726:0.9830852142):0.1546443275):0.4455288732):0.950395176,((t403:0.7802719872,((t657:0.8533000886,(t3520:0.01778965606,t2005:0.4891612378):0.2043992765):0.6761122681,t3628:0.9852999472):0.4273264022):0.03824152448,(t409:0.7434687773,t2485:0.06917876774):0.3052965698):0.3023526508):0.9513719429):0.9452355229,t436:0.6515739495):0.3669611928):0.8413599627,((((((t2186:0.07278544805,t4865:0.7648233511):0.9057840202,(t2087:0.7882120516,((t1025:0.09697709489,t310:0.6350561862):0.1325417957,t4180:0.02870662673):0.03483329038):0.1877279298):0.9212437507,(t1908:0.9230633206,(t459:0.8965792398,(((t2225:0.1210412814,(t891:0.05017616483,t531:0.9493207065):0.8657229147):0.806501017,(t1650:0.8200322816,(t1235:0.1592528375,t111:0.7393423545):0.8525960937):0.4725027331):0.3224223452,((t1688:0.256521483,t2328:0.2419971554):0.2918403777,t676:0.0827699767):0.7778679132):0.4097151277):0.2327106078):0.9283119862):0.1433613857,((t4723:0.6426415988,(t4681:0.476801927,t196:0.04449872836):0.3522045633):0.8927918347,t4542:0.309316216):0.3803462666):0.005959084025,((((((((((t1306:0.6625155385,t3154:0.5788743051):0.6942271364,t4595:0.6462783546):0.3458511811,(t1761:0.6431226134,t3514:0.474795742):0.4640164622):0.4682177082,((t500:0.04715898959,(t2975:0.1638396068,t728:0.390467606):0.4828180331):0.5105539772,((((((t3966:0.3171784254,t4398:0.1057330519):0.9652270568,(t1291:0.04129211209,t2509:0.008624878712):0.1047771352):0.1940012944,((t974:0.3166699824,t1281:0.1223041036):0.04144057841,t3054:0.06751919142):0.3227787917):0.2302918162,(t3199:0.3355228603,(t3116:0.5217979888,(t2271:0.5784333204,t1699:0.165101069):0.7486459159):0.5733405876):0.7615066201):0.5743735849,t900:0.03434542147):0.5279786836,t4105:0.669628117):0.8647306643):0.9980840066):0.9034371506,((t1286:0.2711363558,t1026:0.2862989199):0.2237319851,(t6:0.07400500076,(t4022:0.3678018562,t4868:0.9102277698):0.3532601434):0.9931948367):0.4737812679):0.4719534579,(((((t3907:0.9337142562,(t1074:0.8272509351,t1009:0.1715387641):0.4012475861):0.08845907776,(t3015:0.6320556086,(t1426:0.9724213271,t1919:0.4914185717):0.7823729701):0.6979540864):0.3046035231,t2308:0.2255856206):0.6747009351,((((t1161:0.6109603324,t1944:0.3022694523):0.9122979813,t2189:0.09345585224):0.3761903755,((t1363:0.3734101611,t2216:0.8175721234):0.352427487,((t1474:0.1373761806,t4539:0.5819771958):0.4617048386,t2020:0.09170772671):0.1470321757):0.08851658273):0.7731607505,t3671:0.9741039332):0.7446371799):0.3371518401,((t4904:0.8478458903,(t464:0.4815310899,t4033:0.5043772548):0.5269971234):0.6340927281,((((t82:0.5138032995,t2267:0.7280561368):0.1798360252,t3458:0.5755626212):0.7099283771,t4786:0.6382588882):0.906457162,(t2819:0.7697539437,t1272:0.5290390612):0.9816745485):0.5191024602):0.7031795261):0.7615391072):0.5544314543,(((t3862:0.8909274847,t4473:0.8033184893):0.6809424772,(t872:0.9600820695,(((((((t137:0.2896960774,t3934:0.5574270131):0.191988342,t367:0.2278354552):0.1320280619,((t3572:0.8455217613,t2603:0.8748849405):0.6991057321,t3207:0.2626712676):0.8769331847):0.4287475673,t1578:0.2641025337):0.1135635301,(t2379:0.2068804726,t3535:0.6995875342):0.9549149347):0.1021306932,((t4761:0.9229997268,t4599:0.5993401993):0.4876780952,t1385:0.3649725451):0.8509466222):0.1570861537,(t3223:0.9277206375,(t2651:0.6944487377,t878:0.01476062997):0.1592911712):0.7858840495):0.6738772807):0.772159942):0.5585829311,(t3806:0.1493738552,(t1785:0.07195074577,t1292:0.3258715849):0.1357428702):0.6573399028):0.9519632084):0.938945764,((((((t2766:0.8301594898,(t3590:0.7848921847,t4955:0.8075683445):0.490549589):0.1866396705,((t1837:0.4750313454,(((t1599:0.9633741183,t1555:0.592134688):0.6933864525,t2470:0.8934372724):0.716476371,(t3687:0.4126009156,(t3177:0.9111810306,(t4071:0.9000572613,t2311:0.5913071034):0.7428332102):0.4588288118):0.2644274982):0.3406621828):0.4788396843,(t4703:0.7337097768,t254:0.1605558367):0.5408725387):0.2482185345):0.3822573805,(t3393:0.9401557567,t3812:0.08276883047):0.1854269009):0.1283881173,(((t700:0.904105593,t2278:0.2956481948):0.4576966716,((t2357:0.5898458927,t1741:0.7654539822):0.9976518925,t4876:0.06166582438):0.4478008831):0.2087158132,((t691:0.2371950878,t2821:0.7662359269):0.733832093,(t3908:0.6222557665,t1907:0.4776410062):0.4917978144):0.1147178791):0.8110312023):0.6142546136,(((t2110:0.4068866046,(t480:0.3765701388,(t3412:0.9884318956,(t4448:0.9171438639,t3047:0.6033808696):0.8318900899):0.6741980033):0.3691143668):0.8100946425,t2552:0.8348319742):0.1272541964,t1764:0.3008317272):0.1648190578):0.4921144247,(((((t4847:0.9897845511,t2013:0.1150005404):0.5742557517,t2100:0.7985914436):0.3909685558,t3903:0.1316637693):0.4614736927,t1054:0.7908657016):0.686560662,(((t3565:0.3815201928,t4830:0.4485210071):0.6334995779,t1576:0.4580239502):0.1816751629,(((t70:0.9830735284,t2468:0.2578766143):0.6959347576,(t818:0.5240444785,(t1531:0.7257173958,t3432:0.09622520069):0.7333663937):0.9187780591):0.1743005598,t4027:0.8889026763):0.7262699015):0.1681794105):0.7337998389):0.2630934385):0.04755302961,((((((t4671:0.7442549909,(t874:0.851199711,t2137:0.4230890083):0.236699943):0.7341618135,t4951:0.7607454644):0.733840876,(t2083:0.04760704539,t2163:0.9825374899):0.110005053):0.4061617136,t673:0.1749120215):0.2873740415,(((((t2956:0.5960845023,(((t838:0.7908684884,t34:0.8710529318):0.7887487609,t420:0.5159612976):0.09884430328,(t4754:0.4050655863,t2135:0.1973387431):0.8113414184):0.3808289689):0.4271105549,t520:0.6433347128):0.3988169555,(t2268:0.2554176836,t2874:0.5965922703):0.3500775318):0.8563295116,(((((t3559:0.8817794996,(t376:0.7587874776,t2773:0.5207101298):0.6402119787):0.07355266227,(t3828:0.1502614792,t3636:0.04580697953):0.3912103709):0.3825840859,t789:0.806178628):0.3769880026,((t4486:0.1778437102,(t3309:0.7644431598,(t979:0.3594361143,(t483:0.5529396688,((t1988:0.930516538,t243:0.7716484417):0.719999789,t1600:0.3973555677):0.2252132939):0.3861434972):0.7647850735):0.64130017):0.2068860447,t2749:0.6730570218):0.2033656833):0.3669486181,(t3758:0.04888459831,((t3392:0.05379986833,t287:0.1515188674):0.4357672776,(t2768:0.672232467,((t1070:0.754875151,t2537:0.5540782376):0.9425495574,t896:0.3974358556):0.761196075):0.8987171769):0.6668362382):0.7504312079):0.714615427):0.5349522748,(((t885:0.991088066,(t2342:0.7582814777,((t1357:0.01434732298,(t822:0.01482085302,t2833:0.2260544251):0.03959428356):0.7109994227,t2774:0.4716866894):0.154644626):0.5385832968):0.7989087331,t747:0.6955810771):0.1711032062,(t2575:0.000914911041,(t1497:0.8655935668,t52:0.3589440908):0.9986947051):0.2012554437):0.4113488775):0.1969548052):0.9451415432,(t4381:0.9372320261,(t465:0.6130136971,(t1455:0.9961758414,t941:0.5282937884):0.4460495182):0.9203294376):0.4709608646):0.06176591269):0.5089698557,((((((((((t1938:0.4506780782,t1614:0.5386361699):0.4906993643,t717:0.9253239811):0.2576218052,(((t2843:0.7066054549,t4948:0.5413063662):0.7606470541,t3802:0.4189484511):0.4081786035,t1877:0.1427718799):0.3787695817):0.679254065,((t1720:0.4620407138,t1053:0.1842573939):0.5910443421,t2607:0.579850161):0.01609432581):0.2693065526,(t4117:0.3403181308,(t1868:0.5548204016,(t938:0.6436053249,t607:0.3325492998):0.2859567988):0.9038223624):0.04803479696):0.2625273401,((((t141:0.8432537839,(t1133:0.9733708412,t1933:0.6802309277):0.266705412):0.2081605464,t4297:0.560999989):0.3155386958,(t2831:0.2022556351,(t416:0.2114946467,(t4036:0.03007476823,t4168:0.7299505037):0.8069894302):0.9685357288):0.6955827714):0.8897945664,t4252:0.373821028):0.07343759807):0.05587284337,(t2909:0.02543121763,((t423:0.229050504,(t201:0.756670919,t283:0.9636300642):0.6951249554):0.5098641352,t1311:0.4837725968):0.9555919468):0.08219328779):0.8008514971,(((((t2533:0.4055384023,t2030:0.148349392):0.7950930458,t4890:0.4953629647):0.8163733261,t4607:0.1730753644):0.9381625033,(((t2103:0.6955712952,(t2359:0.9607756922,(t3158:0.8080149069,t2576:0.2177550667):0.9614209188):0.5909861373):0.8428447549,t2051:0.950081232):0.4309321581,t4973:0.2176441564):0.8256461716):0.3971608372,(t1014:0.008585044183,((t2935:0.9731717333,t580:0.07845517201):0.8090646733,t4262:0.02521286788):0.9061860852):0.7562892493):0.736085708):0.8657910067,(((((((t1962:0.2764282033,t4897:0.7748251103):0.6094185496,(t1514:0.6496626851,(t4966:0.1292345531,t4553:0.2425795549):0.580833938):0.7858646319):0.3833850457,(t1425:0.3251084911,t457:0.5218465829):0.2789259893):0.5587647394,((((t4622:0.6883039747,((t3460:0.05981955654,t3955:0.02930417261):0.4105365505,t519:0.2245433386):0.6105637196):0.2866736674,t554:0.2343207463):0.4727460204,((t576:0.1660534365,t644:0.0896624194):0.08148735994,(t4626:0.07391705946,t4408:0.9781142285):0.3282505835):0.5720485121):0.5328586821,t2074:0.1049654153):0.2173090093):0.2013262406,((((t3582:0.0847752043,t4361:0.7883528466):0.158219846,t4497:0.6677595):0.01568840491,((t187:0.4613066034,t3906:0.1082955068):0.1333021545,t1588:0.7501992306):0.1213510714):0.6527330419,((t349:0.9946205963,t4531:0.1951954158):0.6257631481,(t2376:0.6568077942,t4129:0.4123618789):0.2328181588):0.6667658798):0.6299884976):0.7562745872,(((t3004:0.6346339073,t597:0.620633553):0.357741521,(t161:0.392554003,t4980:0.1988916437):0.3960133914):0.9987790298,((((((t1080:0.4690474747,t1560:0.8713968219):0.7616307552,t3824:0.539999132):0.4185333471,(t1630:0.07416326483,(t4086:0.784059162,t3909:0.2223871201):0.9800751642):0.9936017287):0.7710696119,(t338:0.8138993238,(((t1709:0.7687785625,t2879:0.07628527959):0.2422069267,(((t3629:0.8109712144,(t2826:0.6724647165,t4306:0.3102253233):0.08095337986):0.003892149311,t2142:0.8593762042):0.1722882625,(t2765:0.2857820976,t3282:0.5430412453):0.7162899601):0.5053268173):0.5977762137,(t4795:0.9500683988,((t4662:0.6225377815,t1902:0.9040065701):0.2117790605,t319:0.0185329353):0.6059972842):0.09835731774):0.5309695606):0.4371532651):0.223250227,(t105:0.8974860974,(t3466:0.9317452747,t2090:0.4536145667):0.1194735467):0.3637679331):0.08956605988,(((t4941:0.8772421756,t2165:0.03166935919):0.559009165,t3591:0.1495880217):0.2169536962,(((((t1891:0.803135732,(t2316:0.6746139228,t3531:0.4359723686):0.7968061415):0.7120484267,t2144:0.3050360023):0.05563539895,((((t953:0.883375485,t4140:0.9473110309):0.568522935,(t2747:0.07006613328,t3683:0.9785314517):0.08892077953):0.9886806344,((t2848:0.7819524945,(t1002:0.03516155784,t1568:0.4864505634):0.02226023027):0.205183605,t3543:0.9967197163):0.4352953082):0.9203425532,(t1400:0.8890176164,(t1838:0.07583892904,t2949:0.8483078515):0.6295645658):0.9362367564):0.9273173481):0.1270740789,t2125:0.6900143288):0.3252531351,(t4919:0.5798449882,((t387:0.6013426958,t342:0.799759612):0.8069799629,t2440:0.4402328641):0.2240310528):0.5152902934):0.9833524481):0.2571283078):0.06278632535):0.9387170237):0.8079462145,(((((((t4099:0.1476075589,((t1983:0.7779976348,t151:0.4868836931):0.4398316417,(t2566:0.08049368626,t3719:0.5150664123):0.7501435229):0.2025247561):0.3240894505,(t2977:0.2991218856,t3164:0.1800260302):0.2126411742):0.6573841551,t638:0.3981222135):0.7963230368,(t3815:0.04988141055,t4254:0.4772721485):0.2407619844):0.6450764593,((t4459:0.2004495608,((t682:0.2478228027,(t1799:0.950177189,t1954:0.2891972372):0.1529001885):0.5758770576,((t1847:0.2166110654,t149:0.08891969919):0.4781909729,t2560:0.003338381648):0.8941823482):0.1064330689):0.140564396,((t1574:0.09573221439,t3726:0.4771131929):0.6398814209,((((t4409:0.5210505298,t3274:0.6702788989):0.9170753672,t4128:0.5519625407):0.7459044706,(t2628:0.5195446874,t2978:0.4412475321):0.4762363236):0.9895288527,((t1804:0.6409427726,t1909:0.9536479027):0.4142348801,t2677:0.9229315934):0.6280904433):0.9825764794):0.7517568402):0.2207729982):0.5286096432,((t902:0.7106647748,t2273:0.05418361002):0.7888616519,((t391:0.8722135094,t2386:0.005610590335):0.5780541729,t2106:0.6971302987):0.5430430495):0.8371044574):0.3885032986,((((((t2709:0.2680609454,t4315:0.4109451238):0.9607587198,t4208:0.598536036):0.6926135665,t4190:0.836377145):0.06919118483,(t1043:0.9763812278,t824:0.8327646335):0.6247991573):0.7759258393,(t3438:0.31062946,((t4119:0.3578151183,(t779:0.759313771,t2670:0.4942744419):0.5561653636):0.2905909219,t3983:0.7723350637):0.9930033083):0.8207229639):0.87709307,((((t4883:0.1614186526,(t1828:0.04244020698,t3290:0.4884343925):0.825237399):0.05893003824,((t541:0.6257399672,t4543:0.8651569076):0.1371917431,t2571:0.2904471124):0.3945463425):0.1984697715,t3789:0.7216464027):0.1664548849,(t1296:0.344513501,(t1384:0.633941629,t4797:0.5164134887):0.3241540585):0.998613128):0.6874893839):0.8716411465):0.09805524442):0.5344464518):0.7405186919,((((((t260:0.2507187428,t2177:0.7446666346):0.5740439247,((t3883:0.8917173557,t1481:0.001645839773):0.9502385533,(t470:0.9070524068,t3536:0.9270657683):0.6161576016):0.08720533527):0.09771193657,((((t1463:0.9468360564,t4312:0.003088414203):0.5892880079,t226:0.06393030658):0.1208121611,t3610:0.06126804301):0.09399031359,(t2173:0.8652643771,(t1932:0.3783735612,t3807:0.4121621393):0.6579337139):0.9226838665):0.5853285412):0.5338085641,t2543:0.03012549598):0.9833827191,(((t4985:0.4891514953,(t3115:0.01249902183,(t41:0.429718443,t4489:0.7454760836):0.4824624101):0.7208294566):0.505386624,t1849:0.8315349389):0.8382718307,(t1866:0.4240220538,((t1739:0.9013967407,t914:0.592842666):0.3357350887,t2181:0.6979792251):0.3281055274):0.8828910629):0.1753245231):0.444046498,(t3787:0.4238979246,t2592:0.5759206994):0.07062226883):0.8195602051):0.2778694844):0.6936475914):0.7454872169,((((((t331:0.001258900855,(t1842:0.8426810028,t4604:0.101900626):0.8921731319):0.910922359,(t2166:0.2822414548,t3837:0.8861320738):0.5661305289):0.2784726473,t528:0.6692687559):0.09026659932,((t4038:0.688137731,t3869:0.6507896513):0.7819873746,(t10:0.09832983511,(t1964:0.01783519634,(t3295:0.01810539048,t4429:0.3056126037):0.4684832208):0.3024870628):0.32408983):0.06554584857):0.6915811913,((((t140:0.538707447,(t1813:0.9737908228,t406:0.9146813895):0.0139404072):0.5179335913,t1395:0.1671022053):0.06630506739,(((t1846:0.1745882304,t575:0.06606488372):0.05121920654,(t3750:0.8074101179,t4218:0.8760551764):0.6904774948):0.5398307,(((((t1404:0.4372743058,(t1645:0.1120327292,t862:0.1870628074):0.8171742591):0.5200770739,t4727:0.6965951959):0.70958078,(t3808:0.6628386923,t1061:0.6355812792):0.6660233163):0.3928669863,t2772:0.9452636368):0.02398764086,(t2395:0.04642515653,(t1517:0.01527130464,t2851:0.6325122863):0.9750993203):0.004411956761):0.18187857):0.1194557415):0.4567462869,(((((t2958:0.04278241261,t4718:0.007581777871):0.1497793843,t1819:0.6422391315):0.07796975272,t2611:0.6261486476):0.7246003412,t3037:0.03563066735):0.2424288741,t972:0.8600773131):0.4660500409):0.4862626691):0.656405237,(((((t1683:0.8622007535,(t3948:0.7465377178,(t1375:0.7756871185,t2466:0.3822525861):0.3342086221):0.900145835):0.9607019848,((t2410:0.2016909418,t2407:0.3658506537):0.08481021901,(t4078:0.6641064496,t2891:0.2501286268):0.7556717696):0.9054881155):0.8113460585,(((t1472:0.2535919365,t2778:0.7864895789):0.7628663729,(t1524:0.2093437398,t381:0.9656464823):0.7172290673):0.2402605833,t1125:0.4865085741):0.830634614):0.05871765199,((t325:0.7927478591,(((((t4289:0.4740466215,t9:0.3509804467):0.4462221311,(t3507:0.7460888659,t4324:0.0004911550786):0.3317674983):0.5092310328,t4712:0.2344937809):0.8969759038,((((t3263:0.2383883861,t1309:0.52299398):0.7838380616,t553:0.6410430628):0.1125996716,(((t1453:0.891159622,t4831:0.6445487945):0.3789737951,t4239:0.7697436521):0.6862481597,(((t1312:0.9210228608,t1392:0.8763652968):0.3142630875,t3733:0.2371287656):0.2141387952,(t417:0.1014713002,t2075:0.9296532159):0.2078870747):0.8053944334):0.7505275842):0.4161074904,(((t3462:0.3079894737,(((t2174:0.8830183835,t2156:0.2814247061):0.2028957107,t4875:0.711153432):0.3652879107,t4902:0.1109696019):0.8968304184):0.805462094,(t741:0.5453510007,t926:0.272668076):0.6667220898):0.2711983498,(t3980:0.643249108,t3901:0.8473828575):0.5015894428):0.6294423505):0.6942864221):0.7828101702,t2499:0.5651829729):0.03756267065):0.7182828856,((t4325:0.3843760965,(t4371:0.6752849137,t2497:0.8321610214):0.7038713084):0.4947092605,t3440:0.09533341741):0.9511569033):0.9916526615):0.5126578892,(t4829:0.867901766,(t3624:0.09966382943,(t2340:0.8381469827,t25:0.229025753):0.9239300077):0.9157600983):0.8570160838):0.7212946026):0.05614505871):0.1347896699):0.179004024,(((((t4183:0.03498433414,((t372:0.5551300333,(t3780:0.1108565032,t281:0.7668217737):0.2132483809):0.2269229502,(t4969:0.5026890975,((t4921:0.861788224,t1981:0.8658422679):0.8223575565,t4328:0.9785240921):0.09359156899):0.7693149552):0.5861181945):0.4189771602,t3271:0.3634349667):0.8514026599,((((((t2411:0.8555425031,(((t1545:0.8805351097,(t2617:0.9539210703,t1945:0.02046124288):0.1986357721):0.4555004118,t928:0.9091663386):0.2866201417,((t2846:0.8774828447,(t2234:0.2789092725,t1982:0.5240008282):0.7394791341):0.9824146666,(t3511:0.58374977,(t1978:0.9524354003,t1897:0.2839191675):0.900977982):0.4702134603):0.8745409644):0.8755603663):0.4850559512,(t4784:0.8569935164,(t1238:0.3834544918,t1081:0.6871774152):0.7750321925):0.7383749511):0.2173684193,(((t737:0.04707947047,(t2315:0.0131335035,(((t1085:0.1442769289,(t688:0.4225948318,t4273:0.06332091894):0.360167366):0.6402588482,t584:0.5241686848):0.04551452445,t2705:0.3695312897):0.9571782465):0.8510107771):0.1973716598,((t3142:0.2662496693,t2585:0.176249011):0.1439375351,t4825:0.8061203554):0.5573487636):0.9281474296,(((t566:0.01602618676,t4519:0.8811376041):0.01258683391,(t3918:0.8847400728,(t4990:0.04467654042,t662:0.1359336996):0.5715997461):0.4379778684):0.003358415561,(((((t3044:0.7272873225,t2368:0.5111867257):0.7907419445,(t4236:0.5922578021,(t4307:0.79216597,t327:0.4962332442):0.4674668859):0.4607940794):0.3805021483,t3235:0.4245512364):0.1447227809,((((t4630:0.4166991047,t98:0.360728845):0.7336044635,t2006:0.1058789492):0.5290927105,((t242:0.03397620865,(t232:0.792025676,t2945:0.7342344609):0.8379193454):0.374333113,(t592:0.3060678809,(t888:0.3819731651,t2313:0.6949764304):0.6168746925):0.453013896):0.07009968231):0.219409008,t4898:0.07854854059):0.7888847322):0.1785667145,(((((t4231:0.5565471456,(t1995:0.1051999792,t4182:0.1068127737):0.30994653):0.3943217176,((t3996:0.5026715908,t1722:0.161997909):0.7489924845,t948:0.6202464416):0.872788925):0.9570072107,(t3859:0.4782486556,t1607:0.2676505139):0.7284537461):0.6724800514,(t1729:0.08233478293,((t351:0.3908062752,t4039:0.4162951536):0.4119176301,((t2546:0.6391671516,t740:0.8678360889):0.4036977089,t2431:0.6461039258):0.9415479084):0.1670047916):0.5186912704):0.9147595547,(((t1586:0.5844852191,(t4590:0.2725288377,(((t83:0.6073071496,t2820:0.790920946):0.2633254107,t3314:0.8223320653):0.6128519936,((t3195:0.3681525749,t1432:0.8363896655):0.1537865219,t4721:0.1421761694):0.5124192131):0.3938512199):0.98133908):0.8061394785,(t4521:0.3266725265,((t3888:0.9370659885,t4052:0.8451652578):0.109648067,t1342:0.5737883619):0.9790101415):0.5815594292):0.622520054,((t4559:0.1039693684,t1203:0.7782272329):0.02401937777,t4120:0.2803581553):0.7174880006):0.4154839823):0.4478431528):0.9217911873):0.2020240971):0.8064125488):0.6079810599,t1636:0.633023788):0.7194870701,((t4437:0.6211200338,t4924:0.1343696644):0.2199261931,((t4442:0.7164402893,t3645:0.004344208399):0.2203782578,t4668:0.07003209088):0.8123746226):0.1622884963):0.274353212,((t3068:0.02712885779,((t3355:0.3490721274,t2439:0.7126059581):0.1576623369,((((t1940:0.3227759032,(t968:0.6473166358,t1666:0.5610308102):0.07277433854):0.8237063163,(t3389:0.8605106589,t3287:0.4660388273):0.9579659405):0.063101626,t2297:0.5864622192):0.777613508,(t3882:0.9162227951,(t1401:0.1600445909,(t3347:0.5035923193,t4228:0.6511271535):0.6863384787):0.5673014047):0.5360833576):0.7150119154):0.327039612):0.2326927281,((((((t4154:0.04322365928,t4406:0.3913266102):0.5922087433,((t3493:0.3591241143,((t1115:0.1067133374,t17:0.4835998525):0.7195002707,t491:0.3722579293):0.4519817093):0.4757496524,t2417:0.9235911733):0.6281368455):0.8461889941,t4625:0.3438730766):0.1781335664,((t2827:0.002891597338,t4677:0.557582486):0.9978079449,(((t131:0.4150625707,t2361:0.6444458757):0.9158283938,t2631:0.6905614906):0.9060758918,(t4276:0.8273106213,t3884:0.5526829483):0.4579119771):0.08785636025):0.2332825679):0.8059410187,t3814:0.7399132957):0.9743419376,(t2525:0.05654286896,(t2601:0.8358298468,t3965:0.9118884327):0.7776587303):0.7113744542):0.5372428887):0.3227321182):0.6413298144):0.6290934349,(((((t4041:0.5228962381,t1627:0.001575568691):0.867952934,t3574:0.01398085197):0.8202219894,t621:0.9621890488):0.392775184,(t3540:0.1689535941,t2981:0.5189318226):0.9914790229):0.1864537774,((t921:0.3107247734,t2092:0.6435538926):0.6337020474,(((t4194:0.3539550863,t4836:0.748016512):0.008191016968,(t3430:0.3700572492,t189:0.08896864904):0.6616010615):0.556941998,(((t593:0.5615417028,t2712:0.1911697255):0.5061164447,(t2453:0.1456847051,(t3521:0.02072023903,(t2405:0.3015296606,t3239:0.9455536096):0.9027137982):0.5238690295):0.167392408):0.6318228755,t4082:0.1265681905):0.1759760506):0.7682713617):0.1285495104):0.7818435784):0.4639461278,((((((t1065:0.3649198841,t4462:0.9803476299):0.8572854886,t2802:0.9177466545):0.6354648578,t2158:0.8330703538):0.6835954026,(t4095:0.9640308581,(t3914:0.08639404899,t4878:0.2956093063):0.915058604):0.5658768245):0.3130760377,((((((t3513:0.4926985763,t4742:0.1713720474):0.03446522658,((t4944:0.03274010425,((t3881:0.1834514337,(t2133:0.9456696447,t4834:0.9915652992):0.7238658152):0.2551717979,t595:0.001773009775):0.4606404393):0.6174675324,t2416:0.5957349553):0.5638072698):0.5898884651,(t3940:0.4219544439,((t4348:0.3323780179,t2554:0.8437642804):0.1190564423,(t714:0.4813977727,t4564:0.0843265627):0.2549812302):0.002373819007):0.9509006022):0.766826828,((((t2382:0.4162371904,(t4211:0.3241011822,t2813:0.5665224805):0.922518908):0.5596825439,t1248:0.9273098833):0.3171474424,t932:0.2255773256):0.5681617181,(((t851:0.04758917517,t2490:0.5037428553):0.4807859433,(((t3328:0.607637825,(t1036:0.7380286271,t2413:0.6477231707):0.6884423664):0.3673243951,(t4438:0.8178327184,(((t1007:0.3420159628,(t3376:0.526880641,t4922:0.9159669683):0.7963963007):0.7091333291,(t28:0.8811329848,t1858:0.4192148934):0.2757636309):0.3399364909,(t1717:0.5030125584,t4517:0.2275296324):0.07253484474):0.9957819344):0.3820185855):0.4518062847,(t1062:0.2145344666,((t792:0.466161239,((t4562:0.9103371883,t2869:0.6373814903):0.3392106562,(t1537:0.5806159235,t669:0.8572312908):0.3112449534):0.4407875263):0.6739137808,t193:0.5247864882):0.01056895312):0.2867935977):0.1432283572):0.3998494314,(t2721:0.3427498876,(t160:0.3079014064,t139:0.8181248563):0.3993248921):0.3385442861):0.4512501115):0.1643289668):0.6372160485,(((((((t1809:0.2743742915,t1374:0.07588838972):0.163215315,t203:0.08692997764):0.9977863396,(t3833:0.05709264893,(t3950:0.930542568,t4698:0.5188817128):0.8798279769):0.5122426336):0.1084716122,(((t4639:0.3797747679,t3921:0.4872465627):0.4696924475,(t1360:0.1954839027,(t1542:0.09081384027,(t4003:0.5133987428,t132:0.1346397384):0.5166641658):0.1923890305):0.7222856875):0.3579873652,(t2844:0.6246628668,((t3617:0.2293634242,t4445:0.8847098707):0.5363692441,t3036:0.1581476999):0.2868516413):0.04240048537):0.9688619762):0.8906615442,(t4960:0.02161891223,t2522:0.3146707753):0.7744397575):0.9448928288,(t1763:0.175089522,t3632:0.5565167926):0.9762002903):0.2005018219,(((((t1372:0.3979482471,t3873:0.1809465187):0.1026419406,(t3405:0.6877240995,t317:0.2236579154):0.1948082095):0.6740453325,(t1303:0.6145745746,(t328:0.9974307073,t191:0.4738072644):0.6517810205):0.08336393069):0.6262360588,((((t2775:0.853323482,t4501:0.5678611014):0.8511348616,(t2735:0.2143761676,(t3349:0.5723843139,t2197:0.2797142311):0.428057692):0.6681310623):0.1016703749,((((t4545:0.8446875187,t4546:0.9586555904):0.910577344,(t2010:0.9690103161,t4507:0.6674968465):0.5998386431):0.4938527758,t364:0.6595143478):0.6369382769,(t4790:0.8872128958,(t4806:0.6912033148,t4081:0.3098049029):0.7651461118):0.2540600298):0.5727399732):0.2310874076,(t1646:0.6274053974,t4770:0.1649599127):0.1415438459):0.7863129312):0.1311176214,((((t4480:0.02431422216,t3161:0.4293422059):0.974740233,(t3413:0.2783182992,((t1378:0.5338815884,t3449:0.1499826792):0.9598109883,t3986:0.1572936685):0.5383016961):0.2119678424):0.1462068879,t2598:0.5835357963):0.5385114672,(t1590:0.08735408424,(t3866:0.6356867389,(t3452:0.4849746742,t1390:0.1203193658):0.6014444688):0.6266944509):0.813703368):0.85233804):0.4757138109):0.5299160364):0.5006791628,((((t2084:0.06776476582,t4692:0.1935749054):0.2812043815,t2593:0.7352479929):0.402071689,t320:0.3255034469):0.237820639,(((t3829:0.1052221477,t589:0.3303739447):0.5244130429,t4914:0.7424133946):0.2030139873,(t486:0.8172420934,t2257:0.3449840739):0.9405021402):0.1823198164):0.1331453652):0.1866337503):0.4044701776,(t40:0.7396954286,(((((((t3002:0.6852158969,t946:0.5584484143):0.7990920604,(t104:0.06902575097,t3494:0.8091755153):0.8689501577):0.1988386987,t386:0.2276676809):0.1649681414,(t2285:0.3375814916,t3699:0.9767505212):0.6940094689):0.7207634128,((t710:0.06697947322,t2786:0.9867426108):0.1962887687,(t537:0.5994637476,t3082:0.8050547156):0.7374164038):0.9975958951):0.8182417778,(t3422:0.7033255326,t4331:0.3326557186):0.2293707686):0.09979112586,t4793:0.923151321):0.349182544):0.8866503311):0.3725876003):0.1390460476):0.2280120251,((((((t1973:0.5681348487,(((((t3229:0.7805157392,(t466:0.3327716063,t4152:0.3817971484):0.4025114547):0.6651734647,(t732:0.5434208023,t156:0.5412850794):0.3237201551):0.1447944236,t2219:0.9102349412):0.4823831595,t1184:0.4072862128):0.2450032064,t911:0.2835888166):0.5377613746):0.1189110647,(t1122:0.2377422729,((((t1380:0.7807854339,t1308:0.2072210058):0.5720790091,t3222:0.8536675507):0.1517659891,((t2228:0.5361712012,t2138:0.4783078234):0.1864071768,(t1698:0.6768435568,(t2235:0.4367993083,t611:0.680078374):0.05972482287):0.5225880402):0.6123893568):0.8062817527,(t2967:0.7356431375,t615:0.6619515701):0.6836880955):0.9685213517):0.945118469):0.2978463729,(t3183:0.3086081978,(t3951:0.0532471654,t1760:0.03415024164):0.1994543918):0.4938798756):0.4231613828,((t4396:0.1972055018,((t771:0.1347459347,((t4220:0.4199971568,t471:0.05849102349):0.8688212584,t1265:0.2717101292):0.64392026):0.3391475659,((((t4928:0.9370449199,t2302:0.9355290595):0.8499020159,t4106:0.551449612):0.3629592422,t1210:0.2066633799):0.932586422,((t1589:0.8193950169,t4138:0.1252328053):0.2386185986,(t4512:0.05892444891,(t3508:0.3232453533,(t2987:0.8841003783,t1824:0.7019936033):0.5719101583):0.6873349322):0.3162428355):0.8067172014):0.8100727643):0.3706360622):0.7185237675,((((((t3811:0.02107680356,t3834:0.2368205786):0.5584657642,((t4017:0.3043216916,t4695:0.8510380911):0.2780501875,t4989:0.03551002359):0.6405151556):0.8730203134,((((t3236:0.8442994095,(t3831:0.2763952499,(t4528:0.2636937087,t587:0.9072120709):0.7550555654):0.7838176035):0.8740844177,(t4556:0.09125507181,t2374:0.007643377641):0.150889264):0.7683658656,(((t2633:0.7992068757,t2303:0.02005536971):0.6293278912,((t1470:0.7725120594,t1971:0.5546999378):0.8203329467,(((((t4470:0.5059940862,(t906:0.8871804071,t3215:0.4986258205):0.05140326847):0.5173163663,(t957:0.01117769373,t2021:0.3229423512):0.444803556):0.7477383774,(t1139:0.4625291994,t2331:0.5197766582):0.1441617727):0.2521233738,(t733:0.02769374033,t1685:0.8955042909):0.4162672008):0.5213008074,(((((t3684:0.08856470603,t4269:0.2662818376):0.5561260013,(((t2095:0.4003607864,t1365:0.9543576112):0.8567097997,(t3182:0.6542554642,t3067:0.5708788061):0.8750566638):0.7847285131,(t2294:0.4903427805,(t4936:0.8913810854,t837:0.3679510078):0.5707110872):0.9375580689):0.568499259):0.4040739257,t1049:0.2311931355):0.7079991836,t2985:0.9063880169):0.462552292,(t1782:0.8283266071,t228:0.8130566911):0.9290688548):0.6042019625):0.1213189084):0.6614465045):0.09665516275,((t493:0.1447340774,(t329:0.6704639406,t3975:0.03804422752):0.2193056867):0.5729851332,((t4499:0.7190933293,t4385:0.7394440654):0.7739107006,t3325:0.7804009675):0.7355467891):0.5388434415):0.5661206443):0.7432524425,(((t3877:0.4450442507,t2569:0.1403219113):0.9407719139,(t3357:0.02589368471,(t446:0.4004695402,(t4837:0.4379605721,t3424:0.3760701779):0.8215350939):0.3660148084):0.4466754429):0.1294153058,(t847:0.213489006,t2263:0.8867688843):0.8737378272):0.2284750796):0.2603327467):0.899828607,(t4295:0.05474576983,t4993:0.4926645362):0.2793980078):0.7331618185,(((((t404:0.2692052699,(((t2875:0.2962249662,t4970:0.5814666576):0.5213604465,t3111:0.6784509725):0.7169421795,t3791:0.07321951678):0.5351432806):0.8071425313,(((t1947:0.8714264557,t3927:0.8236694389):0.5862342878,(t2344:0.6687899644,t4415:0.5754951099):0.1105119165):0.3638015448,t3144:0.7900502102):0.40638194):0.9886402194,((((t164:0.03858151101,t4702:0.5568432503):0.8817469862,t2295:0.9559942377):0.04625429818,((t4481:0.5223434814,t3477:0.4114326495):0.1025199392,((t619:0.3386846958,(t1321:0.6779230861,t4467:0.4992252209):0.6786007786):0.02111861389,t2609:0.7472152608):0.8529291435):0.3655042709):0.9912209187,((((t2531:0.8329237453,t3771:0.4078233151):0.3647480933,t3317:0.1598947728):0.4351535088,(t3253:0.9512488341,(t2688:0.5810862109,((((t1135:0.2399002539,(t4851:0.3016948188,t514:0.5169305124):0.6658709128):0.3822170547,t4673:0.5276661867):0.7872603245,(t1796:0.6387306058,(t3712:0.2599982698,t2667:0.7856275097):0.6384053552):0.9936061206):0.6556682584,(((t2634:0.3005347394,t2906:0.2392392647):0.7344452916,t1674:0.8617159489):0.3230981717,t4548:0.5729222409):0.3774140733):0.3561273788):0.8041099315):0.08594133356):0.2224021014,(t165:0.3756423013,t3598:0.4661686651):0.2325377406):0.1371505742):0.5309759991):0.445349995,((t4101:0.733551767,(((t3333:0.2547055841,t4638:0.4853408784):0.1054612524,(t3245:0.3426477602,t274:0.4544992005):0.2099003398):0.5370014843,(t2807:0.2238084555,((t2799:0.7879253177,t2926:0.6336122488):0.2761631515,(t4009:0.02825549059,(t1826:0.7701620583,t739:0.3638238281):0.4894896767):0.5767587626):0.6722433202):0.1698046851):0.04157164111):0.276389024,(((t2770:0.4858519675,t124:0.6178175132):0.5646765314,(t4821:0.5258944745,t4261:0.6343801415):0.2452167862):0.9520252817,((((t57:0.1727824407,t1087:0.5076099653):0.6837265086,t3220:0.3042535151):0.2148457123,((t3638:0.8380151535,t315:0.1557895457):0.7073305079,t3300:0.4930051169):0.2224458547):0.1813289518,(t24:0.9768751285,((t1258:0.9233513507,t3588:0.5852432335):0.9873646139,((t2332:0.8053701464,t2510:0.4986264673):0.1329056283,t4419:0.5379136784):0.382758087):0.949759685):0.7425863803):0.2592336817):0.7542579756):0.7333036403):0.06144924741,(t454:0.1460019485,t1719:0.8226911214):0.03393371101):0.4070150023):0.8204112744,((((((t2292:0.748293052,(t2196:0.3429385978,(t2381:0.2443336579,(t2131:0.9729624165,t2507:0.8186007715):0.3065986007):0.2684547822):0.2586890231):0.3386101869,(((t3878:0.7278338587,t880:0.6147270235):0.9365753622,t4351:0.7659432772):0.2398818748,((t3374:0.8922289826,t2203:0.2460515513):0.9668913118,t720:0.7018498126):0.1789269715):0.1218655624):0.861960572,(((t4268:0.6640417795,t2017:0.8501156636):0.7050163336,(t3666:0.1271682917,((((t2596:0.909126516,(t795:0.4240204785,t2284:0.1429779688):0.4582759645):0.1643847714,((t3708:0.8361999497,t2213:0.6154832179):0.2140542411,t4238:0.9220388066):0.4539744849):0.1023058428,(t3340:0.03208618145,t50:0.8600961163):0.06177349994):0.5789808799,t2854:0.1229858967):0.6482802005):0.8009758056):0.3255659833,((t440:0.7058783264,(t192:0.8405572474,t1948:0.0250730163):0.1919589711):0.4904956671,(t4711:0.002739443677,(t4998:0.8118480658,t23:0.2753967468):0.6499649023):0.1824278953):0.0004081011284):0.09842019086):0.8613727053,((((((t2968:0.4552056591,(t947:0.3533337654,t2419:0.477048564):0.7395741297):0.6098678717,t388:0.4350367251):0.834151759,(t4428:0.8294762154,t2946:0.3494584237):0.02741308277):0.2273236816,((((((((t4576:0.2502201002,t4008:0.6835207734):0.878534012,(t856:0.9568232943,t1211:0.1636032383):0.02496005199):0.6843170957,t4043:0.1733671159):0.2716424514,t3905:0.5589628308):0.7930894364,(t3912:0.9776230401,(t4884:0.5602029441,t3825:0.5106574355):0.1959099527):0.01253564074):0.8810927535,((t1765:0.1217542361,t3569:0.5116582713):0.9754076379,t354:0.5338736952):0.7047852322):0.8536742842,t1952:0.6875796088):0.1722936532,((t3688:0.0747548691,(t1327:0.5918712898,t2450:0.4797310829):0.5419676448):0.461589844,t1859:0.5983966419):0.4327027961):0.7291815819):0.7381163279,t2306:0.3976691964):0.8085656778,(((t4064:0.8399842945,t4612:0.01232564542):0.8946208444,((t3510:0.7850880802,t2008:0.01812405279):0.5891563015,t1728:0.6169143638):0.8403662604):0.4019139621,((((t4292:0.6755691119,t2231:0.1982526924):0.3153131849,t763:0.3400511742):0.9811609718,((t3179:0.2933150055,t3415:0.6545480902):0.2167472199,t478:0.5715648343):0.6222857975):0.5494367834,t4535:0.1612658466):0.156308094):0.3467787418):0.591101317):0.3966062542,((t306:0.2069458584,t3007:0.3139159668):0.09378669481,(((t626:0.9913690069,(t7:0.03045098367,t71:0.7070615049):0.302808278):0.07598233898,t4532:0.6809716558):0.0894019336,t1094:0.5922333098):0.7046006129):0.938991586):0.2646000248,((((t4733:0.6874674284,t3187:0.2101472316):0.4669775963,t30:0.1291112311):0.07232087408,(t146:0.006808004575,t4583:0.5057103497):0.1183745698):0.1367379162,((t1083:0.9516856624,t435:0.7188548846):0.9037920733,t2023:0.4446699813):0.2243572443):0.9656858682):0.6086111453):0.6338212332):0.8866167373):0.1765589612,((((((t2423:0.1931195427,(t2534:0.8265983288,t3095:0.8243496944):0.9685227429):0.818495837,t218:0.9061410672):0.1691182684,(t2979:0.2181168981,(t4603:0.3930472101,t3385:0.7549666502):0.3901740469):0.7256010091):0.3328026992,(t4652:0.03233881178,t3107:0.302586054):0.7455977481):0.7628007988,(((t1266:0.983895601,t4699:0.9588194846):0.6439623553,t2185:0.6627187505):0.1626397525,(((t4383:0.1369052392,((t2663:0.6139643532,t4749:0.1409131789):0.4238608999,t590:0.9145384533):0.3411657589):0.8307616871,(t2536:0.1972671619,t1592:0.5925870582):0.6297995944):0.7798964998,t1776:0.4173541584):0.6133281582):0.3215427741):0.6857114967,(((((t2486:0.2534273854,t2275:0.7503685441):0.8798180674,((t2298:0.8103414394,(t2393:0.6213371884,t3630:0.09625232546):0.5807391966):0.2528979054,(t1690:0.2659385919,t2963:0.6608020701):0.7823831602):0.5500285327):0.904479346,(((((t4823:0.6681049403,t3012:0.4771814465):0.001934527187,((t3445:0.4074460873,((t2073:0.05536987958,((t339:0.5311197552,t2984:0.2774925723):0.947368707,t3414:0.8739840162):0.2069304287):0.03447289602,(t2326:0.02159788716,((t4175:0.7152250065,t2763:0.8634950928):0.1842890319,t2973:0.7143661538):0.5395991243):0.8536179641):0.2181519282):0.7373283878,t4962:0.3794792204):0.4467060424):0.4920569763,(t1665:0.3566718965,t4716:0.06141600548):0.6661270389):0.2428287917,(t3121:0.9890797352,t3163:0.447577527):0.3350870924):0.3109745036,(t2645:0.4413208286,((t4569:0.493079684,t692:0.9750845334):0.4223562998,t2665:0.3293162403):0.3930454468):0.3956420592):0.6204367881):0.9429676086,((((((t2430:0.5746829135,((t2387:0.1351242282,t2952:0.4281056658):0.7893913621,t324:0.2337104923):0.05011863238):0.8662156262,((t3627:0.2208117065,t3162:0.9804464094):0.01925483998,t1373:0.3310051456):0.6589865861):0.1584067957,(((t1097:0.7363349653,(t3461:0.1723615169,t4395:0.2414065036):0.9996072757):0.6580929365,(t3208:0.09314439818,t3308:0.3058410604):0.6357309732):0.2954049155,(t2435:0.4997220875,t4019:0.05289115245):0.9358429248):0.588728331):0.1243982443,(t3441:0.3734580029,t1528:0.03928226442):0.8957787689):0.8023243253,t2750:0.8384765207):0.8156214131,(((t135:0.3041282676,t3383:0.2028905882):0.2016111831,((((t2447:0.3909312193,t1810:0.5047113306):0.5381376615,(t3298:0.3905494898,t1071:0.4519690571):0.1127350007):0.03558178642,((t4657:0.8147869897,(t1783:0.4115657948,t455:0.8079520438):0.5385496116):0.3866672067,(t1388:0.7102561847,t4549:0.4729876851):0.6076403616):0.6183508371):0.09672315838,(t1852:0.9862481051,t2337:0.5894514318):0.1018238291):0.9706920604):0.5705323697,((((((t3418:0.630589288,t1275:0.552800958):0.626953651,t356:0.8769844605):0.8894203347,t2160:0.8381429922):0.06424663588,(t760:0.2627444521,t434:0.1229700453):0.9742061791):0.8042677077,t3447:0.187801935):0.9826317083,((t4996:0.6019638295,t2947:0.2761884399):0.04066902562,((t4335:0.04200083041,t2254:0.2211135556):0.7957244494,(t4040:0.1525988658,t1958:0.7806118259):0.1824159967):0.2243631266):0.2031931076):0.9341691528):0.8750062145):0.985997125):0.2238566605,((((((((t2748:0.5147456988,t1450:0.1291040692):0.282114788,t2785:0.05630724551):0.2112486598,t2551:0.2134872985):0.3733677457,t4141:0.5857856236):0.2292994589,((t3201:0.5778236259,t803:0.7630242116):0.3694555731,(t808:0.4002081675,(t4290:0.6134481151,(t2151:0.984259336,t2388:0.8453873931):0.5660084763):0.3215975091):0.5562006109):0.1521002862):0.995996942,(((t3077:0.9619504833,t4030:0.5840772144):0.4716762679,t813:0.9034354612):0.3038505164,(((t1750:0.6701757808,(t3041:0.2480250197,t3767:0.3457303897):0.4241412666):0.02314847778,((((t3350:0.01810546033,t4945:0.4614656721):0.6157923194,t3375:0.341232372):0.9711858954,(t3221:0.1464637632,t4317:0.7987553533):0.5066201154):0.3606127743,(t2035:0.405798197,(t3103:0.747801837,t1925:0.7850207575):0.5787311855):0.1865394271):0.7217439034):0.9823501331,((((t1570:0.8203902727,t965:0.1333067529):0.9048731402,t1130:0.9299169162):0.5035231116,(t1559:0.5921176076,(t3048:0.3960266709,t1136:0.3922457895):0.3909584808):0.9072791217):0.1079079628,(((((t2009:0.4942429264,t2099:0.241926902):0.02628811472,t2363:0.4211727823):0.7634884361,t1778:0.9654949405):0.9248338398,t4326:0.7726047933):0.4332441075,t2937:0.5632132294):0.795783286):0.7780447309):0.932150177):0.2160618624):0.1000490414,((((t506:0.2078445498,t2488:0.889951654):0.2628886627,t777:0.07831284381):0.7166936682,(t2130:0.5421452236,(((t4848:0.5781095168,t1277:0.1640591684):0.7114398866,t2885:0.2409533134):0.9576027545,(((t4302:0.3168382023,t754:0.6732195586):0.4850227439,((t4567:0.1100374947,t2348:0.7073331592):0.8015286857,(t3172:0.3295738439,t1442:0.4322834858):0.2422076396):0.1587184076):0.3766002597,t4150:0.6743513003):0.5084877575):0.403385967):0.5630816938):0.5360187853,(((t3379:0.5832756017,t2319:0.7406214147):0.4977829461,(((t3200:0.4904401915,t4083:0.7751636889):0.6517900915,((t2564:0.2804510219,t2168:0.3410268098):0.2062114044,t1966:0.4478792124):0.920973025):0.9055810389,t3302:0.2922328282):0.2938925768):0.2391921822,(t2652:0.7000637052,(((t1205:0.5777065705,t1020:0.5073747342):0.7099037117,t1594:0.887284691):0.9458675417,t1867:0.7512733827):0.8785453064):0.4940198082):0.624363808):0.4717816585):0.2085660666,((((((t572:0.379532306,t4070:0.6123310712):0.7303390841,t4524:0.1788888681):0.2872162133,(((((t1088:0.3532999163,t1264:0.4849720895):0.4018452859,t4014:0.03348378837):0.8097322928,(((t4557:0.8371669352,t3339:0.2984669444):0.9601581199,t721:0.5613423467):0.7700469464,(t770:0.8289872683,(t2505:0.0979451118,t2698:0.6768026797):0.5925026506):0.6979261627):0.33731049):0.4637486436,(t920:0.6574982002,(((t3734:0.8100221425,t2716:0.08582024323):0.03301334707,t860:0.5374278997):0.719058923,t1851:0.8791913262):0.6500799684):0.8640943333):0.2830932396,t3704:0.3622485015):0.3388570235):0.3210957269,(((t266:0.8070734625,((t2838:0.5118395165,(((t3805:0.9000154028,t1772:0.9251944011):0.09063745057,(t68:0.1498751407,t1616:0.7297034457):0.1283029625):0.8368552891,((((t1534:0.1605187713,t1612:0.3844619761):0.7593841029,((t152:0.4901892631,t4788:0.2173398167):0.1914218902,t647:0.2524600618):0.2133166476):0.3881987894,t4879:0.04109584726):0.3808377746,(t173:0.7821991851,t3891:0.7861911105):0.2433159414):0.9594336278):0.156430877):0.4771051374,t1216:0.5008824256):0.01799923601):0.2551365206,(((t950:0.3982100484,(t2220:0.5394569819,t4935:0.3081180819):0.1425234729):0.9693913762,(t2805:0.2292339762,(t2129:0.7433598125,(t845:0.2803928845,t3304:0.7964382907):0.2958148243):0.6521595784):0.6605819915):0.09217150649,((t2817:0.1206720865,((t3713:0.5433490546,t2211:0.2905615226):0.06590573862,t1931:0.9975637584):0.592594242):0.4749389139,(t1771:0.4169110474,t395:0.9526264132):0.7062237912):0.8062261732):0.9369536764):0.02601226442,((((t3776:0.3378560282,t2691:0.8420331138):0.593576598,t3779:0.9705707263):0.5653205868,t1341:0.4235697424):0.4848375767,(((t1530:0.8114094511,(t4895:0.9634755745,t3143:0.5191032316):0.9955240546):0.7111812828,((t624:0.4989298831,t3205:0.9874260847):0.8355864293,t565:0.8201916947):0.111524242):0.1626242092,(t4347:0.3448880739,(t3146:0.7751316263,t4492:0.9612808193):0.04527360504):0.0450264432):0.6674791523):0.673401268):0.8590438054):0.5752116239,((((t3745:0.3185973421,t4940:0.04776686919):0.07478568028,(t244:0.2252970303,t551:0.628346398):0.03934562579):0.6898457739,(((t4319:0.4919053507,((t705:0.1891188803,t4265:0.8596725466):0.7656589863,t850:0.3478679662):0.9094135438):0.6368377386,((((((t2614:0.6073432744,t76:0.5420673529):0.1312365578,t2914:0.171136348):0.8098387904,t3783:0.6902297013):0.6755238217,t2923:0.6870346002):0.6546089521,(t3016:0.2255403192,t4145:0.6210981531):0.4195707596):0.9879787012,(t2658:0.3318476437,(t269:0.1757362757,t1032:0.167580812):0.1086388414):0.9542251762):0.01195296762):0.282735395,((t1004:0.8293355212,t3605:0.2742171576):0.9109663779,((t206:0.9336392879,t4102:0.6185394912):0.4976757427,t1974:0.5364129208):0.75979211):0.3271802349):0.5149520426):0.8091221796,t2070:0.4599797614):0.6744872606):0.4556475009,(((t1916:0.6711884656,(t4850:0.4626499987,t1204:0.3298903366):0.7507011695):0.2849657605,t4334:0.5787722857):0.6108045524,t3698:0.07360934606):0.452089644):0.9232895148):0.5710479196):0.2088616579):0.1036200167):0.9835169467,(((((t858:0.06411242601,t3241:0.6167466976):0.1597068114,(t1541:0.4900703458,(((t2272:0.7899209,t4424:0.8265837955):0.1608338694,(t2253:0.08435524232,t1808:0.7128797399):0.9365540836):0.1442598519,(t3958:0.2045844104,((t4739:0.8380573962,t4678:0.4767366378):0.176096939,t4956:0.07749349973):0.8375570385):0.1329541006):0.8871714333):0.8702931136):0.3858317055,(((t1109:0.540027864,t3677:0.3816335269):0.8096927011,(t3396:0.803992528,t4016:0.7597528619):0.5524927927):0.0880539692,(((((t3268:0.1683249963,(t1825:0.7506772834,t4076:0.6392439331):0.9240729709):0.63906717,(t1669:0.4184154717,t1789:0.2789682061):0.8277164558):0.001375993947,(t1773:0.935941552,(t4992:0.1547551188,t3682:0.9978090229):0.7883346223):0.1852043264):0.5674477248,(t3944:0.8792232096,t3071:0.5456949391):0.4710590157):0.04378668033,((t2718:0.4613593251,(t4946:0.323215971,t4035:0.4105415372):0.3110346287):0.2111645374,(t924:0.04812495364,t887:0.8861145906):0.5511388378):0.3112093078):0.6461817189):0.2764341964):0.6193111232,(((((t1082:0.3545431283,(t112:0.9897957388,t533:0.6385162552):0.5577652529):0.3026472079,(t1485:0.9488416535,(t629:0.1344688442,t1199:0.7352579175):0.4240722817):0.7483503884):0.7632226036,(((((t3801:0.2241399826,t1901:0.2831259898):0.9354316208,t3723:0.3100351428):0.1441025338,(t259:0.7228130172,t2674:0.5040441325):0.5963819926):0.3485045191,(((t322:0.1461221024,t1554:0.3647905516):0.9185589897,(((t3803:0.6605719575,t992:0.4497298943):0.6843213777,t3640:0.65091625):0.6794951626,(((t3746:0.07412167033,t2128:0.2570761496):0.7678958459,t1746:0.9027049204):0.613769741,t2274:0.8372515563):0.7715896757):0.1552456077):0.1046929746,(t235:0.5276032682,((t1246:0.7057658478,t1452:0.4426586833):0.5321583336,(t1464:0.845162042,t2511:0.6405652175):0.462449142):0.03457865026):0.4133716801):0.8167003952):0.0843595576,(t4912:0.5801591789,(t4142:0.8180290542,t3480:0.8435524576):0.8996644802):0.5256826757):0.7885311178):0.9829331683,(((((((t1448:0.4981373355,(t1725:0.484600334,t1320:0.2279035149):0.1896492571):0.2846998344,((t3672:0.1743044206,t1153:0.5483347054):0.5445810712,(((t4309:0.3829213898,t3618:0.488213955):0.870641639,t852:0.4823688797):0.2478069675,t4263:0.8091821475):0.6262365412):0.7395905533):0.7988226681,((((t2737:0.3169172311,t1496:0.5678035892):0.7635582353,t1886:0.07464399491):0.8886354889,t1468:0.3775549468):0.995909086,((t1853:0.1006418986,t3826:0.2115952426):0.3963795118,((t4399:0.5926499446,(t2242:0.6836288641,(t774:0.9055709962,t4365:0.09117947379):0.2154549742):0.002541460097):0.6309567771,(((t3924:0.09210896259,t967:0.8954006704):0.9270673953,t3003:0.3667957832):0.6712272374,(t4288:0.6774673092,(t297:0.6477955675,t2971:0.9330043944):0.5347735363):0.4678568786):0.2744014449):0.5362804357):0.1896059418):0.9824639419):0.2910927257,(t4838:0.5823269445,(t1829:0.09542567679,t1494:0.9373426249):0.8674853742):0.4305811899):0.9418918968,((((t2408:0.8752732803,t2822:0.2914511007):0.4904692224,((t2339:0.706327063,(t1621:0.04946335545,((t4656:0.4187640275,t507:0.5830970602):0.730714587,t278:0.7531718691):0.3621049337):0.4529870385):0.4615745521,((t3270:0.7319646161,(t4318:0.5327777814,t1658:0.2603637984):0.5142575572):0.4667672347,(((t3243:0.01085781376,t2685:0.3068525738):0.5892158072,t2494:0.1190970417):0.9669882841,t960:0.01768629532):0.7562129954):0.2649265544):0.04158595973):0.5476686584,((((((t4885:0.6036086108,((t163:0.05341434479,t3678:0.3839624913):0.8366795059,t4646:0.7459367712):0.9285725402):0.2042326485,(t800:0.1949443868,t1529:0.637674721):0.7487082528):0.9519212856,(t4298:0.940584904,(t3112:0.3193803735,t421:0.9848027977):0.2660647412):0.3867670198):0.8447402944,(((t4611:0.2840887408,t697:0.5459590608):0.1260031154,(((t817:0.168413579,t1005:0.04619488539):0.5606830614,(t2461:0.5133481955,t719:0.5807964071):0.2418366654):0.3707122523,(((t4015:0.2809359985,t3232:0.9727459485):0.4669691774,t1691:0.1167021689):0.1773620148,((t3960:0.7819925714,(t1270:0.8251114942,t1657:0.8424768401):0.9290333618):0.8754190612,(t2385:0.726571664,((t136:0.9743728598,(t1037:0.9002091601,t1693:0.1854209953):0.1567238006):0.2268195439,t3695:0.168481156):0.9425161555):0.7382732432):0.1674307517):0.9388361066):0.9433112922):0.07135570259,(((((t698:0.09476080979,t3612:0.5202466031):0.3063060495,t4308:0.7797084867):0.4945989328,(t3656:0.6734930673,(t1504:0.9186055744,t522:0.4559930016):0.733440347):0.6986998825):0.6505058671,t2188:0.5812294136):0.2950857924,t1511:0.1919472597):0.9852753803):0.9860354227):0.8168691553,((((t2346:0.8252629736,t4999:0.7314522774):0.932968115,(t1906:0.6576297546,t1801:0.3423777614):0.3427604758):0.9780303473,(((((t1872:0.4545243341,t3976:0.407866965):0.8170823257,t2046:0.6592412295):0.605292287,t2616:0.8327596944):0.3932966609,(t865:0.8652938486,(((t1100:0.4206713825,t1367:0.7636579236):0.8426675398,(t3316:0.1339187864,t724:0.9703182587):0.06509670545):0.7531080809,((t352:0.6948613795,t1059:0.3993287652):0.111172345,(t26:0.37449057,((t4619:0.6784601388,t3400:0.6304920518):0.777129425,(t821:0.06690427661,t2921:0.3082080726):0.1424122038):0.9643930101):0.009625436272):0.6136510167):0.2339827868):0.1596097811):0.2322916591,((t3073:0.7737157289,(t3171:0.993981977,t3459:0.4856949998):0.995985952):0.3542228341,((t1348:0.9890636127,t659:0.4483737007):0.1820985358,(((t1715:0.814271773,t282:0.2210659839):0.6863403202,(t2811:0.3204415792,t1881:0.001117120963):0.5955292173):0.2411594845,(t181:0.3470769129,(t3530:0.1804235061,(t268:0.6668869823,t1493:0.8584465648):0.4320758998):0.9514574884):0.6850090863):0.1220752629):0.03290794301):0.9641948475):0.29137076):0.6051525972,((t4498:0.5664488261,t184:0.07591620577):0.06855167286,((t2809:0.06503812806,t989:0.1750362192):0.6576453603,t3851:0.6571312097):0.6393228138):0.0267996327):0.3974038174):0.4680626276,(((((t4337:0.8608171451,(t536:0.6687977931,t841:0.4141964123):0.8912750718):0.7674989346,t2659:0.7293822088):0.8846026615,t1803:0.2793462821):0.8416803593,(((t3272:0.7777515596,t2080:0.7941319598):0.02631839667,t3214:0.5942487724):0.5160064469,(t4013:0.4476619852,t1807:0.2488512525):0.729859788):0.9063261263):0.2504933844,((((((t2695:0.6049396207,t2760:0.2017731613):0.6053460324,t942:0.2537483457):0.1888931596,t2422:0.8775926859):0.6970652381,(((t3573:0.932032407,t3795:0.3131393332):0.6880443385,t4037:0.9025869428):0.7243335943,(t4817:0.6195163475,t461:0.03366296994):0.1930365085):0.4740079097):0.9647499376,(t2355:0.3847696183,t157:0.1842326594):0.1302771189):0.7576121869,t2038:0.6688211027):0.4653305307):0.1350538032):0.7489073463):0.7532517379,t3963:0.5097164386):0.6461684881):0.2250971582,(((t1941:0.9922585045,t1781:0.8412561384):0.5502698335,(((t2396:0.4528068693,t867:0.7831091029):0.1561398564,(((((t731:0.676843561,t2501:0.4730992818):0.957833352,t993:0.8961748842):0.08554596337,t970:0.005601833574):0.4731912559,(((t3126:0.8893651974,t3673:0.2956292017):0.711548175,t784:0.02041305206):0.1260020446,t870:0.04179394385):0.8649424119):0.7283189215,(t2814:0.7600165009,t3604:0.8496893423):0.2718608612):0.6049078095):0.4245714592,((t3842:0.6401632235,(t864:0.6753688771,t1446:0.6583346301):0.8580043251):0.2429852469,((t1436:0.5414854873,(t4713:0.4202378707,(t4067:0.5034863369,t2629:0.8074325623):0.5543840749):0.9349443933):0.02426961903,((t3522:0.9907591171,t704:0.4116551063):0.6100854974,(((((t2236:0.9821746647,t2856:0.5976271427):0.4596348302,t474:0.7539694754):0.7961019615,(t1670:0.844424784,t1914:0.4355164059):0.4819602959):0.1552385129,(t1217:0.09929499333,(t4313:0.2940169228,t4195:0.4057241958):0.3305140531):0.1464614107):0.9916022678,((t1402:0.7184557286,((t2830:0.8429572626,t4338:0.8593967271):0.1068637548,(t2198:0.6595575139,t3900:0.5030493571):0.3447983882):0.5798652845):0.3265116732,(((t2459:0.7050274473,(t807:0.02875242569,t1116:0.4342548172):0.5984207008):0.4081060758,t3743:0.4978628801):0.8867255775,((t3:0.4002111179,t2529:0.7841629293):0.9892691621,t4563:0.2127349749):0.7635733793):0.9813009251):0.4457525031):0.7050674709):0.01797982049):0.1475887189):0.4840322225):0.952921378):0.6017595802,((((t371:0.2917505798,((t2445:0.5741511493,(t4053:0.5087266094,t4199:0.9622469801):0.8829812761):0.3686938186,((t1017:0.1899550389,(t1675:0.1847110714,t3321:0.1000435338):0.8136302102):0.05761236348,(t1187:0.9236007365,(t1297:0.643090605,t2478:0.4971563988):0.01230134862):0.3631388545):0.9496942644):0.5330755601):0.3273670757,(t4147:0.7434184744,(t4209:0.006608442636,t3547:0.3864654989):0.5036692368):0.7791287575):0.5481267711,(t4655:0.2857632346,(((((t4503:0.491377546,(t4446:0.6829332202,t4068:0.8872857341):0.101997524):0.4042064734,(t3063:0.3124079481,t456:0.7027420413):0.2071606708):0.213365573,t3648:0.4300736585):0.4465680362,t4234:0.6146167591):0.7072238619,(t738:0.9487351964,(t708:0.03216825891,(t3058:0.02041206858,t939:0.8475363867):0.3119547649):0.9873044582):0.7471097999):0.5353735681):0.7399762019):0.3138862944,(((t3577:0.4774673784,t1079:0.8010679984):0.7665190892,(((((t956:0.8557815284,(t1536:0.01053266483,t4947:0.2072744241):0.1867820737):0.9786464318,t2366:0.2062665191):0.9265410195,(t833:0.7665833295,t2913:0.6900583762):0.6524839972):0.7507895418,(t2266:0.9176654874,t810:0.2819611947):0.6825832187):0.5155524861,t1850:0.01931662858):0.5042956853):0.5280335112,(((t3954:0.03723145206,t4026:0.2671931747):0.621120679,t3663:0.9844085721):0.3474534666,(t2610:0.9074339687,t3252:0.1153040451):0.6771469368):0.2899941714):0.7533737174):0.9421177851):0.214113242):0.4109271232,((((((t3738:0.4897033526,t4061:0.5179669438):0.5344402904,t1478:0.733168399):0.81612151,t4765:0.7530770674):0.4004520692,t3334:0.7293370827):0.1946834333,t517:0.9455822895):0.509756065,(((t3715:0.411435524,t301:0.2152178786):0.9039262128,(t379:0.5406909853,t4158:0.5671118884):0.2476651608):0.2585469089,((((t2019:0.1609158688,t4672:0.9744254511):0.5824158809,((t3818:0.7242818398,t279:0.6975175098):0.06363215949,t1705:0.577070741):0.6929105257):0.8594568998,(((t2223:0.1445876227,(((t1167:0.3875568907,t407:0.4401276307):0.02390055754,(t3156:0.8969328478,(((t4515:0.6290904046,t246:0.8875405404):0.9459705176,((t1821:0.1054594966,(t4860:0.8164967902,t2463:0.3606067856):0.4598261092):0.8025708992,t3732:0.4307876192):0.9053041022):0.9632835123,((t2758:0.6740479255,t4363:0.9797021735):0.369993452,t251:0.1658985827):0.8324960438):0.923767681):0.7272378544):0.7545190814,((((t2451:0.8340366441,t4225:0.4726320782):0.2216190945,((t744:0.06684687221,(t985:0.8464032472,(t3764:0.5871600183,t1034:0.200227855):0.1730330288):0.3641210548):0.03705981933,t4690:0.1805881504):0.001105027972):0.9149940698,t3664:0.1224591259):0.047328969,(((t1871:0.4440514371,t1382:0.5654030533):0.2993641077,t398:0.733409388):0.5125037802,t2999:0.7959689612):0.7150404458):0.4553740651):0.1601598465):0.2918342238,(t1411:0.6928606085,t2449:0.9594068481):0.9958454215):0.3353172482,((((t3679:0.1677752833,t1376:0.04711970338):0.6202824602,t1756:0.4104493335):0.8587978831,(t3170:0.3176038689,t2864:0.957967134):0.02636153693):0.6062387773,(t4610:0.3108427317,(t1362:0.9030945443,t1068:0.5604104076):0.8757567252):0.7632295315):0.04064894933):0.2232549784):0.628997812,(((t3768:0.1566263596,(t3607:0.4206105024,(t3703:0.3426952295,t123:0.3864531559):0.2454953203):0.9778551538):0.1311787055,(((t4818:0.05779135739,t2380:0.8055878512):0.7620784689,t1679:0.6460430915):0.6275952449,(((t530:0.7700445983,t253:0.4565344534):0.9460171629,t4632:0.761197855):0.4460401647,t4680:0.3564638582):0.5242779846):0.4834057768):0.8769592731,(t761:0.3393602732,t4731:0.9288442452):0.8950331125):0.9656578552):0.5733375484):0.8089238452):0.371760186):0.4571701568):0.5839023788,((((((t1405:0.3410794481,t4093:0.8740192691):0.4900524048,t4447:0.5486000609):0.4527245886,t894:0.9202072534):0.5744145846,t4544:0.4655123905):0.2877174865,(((((t3714:0.09355640807,t687:0.9913308492):0.1579199715,t4246:0.05942391441):0.1237697485,(((t3978:0.6360942677,(t4241:0.5603810607,t1976:0.3958477092):0.8848404167):0.1439993505,t365:0.06173945032):0.04628659366,(t1652:0.6518087266,t2217:0.6493082752):0.5947727524):0.1308768615):0.8713698816,t3525:0.3978211852):0.1043918738,(((((t2237:0.4411648072,t4785:0.7887479833):0.6400109541,(((t579:0.1236431387,t4608:0.3669953754):0.8551357121,(t2883:0.2008376981,(t654:0.2831728391,t549:0.04207564401):0.1124596118):0.2878458707):0.1101977334,(t1522:0.7643014907,((((t3637:0.7181239906,t2839:0.5545086074):0.1233856338,t2377:0.5307070061):0.2645297893,(t2112:0.1060882825,t674:0.1496263011):0.4675709056):0.9756635111,t1398:0.7076241844):0.7406989874):0.5116205458):0.4633492469):0.4129221223,((((t1263:0.4424055975,((t804:0.7907508074,t4250:0.3605149454):0.5134488565,(t1473:0.4992189119,t618:0.2603028438):0.1634404908):0.7843309953):0.3743015395,((t2320:0.3843308464,t3465:0.9278697739):0.8083463043,t4440:0.628224422):0.05299037904):0.9401315134,(t3583:0.3739482611,(t1076:0.3160902497,((t1215:0.7531720635,t1735:0.2548443023):0.7895536548,(t543:0.6813795976,t4959:0.212971952):0.7440906379):0.4517899104):0.502027449):0.2298337934):0.533468059,((t4006:0.4790110339,t2209:0.6109415914):0.6524037013,((t2347:0.9133991173,(t1822:0.9322780124,t2870:0.3376011704):0.1167531998):0.7109583481,((((t2415:0.2207468068,t2330:0.7700544712):0.3702927576,t3153:0.7907847287):0.9436856124,t1864:0.1997117186):0.9736551445,(t3916:0.4504204586,t4894:0.2011600961):0.04128022632):0.2135370239):0.8806718746):0.8986412424):0.5699410723):0.000889308285,(((t558:0.3986893112,t3550:0.5465479738):0.9157213969,t2801:0.6400023517):0.1303906906,((t3017:0.4600611243,(t925:0.07483185246,(t726:0.08860425605,t712:0.6382322453):0.9106433736):0.6903121625):0.1526340102,((t2769:0.07937857555,t2484:0.00636099698):0.327379134,t3840:0.06049633096):0.1150006331):0.1386192669):0.4229549451):0.14104647,((((t2797:0.6915456241,t3845:0.2853532957):0.002746100072,((t4299:0.04495831905,t2547:0.832870461):0.6874656754,(t1714:0.8282494994,t3336:0.6876233588):0.5157194266):0.7740377728):0.4490776972,((t2162:0.3726100142,t4163:0.5217220357):0.8599568645,t4798:0.4332366856):0.003032601671):0.599205577,((t3793:0.2167206712,t1602:0.4828743548):0.6336604476,t4942:0.129051452):0.8173094194):0.6487480053):0.426485128):0.9773376414):0.2332423734,(((((t3675:0.2571347454,((t648:0.5368512599,t4614:0.1861010774):0.6708834157,((((t1019:0.3640183373,(t4222:0.5973960282,t3504:0.7826897602):0.9706250103):0.03938376624,t4913:0.3402702992):0.1179172271,(t3062:0.2907878899,t14:0.04600134864):0.5087479879):0.5838079664,t4339:0.1632072914):0.2161066621):0.8087973462):0.9553436746,(t2568:0.7699490909,((t211:0.6279546837,t2371:0.7174582616):0.9557109701,t2180:0.7659825566):0.5124245761):0.8390805058):0.9175898388,((((t685:0.3090886951,t3386:0.1632468766):0.6776911383,(t1431:0.8405884341,t2966:0.1350890822):0.3237425582):0.6846183382,((t1350:0.8948231146,((((((t3346:0.4652242505,t3233:0.8187016025):0.6190449626,(t2927:0.1817500934,(t4766:0.5061134007,t1243:0.2611983921):0.874855252):0.191382831):0.240372947,(t2127:0.4083701479,t2561:0.9648618139):0.3547373989):0.1857547446,(t2465:0.3640749089,t4242:0.8528760753):0.1369490041):0.1869288546,(t3474:0.255303846,((t1520:0.3506630973,(t658:0.1827699845,t4455:0.6013996941):0.003884866601):0.2177621098,t3014:0.4282708431):0.4948792197):0.5679433024):0.7563491242,((((t3285:0.5464606709,t3654:0.8065130122):0.485662417,t4369:0.4019155286):0.421482132,t2944:0.852736376):0.4287808735,(((t2938:0.7252988992,t2226:0.5725544426):0.2285876956,t2565:0.941062941):0.09651523549,((((t5:0.5622136258,t2550:0.7672292592):0.9598785795,t1619:0.9202110937):0.6087566302,t2033:0.1718263414):0.3696553046,(t736:0.5931533442,t2964:0.1665791923):0.08873515227):0.7938032232):0.604756119):0.3674918958):0.9211683061):0.977715356,((((t4321:0.5483830238,(t60:0.8771601054,((t3086:0.4951712782,t2713:0.7673781179):0.9599792035,(t3483:0.559349718,(t2397:0.6046585266,t3755:0.1617642222):0.3125891963):0.3613814288):0.6659193118):0.2526101363):0.2362044044,t87:0.8992475935):0.6564086722,((t3093:0.5009844925,((t179:0.7124912015,t199:0.6642799305):0.7274399356,((t4025:0.662153438,t788:0.3679777402):0.4186581359,t2487:0.9260152183):0.01705316757):0.6735027055):0.06042522332,((t3984:0.578337745,(t2993:0.5466957504,((t2351:0.3863696747,t922:0.003211963223):0.1377389745,t3134:0.0002677245066):0.1861927486):0.2443795658):0.2822314708,(((t2022:0.9661625989,t4965:0.3808704342):0.410354868,t1503:0.9690009682):0.04416323476,((t115:0.6886158257,t623:0.1603065783):0.1316250502,t3613:0.9137728077):0.2649233781):0.4119740911):0.1699066835):0.9302723659):0.09755894146,((((t820:0.41237715,t4577:0.3932309777):0.0177700331,(t3863:0.3919039837,(t1843:0.8328757617,t502:0.9966381101):0.9595946865):0.8455722586):0.7603968547,t2270:0.1011411694):0.2674119461,(((t3251:0.75886738,t4456:0.02969338908):0.1968087065,(t1113:0.1841469354,t2077:0.9948621711):0.1941053602):0.9457943118,(t4224:0.9676091748,t4400:0.7844602531):0.6271663648):0.8098717353):0.3605214506):0.3990343572):0.72140503):0.7477272335,(t3816:0.237037628,(t3189:0.3180443179,(t3844:0.9766364673,(t1663:0.5793221609,t2612:0.1566872641):0.2382052164):0.8207444798):0.7014923515):0.000950576039):0.6527773989):0.08068528376,((t1913:0.2927209353,(((t564:0.9854166233,t3166:0.724259912):0.06748726103,(t3964:0.9388971915,t2878:0.9828156806):0.4245496443):0.8142656272,(((t4570:0.669150352,t4475:0.1197499835):0.04868988553,t2146:0.05920276046):0.8770124083,((t304:0.8131135686,t3264:0.3294975276):0.02364071808,((t3830:0.9030749612,t2390:0.4097846355):0.1162557362,(t4207:0.4562103222,t2588:0.3052081454):0.7898879626):0.3730835761):0.5011584221):0.3482263223):0.2859137726):0.7367207818,(((((t3275:0.2965475882,t3938:0.6318004408):0.6899144626,t1075:0.9548938728):0.4058938681,(((t2401:0.6032664403,(t91:0.1916772807,t1633:0.2797593242):0.8171005715):0.9557044648,t4886:0.1509225138):0.7267728702,(t667:0.69198466,((t2939:0.204459033,t3796:0.5214365677):0.7705864243,(t3147:0.6041777888,t2026:0.5567409468):0.1945171847):0.5293050292):0.5256341663):0.2590727985):0.3291487223,(((t3717:0.3102658635,t2212:0.6570851081):0.1496878567,((t3075:0.570383966,(t3509:0.596302384,t2200:0.7389818386):0.07671168726):0.07556008059,t570:0.7031274596):0.9816620047):0.7960608648,(((t1182:0.9967376189,t1626:0.4292174319):0.5938372426,(t3797:0.2202536878,(t2650:0.9118863153,t458:0.7497696455):0.7708439888):0.2235526047):0.1152040223,(t3753:0.1393688803,(t4582:0.6319757288,t4034:0.1301449435):0.4675617015):0.1175752387):0.2792746995):0.1754937284):0.8086005675,t1569:0.9257724534):0.9076594345):0.6875401447):0.5104553818,(((((((((t2563:0.1363624658,t3765:0.9202692467):0.9257132113,(t2520:0.6759444729,t1298:0.5676707628):0.8963690957):0.3723585384,t1573:0.31263947):0.9224747794,t3731:0.903335416):0.9975980555,t889:0.8668577496):0.232557948,(((((t1546:0.2543979045,t2409:0.1255876366):0.836668466,(t3180:0.9936766224,t19:0.8435047844):0.08403993864):0.4758515987,t2540:0.870227223):0.4236848291,t4777:0.7962760825):0.03359326138,(((t2098:0.3824362066,t1631:0.7811347474):0.687681691,(t4484:0.9383989652,t2930:0.1308226418):0.3607058111):0.09610536019,((t3206:0.04067969532,(t863:0.6647462756,(t1751:0.7216929472,(t4560:0.6345807051,(t4581:0.5007709875,t1001:0.6118313314):0.7747018184):0.4160925555):0.435028417):0.1332621083):0.1909463892,t2352:0.2718817648):0.7102355091):0.8143547003):0.36480806):0.7283255935,t2960:0.3068709415):0.4948668045,((((((t2753:0.764312085,t716:0.218243862):0.9250942331,t1228:0.5936011197):0.4565174433,t3488:0.4740776198):0.5358699113,(t1550:0.8656544944,t1072:0.05063128681):0.1541706251):0.51893936,((t183:0.6190196164,t1381:0.07202614751):0.5134890084,t4217:0.4622565708):0.4610308979):0.2344676564,t540:0.6314234834):0.8761790851):0.7457153781,((t4724:0.777753806,t4653:0.9002032038):0.3986418915,t418:0.9348111798):0.8059940161):0.5312227998):0.6387782258):0.8071524436):0.3774198121):0.5995206884,(((((((((t2072:0.3388171913,(t170:0.5656538592,t3967:0.4063566003):0.5896465038):0.9441501922,(t627:0.4855824807,((t1189:0.5794946223,t3197:0.1915091269):0.2378133114,t3481:0.5996205928):0.7407016151):0.01266718842):0.2284986938,((t806:0.08717884682,(t1507:0.02452146541,t2341:0.720454541):0.7639606923):0.4820979536,((t3977:0.4992395511,t3030:0.2948540405):0.9467841212,t314:0.2200207585):0.16470029):0.1974597035):0.2590556734,(t4906:0.7817889815,t871:0.9511088107):0.6128140548):0.425111552,((t1977:0.7853290171,((t2194:0.3953421833,t4284:0.1290105192):0.4117005398,t4579:0.914105603):0.9291999149):0.2821669155,(t1326:0.1465432816,(((t4375:0.2596870905,t4541:0.2010122901):0.3766071773,t1269:0.3674016863):0.1208044465,t4204:0.3698100182):0.8646778397):0.4921671802):0.8154761821):0.5094137727,((t802:0.5341457941,t1937:0.7998197789):0.05112488824,t1492:0.6897424383):0.8094769791):0.8457810935,((t1057:0.4046775179,((t1923:0.6462825201,t4200:0.2072024788):0.3298604705,t488:0.6419402526):0.003014594782):0.7265125066,(((t3155:0.5750570076,t3676:0.9892197067):0.906002905,t4085:0.9781129526):0.996173508,t4069:0.06360293226):0.3592883812):0.5619509818):0.6953451682,(((((t1415:0.09918350447,(t2442:0.155588863,t4412:0.3040755172):0.492990284):0.8465760103,t978:0.3687013045):0.02367241774,((t855:0.4608079048,t2912:0.09758407623):0.3701218856,((t3694:0.07802248141,t3138:0.8096938841):0.4384925135,t3606:0.6476927546):0.9487582464):0.4470769409):0.7942296606,(t1123:0.6678380442,t1625:0.1613316855):0.8397362826):0.3529371219,(t305:0.1103538726,((((t428:0.4417167683,((t4772:0.8882771546,t4091:0.4981870821):0.4756792276,t4229:0.38463756):0.864977079):0.8223084074,(t3363:0.5329570151,t2744:0.4839793602):0.7691739288):0.3956305948,(t15:0.4813148049,((t609:0.3889157842,t1603:0.4138532944):0.1289779663,t2136:0.6060533351):0.0904724244):0.03975326265):0.1583935879,t3523:0.08873104909):0.9958280958):0.7265755415):0.2345846083):0.5260387631,(t1624:0.6208119127,(t3032:0.3652262872,t4186:0.4689125395):0.9585901925):0.6533458913):0.8967711527):0.1973156768):0.5217929904):0.1052631885):0.2312479161):0.1278742545,(((t2806:0.7516691501,(t1505:0.5894373418,t4170:0.8753572488):0.5608480333):0.3014607499,(t3070:0.6524684269,t3124:0.2174219687):0.9468783992):0.8326292518,t490:0.4388881766):0.9946793513); diff --git a/R/unifrac_cpp/R_interface/rapi_test.R b/src/R_interface/rapi_test.R similarity index 97% rename from R/unifrac_cpp/R_interface/rapi_test.R rename to src/R_interface/rapi_test.R index 2b6e56b5a..086ddd1e3 100644 --- a/R/unifrac_cpp/R_interface/rapi_test.R +++ b/src/R_interface/rapi_test.R @@ -4,7 +4,7 @@ library(miaSim) library(ape) library(picante) -source = "R/unifrac_cpp/su_R.cpp" +source = "src/faith_R.cpp" sourceCpp(source) data(GlobalPatterns, package = "mia") diff --git a/R/unifrac_cpp/su_R.cpp b/src/faith_R.cpp similarity index 100% rename from R/unifrac_cpp/su_R.cpp rename to src/faith_R.cpp diff --git a/R/unifrac_cpp/propstack.cpp b/src/propstack.cpp similarity index 100% rename from R/unifrac_cpp/propstack.cpp rename to src/propstack.cpp diff --git a/R/unifrac_cpp/propstack.hpp b/src/propstack.hpp similarity index 100% rename from R/unifrac_cpp/propstack.hpp rename to src/propstack.hpp diff --git a/R/unifrac_cpp/tree.cpp b/src/tree.cpp similarity index 100% rename from R/unifrac_cpp/tree.cpp rename to src/tree.cpp diff --git a/R/unifrac_cpp/tree.hpp b/src/tree.hpp similarity index 100% rename from R/unifrac_cpp/tree.hpp rename to src/tree.hpp diff --git a/R/unifrac_cpp/tse.cpp b/src/tse.cpp similarity index 100% rename from R/unifrac_cpp/tse.cpp rename to src/tse.cpp diff --git a/R/unifrac_cpp/tse.hpp b/src/tse.hpp similarity index 100% rename from R/unifrac_cpp/tse.hpp rename to src/tse.hpp From 916560fc9fcab99dfd0c3edbbc2ec269b7235c16 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 17 Feb 2025 07:15:21 +0200 Subject: [PATCH 08/48] Integrate Rcpp and C++ code into the package Some datasets still produce divergent values. Likely a bug in my implementation. --- DESCRIPTION | 5 +- NAMESPACE | 2 + R/RcppExports.R | 7 ++ R/estimateDiversity.R | 21 +++- src/.gitignore | 12 ++ src/R_interface/rapi_test.R | 218 +++++++++++++++++++++++++++++++----- src/RcppExports.cpp | 34 ++++++ src/{tse.cpp => assay.cpp} | 36 +++--- src/{tse.hpp => assay.hpp} | 14 +-- src/faith_R.cpp | 14 ++- src/propstack.cpp | 12 +- src/propstack.hpp | 6 +- src/tree.cpp | 16 +-- src/tree.hpp | 6 +- 14 files changed, 314 insertions(+), 89 deletions(-) create mode 100644 R/RcppExports.R create mode 100644 src/.gitignore create mode 100644 src/RcppExports.cpp rename src/{tse.cpp => assay.cpp} (64%) rename src/{tse.hpp => assay.hpp} (88%) diff --git a/DESCRIPTION b/DESCRIPTION index f51a5ae96..cb60ef483 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -86,7 +86,8 @@ Imports: tibble, tidyr, utils, - vegan + vegan, + Rcpp Suggests: ade4, BiocStyle, @@ -107,6 +108,8 @@ Suggests: topicdoc, topicmodels, yaml +LinkingTo: + Rcpp URL: https://github.com/microbiome/mia BugReports: https://github.com/microbiome/mia/issues Roxygen: list(markdown = TRUE) diff --git a/NAMESPACE b/NAMESPACE index 2808d913e..4c2d6c1c7 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -349,6 +349,7 @@ importFrom(MultiAssayExperiment,ExperimentList) importFrom(MultiAssayExperiment,MultiAssayExperiment) importFrom(MultiAssayExperiment,experiments) importFrom(MultiAssayExperiment,sampleMap) +importFrom(Rcpp,sourceCpp) importFrom(S4Vectors,"metadata<-") importFrom(S4Vectors,DataFrame) importFrom(S4Vectors,SimpleList) @@ -467,3 +468,4 @@ importFrom(vegan,permutest) importFrom(vegan,rrarefy) importFrom(vegan,scores) importFrom(vegan,vegdist) +useDynLib(mia) diff --git a/R/RcppExports.R b/R/RcppExports.R new file mode 100644 index 000000000..f73465e76 --- /dev/null +++ b/R/RcppExports.R @@ -0,0 +1,7 @@ +# Generated by using Rcpp::compileAttributes() -> do not edit by hand +# Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 + +faith_cpp <- function(assay, rowTree) { + .Call('_mia_faith_cpp', PACKAGE = 'mia', assay, rowTree) +} + diff --git a/R/estimateDiversity.R b/R/estimateDiversity.R index f18ceadde..c9ed76572 100644 --- a/R/estimateDiversity.R +++ b/R/estimateDiversity.R @@ -115,11 +115,25 @@ vegan::fisher.alpha(t(mat)) } -.calc_faith <- function(mat, tree, only.tips = FALSE, ...){ +# These tags are required to enable the use of Rcpp in the package +#' @useDynLib mia +#' @importFrom Rcpp sourceCpp +NULL + +.calc_faith <- function(mat, tree, only.tips = FALSE, fast_faith = TRUE, ...){ # Input check if( !.is_a_bool(only.tips) ){ stop("'only.tips' must be TRUE or FALSE.", call. = FALSE) } + if( !.is_a_bool(fast_faith) ){ + stop("'fast_faith' must be TRUE or FALSE.", call. = FALSE) + } + # If using fast algorithm, check that the tree is rooted + if(fast_faith && !is.rooted(tree) ){ + stop("The fast C++ algorithm currently only works on rooted trees. ", + "Use fast_faith = FALSE for unrooted trees.", + call. = FALSE) + } # # Remove internal nodes if specified if( only.tips ){ @@ -128,6 +142,11 @@ # To ensure that the function works with NA also, convert NAs to 0. # Zero means that the taxon is not present --> same as NA (no information) mat[ is.na(mat) ] <- 0 + + # Use fast algorithm if requested + if( fast_faith ){ + return(faith_cpp(mat, tree)) + } # Gets vector where number represent nth sample samples <- seq_len(ncol(mat)) diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 000000000..f7bb13d44 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1,12 @@ +api.o +api_s.o +biom.o +biom_s.o +propstack.o +tree.o +tree_s.o +tse.o +unifrac.o +unifrac_internal.o +unifrac_internal_s.o +unifrac_s.o diff --git a/src/R_interface/rapi_test.R b/src/R_interface/rapi_test.R index 086ddd1e3..80c393f67 100644 --- a/src/R_interface/rapi_test.R +++ b/src/R_interface/rapi_test.R @@ -7,11 +7,14 @@ library(picante) source = "src/faith_R.cpp" sourceCpp(source) -data(GlobalPatterns, package = "mia") -data(esophagus, package = "mia") -data(HintikkaXOData, package = "mia") -data(Tengeler2020, package = "mia") # This dataset produces divergent values for some reason - Presumably something to do with the tree being unrooted -tse <- GlobalPatterns +data(GlobalPatterns, package = "mia") # Matches flawlessly, no difference between picantes +data(esophagus, package = "mia") # Matches flawlessly, no difference between picantes +data(Tengeler2020, package = "mia") # Wrong values, probably due to the tree being unrooted - can be left for the old code? Reroot tree? +tse <- Tengeler2020 + +tse <- microbiomeDataSets::artificialgut() # This fails every comparison - probably not the zero-length edges, since they work fine in the others +tse <- microbiomeDataSets::baboongut() # fails every comparison +tse <- microbiomeDataSets::SprockettTHData() # But this one matches picante with include.root=TRUE! So whether or not to include root is ONE issue, at least. rowTree(tse) <- ape::reorder.phylo(rowTree(tse), "cladewise") ts1 <- rowTree(tse) @@ -24,42 +27,199 @@ newick <- readChar(fname, file.info(fname)$size) y <- rowTree_to_bp(ts1) x <- newick_to_bp(newick) -faith <- faith_cpp(tse) -x <- estimateDiversity(tse, index="faith") -faith2 <- colData(x)$faith -faith - faith2 -#Checks -#This functions assumes that the tree representation is in cladewise order - Ensure this with ape's reorder.phylo() function -#Ensure that the tree is non-empty, etc -#Ensure that the tree doesn't get modified at any point -#Which assays are normally used for the calculations? +x <- estimateFaith(tse, index="faith", fast_faith=TRUE) +faith <- colData(x)$faith +x2 <- estimateFaith(tse, index="faith", fast_faith=FALSE, only.tips = TRUE) +faith2 <- colData(x2)$faith +x3 <- picante::pd(t(assay(tse)), rowTree(tse), include.root = TRUE) +faith3 <- as.vector(x3[[1]]) +x4 <- picante::pd(t(assay(tse)), rowTree(tse), include.root = FALSE) +faith4 <- as.vector(x4[[1]]) +faith - faith2 +faith - faith3 +faith - faith4 +faith2 - faith3 +faith3 - faith4 +sum(abs(faith-faith2) > 0.00000001) +sum(abs(faith-faith3) > 0.00000001) +sum(abs(faith-faith4) > 0.00000001) +ape::write.tree(rowTree(tse), "//utuhome.utu.fi/jealpa/downloads/agut.tre") +write.biom() -tse2 <- estimateDiversity(tse_hubbell, index = "faith") -colData(tse2)$faith -faith_pd(tse_hubbell) +#Checks +#This functions assumes that the tree representation is in cladewise order - Ensure this with ape's reorder.phylo() function +#Ensure that the tree is non-empty, etc +#Ensure that the tree doesn't get modified at any point +#Which assays are normally used for the calculations? - handled in the r functions -t <- ape::rtree(12706, rooted = F, tip.label = rownames(tse2)) -tse2 <- tse[[1]] -rowTree(tse2) <- t -tse2 <- estimateDiversity(tse2) -faith2 <- colData(tse2)$faith -faith <- faith_pd(tse2, T) -faith - faith2 +# The algorithm only calculates include.root = true values - These can differ significantly from the opposite case! +# Possibly correctable in the c++ code? -data(phylocom, package = "picante") -x <- phylocom$sample y <- assay(tse) -t1 <- phylocom$phylo t2 <- rowTree(tse) -picante::pd(x, t1) -picante::pd(t(y), t2, include.root=F) +f4 <- picante::pd(t(y), t2)[1] +faith - f4 rowTree(tse) <- ape::reorder.phylo(rowTree(tse), "cladewise") + + + +picante::pd(t(ass), tree, include.root = FALSE)[1] + + +samples <- 10 +obs <- 200 +tree <- ape::rtree(obs) +obsnames <- tree$tip.label +samplenames <- paste0("s", 1:samples) + +#v <- rbinom(samples*obs, 1, 0.2) * rgeom(samples*obs, 0.05) +v <- rgeom(samples*obs, 0.05) + +ass <- matrix(v, nrow=obs, ncol = samples) +colnames(ass) <- samplenames +rownames(ass) <- obsnames + +testtree <- TreeSummarizedExperiment(assays=SimpleList(counts=ass)) +rownames(testtree) <- obsnames +colnames(testtree) <- samplenames +rowTree(testtree) <- tree + +z <- estimateFaith(testtree, index="faith", fast_faith=TRUE) +faith_r <- colData(z)$faith +z2 <- estimateFaith(testtree, index="faith", fast_faith=FALSE) +faith_r2 <- colData(z2)$faith +z3 <- picante::pd(t(assay(testtree)), rowTree(testtree), include.root = TRUE) +faith_r3 <- as.vector(z3[[1]]) +z4 <- picante::pd(t(assay(testtree)), rowTree(testtree), include.root = FALSE) +faith_r4 <- as.vector(z4[[1]]) + + +faith_r - faith_r2 +faith_r - faith_r3 +faith_r - faith_r4 + +faith_r2 - faith_r3 +faith_r2 - faith_r4 +faith_r3 - faith_r4 + +# So far, every randomly generated sample matches picante with include.root=TRUE +# in maybe 90% of cases the match the other methods, as well + +# good samples for examining +#partiallyworks <- testtree # Matches picante with include.root=TRUE +#oldcodefails<- testtree # Seems like the old code fails with single-species samples in some cases? + #Specifically when ape::drop.tip tries to reduce a two-edge tree to a single edge... + #Otherwise matches picante with include.root=TRUE + + +edges <- c(1,2,3,4,5,7,8,10,11,12,14,19,20,21,23,24) +sum(tree$edge.length[edges]) + +# The current version of the code doesn't include a way to choose whether to include root or not +# Manually calculating the sum of edges for this tree produces a result that matches fastfaith! +# What is the old version doing differently? +# This seems to happen when the tree is simplified at the end of the process - collapse.singles eliminates the root node +# faith + +.prune_tree <- function(treent, nodes){ + # Get those tips that can not be found from provided nodes + remove_tips <- treent$tip.label[!treent$tip.label %in% nodes] + # As long as there are tips to be dropped, run the loop + while( length(remove_tips) > 0 ){ + # Drop tips that cannot be found. Drop only one layer at the time. Some + # dataset might have taxa that are not in tip layer but they are in + # higher rank. If we delete more than one layer at the time, we might + # loose the node for those taxa. --> The result of pruning is a tree + # whose all tips can be found provided nodes i.e., rows of TreeSE. Some + # taxa might be higher rank meaning that all rows might not be in tips + # even after pruning; these rows have still child-nodes that represent + # other rows. + # Suppress warning: drop all tips of the tree: returning NULL + suppressWarnings( + treent <- ape::drop.tip( + treent, remove_tips, + trim.internal = FALSE, + collapse.singles = FALSE) + ) + # If all tips were dropped, the result is NULL --> stop loop + if( is.null(treent) ){ + warning("Pruning resulted to empty tree.", call. = FALSE) + break + } + # Again, get those tips of updated tree that cannot be found from + # provided nodes + remove_tips <- treent$tip.label[!treent$tip.label %in% nodes] + } + # Simplify the tree structure. Remove nodes that have only single + # descendant. + if( !is.null(treent) && length(treent$tip.label) > 1 && ape::has.singles(treent) ){ + treent <- ape::collapse.singles(treent) + } + return(treent) +} +treent <- tree +nodes <- present[[7]] +.prune_tree(treent, nodes ) + + + # Gets vector where number represent nth sample + ss <- seq_len(ncol(ass)) + + # Repeats taxa as many times there are samples, i.e. get all the + # taxa that are analyzed in each sample. + taxa <- rep(rownames(ass), length(ss)) + + # Gets those taxa that are present/absent in each sample. + # Gets one big list that combines + # taxa from all the samples. + present_combined <- taxa[ ass[, ss] > 0 ] + + # Gets how many taxa there are in each sample. + # After that, determines indices of samples' first taxa with cumsum. + split_present <- as.vector(cumsum(colSums(ass > 0))) + + # Determines which taxa belongs to which sample by first determining + # the splitting points, + # and after that giving every taxa number which tells their sample. + split_present <- as.factor(cumsum((seq_along(present_combined)-1) %in% + split_present)) + + # Assigns taxa to right samples based on their number that they got from + # previous step, and deletes unnecessary names. + present <- unname(split(present_combined, split_present)) + + # If there were samples without any taxa present/absent, the length of the + # list is not the number of samples since these empty samples are missing. + # Add empty samples as NULL. + names(present) <- names(which(colSums2(ass) > 0)) + present[names(which(colSums2(ass) == 0))] <- list(NULL) + present <- present[colnames(ass)] + + # Assign NA to all samples + faiths <- rep(NA,length(ss)) + + # If there are no taxa present, then faith is 0 + ind <- lengths(present) == 0 + faiths[ind] <- 0 + + # If there are taxa present + ind <- lengths(present) > 0 + # Loop through taxa that were found from each sample + faiths_for_taxa_present <- lapply(present[ind], function(x){ + # Trim the tree + temp <- .prune_tree(tree, x) + # Sum up all the lengths of edges + temp <- sum(temp$edge.length) + return(temp) + }) + faiths_for_taxa_present <- unlist(faiths_for_taxa_present) + faiths[ind] <- faiths_for_taxa_present \ No newline at end of file diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp new file mode 100644 index 000000000..61c7092bb --- /dev/null +++ b/src/RcppExports.cpp @@ -0,0 +1,34 @@ +// Generated by using Rcpp::compileAttributes() -> do not edit by hand +// Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 + +#include + +using namespace Rcpp; + +#ifdef RCPP_USE_GLOBAL_ROSTREAM +Rcpp::Rostream& Rcpp::Rcout = Rcpp::Rcpp_cout_get(); +Rcpp::Rostream& Rcpp::Rcerr = Rcpp::Rcpp_cerr_get(); +#endif + +// faith_cpp +Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix& assay, const Rcpp::List& rowTree); +RcppExport SEXP _mia_faith_cpp(SEXP assaySEXP, SEXP rowTreeSEXP) { +BEGIN_RCPP + Rcpp::RObject rcpp_result_gen; + Rcpp::RNGScope rcpp_rngScope_gen; + Rcpp::traits::input_parameter< const Rcpp::NumericMatrix& >::type assay(assaySEXP); + Rcpp::traits::input_parameter< const Rcpp::List& >::type rowTree(rowTreeSEXP); + rcpp_result_gen = Rcpp::wrap(faith_cpp(assay, rowTree)); + return rcpp_result_gen; +END_RCPP +} + +static const R_CallMethodDef CallEntries[] = { + {"_mia_faith_cpp", (DL_FUNC) &_mia_faith_cpp, 2}, + {NULL, NULL, 0} +}; + +RcppExport void R_init_mia(DllInfo *dll) { + R_registerRoutines(dll, NULL, CallEntries, NULL, NULL); + R_useDynamicSymbols(dll, FALSE); +} diff --git a/src/tse.cpp b/src/assay.cpp similarity index 64% rename from src/tse.cpp rename to src/assay.cpp index 8dd4dd91c..b18dc5fc2 100644 --- a/src/tse.cpp +++ b/src/assay.cpp @@ -12,29 +12,23 @@ #include #include -#include "tse.hpp" +#include "assay.hpp" #include using namespace su; -tse::tse(const Rcpp::S4 & treeSE) { +Assay::Assay(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree) { + table = assay; + sample_ids = std::vector(); obs_ids = std::vector(); - Rcpp::S4 colData = treeSE.slot("colData"); - Rcpp::StringVector rownames = colData.slot("rownames"); - sample_ids = Rcpp::as>(rownames); + Rcpp::StringVector colnames = Rcpp::colnames(table); + sample_ids = Rcpp::as>(colnames); - Rcpp::List rowTree = treeSE.slot("rowTree"); - Rcpp::List phylo = rowTree["phylo"]; - Rcpp::StringVector tip_label = phylo["tip.label"]; + Rcpp::StringVector tip_label = rowTree["tip.label"]; obs_ids = Rcpp::as>(tip_label); - - Rcpp::S4 assays = treeSE.slot("assays"); - Rcpp::S4 data = assays.slot("data"); - Rcpp::List listData = data.slot("listData"); - assay = Rcpp::as(listData["counts"]); n_samples = sample_ids.size(); n_obs = obs_ids.size(); @@ -45,16 +39,16 @@ tse::tse(const Rcpp::S4 & treeSE) { create_id_index(obs_ids, obs_id_index); create_id_index(sample_ids, sample_id_index); - + sample_counts = get_sample_counts(); - + } -tse::~tse() { +Assay::~Assay() { } -void tse::create_id_index(std::vector &ids, +void Assay::create_id_index(std::vector &ids, std::unordered_map &map) { uint32_t count = 0; map.reserve(ids.size()); @@ -64,22 +58,22 @@ void tse::create_id_index(std::vector &ids, } -std::vector tse::get_obs_data(const std::string &id) const { +std::vector Assay::get_obs_data(const std::string &id) const { std::vector out = std::vector(); uint32_t idx = obs_id_index.at(id); for(unsigned int i = 0; i < n_samples; i++) { - out.push_back(assay(idx, i)); + out.push_back(table(idx, i)); } return out; } -std::vector tse::get_sample_counts() { +std::vector Assay::get_sample_counts() { std::vector sample_counts = std::vector(); for(unsigned int i = 0; i < n_samples; i++) { unsigned int sum = 0; for(unsigned int j = 0; j < n_obs; j++){ - sum += assay(j, i); + sum += table(j, i); } sample_counts.push_back(sum); } diff --git a/src/tse.hpp b/src/assay.hpp similarity index 88% rename from src/tse.hpp rename to src/assay.hpp index e3076a3d8..8dd8c03e2 100644 --- a/src/tse.hpp +++ b/src/assay.hpp @@ -7,8 +7,8 @@ * See LICENSE file for more details */ -#ifndef __FAITH_TSE_H -#define __FAITH_TSE_H 1 +#ifndef __FAITH_ASSAY_H +#define __FAITH_ASSAY_H 1 #include #include @@ -16,7 +16,7 @@ #include namespace su { - class tse { + class Assay { public: // cache the IDs contained within the table std::vector sample_ids; @@ -30,13 +30,13 @@ namespace su { * * @param treeSE An R TreeSummarizedExperiment object */ - tse(const Rcpp::S4 & treeSE); + Assay(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree); /* default destructor * * Temporary arrays are freed */ - ~tse(); + ~Assay(); /* get a dense vector of observation data * @@ -48,7 +48,7 @@ namespace su { std::vector get_obs_data(const std::string &id) const; private: - Rcpp::NumericMatrix assay; // Access to the raw sample counts in R's memory + Rcpp::NumericMatrix table; // Access to the raw sample counts in R's memory std::vector get_sample_counts(); @@ -69,5 +69,5 @@ namespace su { }; } -#endif /* __FAITH_TSE_H */ +#endif /* __FAITH_ASSAY_H */ diff --git a/src/faith_R.cpp b/src/faith_R.cpp index 9337057a9..9def651de 100644 --- a/src/faith_R.cpp +++ b/src/faith_R.cpp @@ -3,7 +3,7 @@ #include -#include "tse.hpp" +#include "assay.hpp" #include "tree.hpp" #include "propstack.hpp" @@ -25,10 +25,10 @@ * */ // [[Rcpp::export]] -Rcpp::NumericVector faith_cpp(const Rcpp::S4 & treeSE){ +Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree){ - su::BPTree tree = su::BPTree(treeSE); - su::tse table = su::tse(treeSE); + su::BPTree tree = su::BPTree(rowTree); + su::Assay table = su::Assay(assay, rowTree); su::PropStack propstack(table.n_samples); @@ -46,10 +46,12 @@ Rcpp::NumericVector faith_cpp(const Rcpp::S4 & treeSE){ length = tree.lengths[node]; // get node proportions and set intermediate scores - node_proportions = set_proportions(tree, node, table, propstack); + node_proportions = set_proportions(tree, node, table, propstack); // this would probably be the most likely culprit for something going wrong for (unsigned int sample = 0; sample < table.n_samples; sample++){ // calculate contribution of node to score + // Is it possible to somehow set the proportions to 0 if we're dealing with the root in a include.root=FALSE scenario? + //if(sample == 0) std::cout << k << " " << node_proportions[sample] << " " << (node_proportions[sample] > 0) << " " << length << "\n"; results[sample] += (node_proportions[sample] > 0) * length; } } @@ -61,4 +63,4 @@ Rcpp::NumericVector faith_cpp(const Rcpp::S4 & treeSE){ } return faith; -} +} \ No newline at end of file diff --git a/src/propstack.cpp b/src/propstack.cpp index 7b748bad0..bbd197b6b 100644 --- a/src/propstack.cpp +++ b/src/propstack.cpp @@ -8,7 +8,7 @@ */ #include "tree.hpp" -#include "tse.hpp" +#include "assay.hpp" #include "propstack.hpp" #include @@ -24,8 +24,7 @@ using namespace su; PropStack::PropStack(uint32_t vecsize) -: prop_stack() -, prop_map() +: prop_map() , defaultsize(vecsize) { prop_map.reserve(1000); @@ -53,7 +52,7 @@ void PropStack::update(uint32_t node, std::vector vec) { std::vector su::set_proportions(const BPTree &tree, uint32_t node, - const tse &table, + const Assay &table, PropStack &ps, bool normalize) { @@ -62,7 +61,6 @@ std::vector su::set_proportions(const BPTree &tree, std::string leaf = tree.names[node]; props = table.get_obs_data(leaf); // Here we basically just need the row for the specified node if (normalize) { -//#pragma omp parallel for schedule(static) for(unsigned int i = 0; i < table.n_samples; i++) { props[i] /= table.sample_counts[i]; } @@ -71,8 +69,6 @@ std::vector su::set_proportions(const BPTree &tree, unsigned int current = tree.leftchild(node); unsigned int right = tree.rightchild(node); -//#pragma omp parallel for schedule(static) - for(unsigned int i = 0; i < table.n_samples; i++){ props.push_back(0); } @@ -80,7 +76,7 @@ std::vector su::set_proportions(const BPTree &tree, while(current <= right && current != 0) { std::vector vec = ps.get(current); // pull from prop map ps.clear(current); // remove from prop map, place back on stack -//#pragma omp parallel for schedule(static) + for(unsigned int i = 0; i < table.n_samples; i++) props[i] = props[i] + vec[i]; diff --git a/src/propstack.hpp b/src/propstack.hpp index 42fc46e4a..60480b1ba 100644 --- a/src/propstack.hpp +++ b/src/propstack.hpp @@ -14,13 +14,13 @@ #include #include -#include "tse.hpp" +#include "tree.hpp" +#include "assay.hpp" namespace su { class PropStack { private: - std::stack> prop_stack; std::unordered_map> prop_map; uint32_t defaultsize; public: @@ -32,7 +32,7 @@ namespace su { }; std::vector set_proportions(const BPTree &tree, uint32_t node, - const tse &table, + const Assay &table, PropStack &ps, bool normalize = true); } diff --git a/src/tree.cpp b/src/tree.cpp index b020428ea..185738884 100644 --- a/src/tree.cpp +++ b/src/tree.cpp @@ -27,7 +27,7 @@ BPTree::BPTree(std::vector input_structure, std::vector input_leng index_and_cache(); } -BPTree::BPTree(const Rcpp::S4 & treeSE) { +BPTree::BPTree(const Rcpp::List & rowTree) { //Initialize vectors openclose = std::vector(); @@ -41,7 +41,6 @@ BPTree::BPTree(const Rcpp::S4 & treeSE) { //Load the tree structure structure = std::vector(); structure.reserve(500000); // a fair sized tree... avoid reallocs, and its not _that_ much waste if this is wrong - const Rcpp::List & rowTree = treeSE.slot("rowTree"); rowTree_to_bp(rowTree); //Also sets the size of nparens //Resize vectors @@ -284,8 +283,7 @@ int32_t BPTree::bwd(uint32_t i, int d) const { // Need to check whether tree being rooted or not affects construction // If rooted, root is by definition ntips+1 // If unrooted, root is chosen arbitrarily? -void BPTree::rowTree_to_bp(const Rcpp::List & rowTree) { - Rcpp::List phylo = rowTree["phylo"]; +void BPTree::rowTree_to_bp(const Rcpp::List & phylo) { Rcpp::NumericMatrix edge = phylo["edge"]; Rcpp::StringVector tips = phylo["tip.label"]; @@ -293,12 +291,12 @@ void BPTree::rowTree_to_bp(const Rcpp::List & rowTree) { std::stack nodes; // Keeps track of the branch's internal nodes - int currentNode = 0; - int nextNode = 0; + unsigned int currentNode = 0; + unsigned int nextNode = 0; // Goal: Insert true when a branch starts, a false when it closes, and a true-false for each tip. - for (unsigned int i = 0; i < edge.nrow(); i++){ + for (int i = 0; i < edge.nrow(); i++){ currentNode = edge(i, 0); nextNode = edge(i, 1); @@ -357,8 +355,7 @@ void BPTree::structure_to_openclose() { //edge.length has (nodes + tips) elements - leaves at the start, nodes at the end //tip.label has (tips) elements //root.edge and node.labels are optional, giving the length of the root and the internal node (including root) labels, respectively -void BPTree::rowTree_to_metadata(const Rcpp::List & rowTree) { - Rcpp::List phylo = rowTree["phylo"]; +void BPTree::rowTree_to_metadata(const Rcpp::List & phylo) { Rcpp::NumericVector edgelength = phylo["edge.length"]; Rcpp::NumericMatrix edges = phylo["edge"]; Rcpp::StringVector tips = phylo["tip.label"]; @@ -387,7 +384,6 @@ void BPTree::rowTree_to_metadata(const Rcpp::List & rowTree) { unsigned int tip_idx = 0; // tip indices run from 0 to ntips-1 unsigned int node_idx = 0; // node indices run from ntips to ntips + nnodes - 1 - unsigned int edge_idx = 0; // Used to store the index of the edge for picking lengths; for(unsigned int i = 0; i < structure.size(); i++) { if(structure[i]){ diff --git a/src/tree.hpp b/src/tree.hpp index a470604b1..48678236b 100644 --- a/src/tree.hpp +++ b/src/tree.hpp @@ -40,7 +40,7 @@ namespace su { * * @param treeSE An R treeSE object */ - BPTree(const Rcpp::S4 & treeSE); + BPTree(const Rcpp::List & rowTree); ~BPTree(); @@ -124,8 +124,8 @@ namespace su { std::vector excess; void index_and_cache(); // construct the select caches - void rowTree_to_bp(const Rcpp::List & rowTree); // convert ape tree structure to boolean structure - void rowTree_to_metadata(const Rcpp::List & rowTree); // assign attributes + void rowTree_to_bp(const Rcpp::List & phylo); // convert ape tree structure to boolean structure + void rowTree_to_metadata(const Rcpp::List & phylo); // assign attributes void newick_to_metadata(std::string newick); // convert newick to attributes void structure_to_openclose(); // set the cache mapping between parentheses pairs void set_node_metadata(unsigned int open_idx, std::string label, double length); // set attributes for a node From 15d8fa2f9cab9a50dbada3e81e4a338eb8631c60 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Fri, 21 Feb 2025 11:52:09 +0200 Subject: [PATCH 09/48] Fix the bug that was causing divergent results Bringing the assay into C++ was using rowTree tip labels for the observation ids, causing nonsense results when they were in different order from the actual rownames. Also added a check for cladewise tree ordering. --- R/RcppExports.R | 4 +++ R/estimateDiversity.R | 12 +++------ src/R_interface/rapi_test.R | 51 ++++++++++++++++++------------------- src/RcppExports.cpp | 12 +++++++++ src/assay.cpp | 8 +++--- src/assay.hpp | 2 +- src/faith_R.cpp | 33 +++++++++++++++++++----- src/propstack.cpp | 9 +++++-- 8 files changed, 84 insertions(+), 47 deletions(-) diff --git a/R/RcppExports.R b/R/RcppExports.R index f73465e76..92cb60774 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -5,3 +5,7 @@ faith_cpp <- function(assay, rowTree) { .Call('_mia_faith_cpp', PACKAGE = 'mia', assay, rowTree) } +sumrowt <- function(rowTree) { + .Call('_mia_sumrowt', PACKAGE = 'mia', rowTree) +} + diff --git a/R/estimateDiversity.R b/R/estimateDiversity.R index c9ed76572..31ca81c92 100644 --- a/R/estimateDiversity.R +++ b/R/estimateDiversity.R @@ -120,6 +120,7 @@ #' @importFrom Rcpp sourceCpp NULL +#' @importFrom ape reorder.phylo .calc_faith <- function(mat, tree, only.tips = FALSE, fast_faith = TRUE, ...){ # Input check if( !.is_a_bool(only.tips) ){ @@ -128,13 +129,6 @@ NULL if( !.is_a_bool(fast_faith) ){ stop("'fast_faith' must be TRUE or FALSE.", call. = FALSE) } - # If using fast algorithm, check that the tree is rooted - if(fast_faith && !is.rooted(tree) ){ - stop("The fast C++ algorithm currently only works on rooted trees. ", - "Use fast_faith = FALSE for unrooted trees.", - call. = FALSE) - } - # # Remove internal nodes if specified if( only.tips ){ mat <- mat[ rownames(mat) %in% tree$tip.label, ] @@ -145,7 +139,9 @@ NULL # Use fast algorithm if requested if( fast_faith ){ - return(faith_cpp(mat, tree)) + # The tree must be in cladewise order for the algorithm to work correctly + temp <- reorder.phylo(tree, "cladewise") + return(faith_cpp(mat, temp)) } # Gets vector where number represent nth sample diff --git a/src/R_interface/rapi_test.R b/src/R_interface/rapi_test.R index 80c393f67..04d6168d2 100644 --- a/src/R_interface/rapi_test.R +++ b/src/R_interface/rapi_test.R @@ -9,29 +9,26 @@ sourceCpp(source) data(GlobalPatterns, package = "mia") # Matches flawlessly, no difference between picantes data(esophagus, package = "mia") # Matches flawlessly, no difference between picantes -data(Tengeler2020, package = "mia") # Wrong values, probably due to the tree being unrooted - can be left for the old code? Reroot tree? -tse <- Tengeler2020 - -tse <- microbiomeDataSets::artificialgut() # This fails every comparison - probably not the zero-length edges, since they work fine in the others -tse <- microbiomeDataSets::baboongut() # fails every comparison -tse <- microbiomeDataSets::SprockettTHData() # But this one matches picante with include.root=TRUE! So whether or not to include root is ONE issue, at least. -rowTree(tse) <- ape::reorder.phylo(rowTree(tse), "cladewise") - -ts1 <- rowTree(tse) - -fname <- "R/unifrac_cpp/R_interface/tree.tre" -ape::write.tree(ts1, fname) - -newick <- readChar(fname, file.info(fname)$size) - -y <- rowTree_to_bp(ts1) -x <- newick_to_bp(newick) - - - -x <- estimateFaith(tse, index="faith", fast_faith=TRUE) +data(Tengeler2020, package = "mia") # Seems to work fine now, despite the tree being unrooted +tse <- GlobalPatterns + +tse <- microbiomeDataSets::artificialgut() # All good (minor differences) +tse <- microbiomeDataSets::baboongut() # All good (minor differences) +tse <- microbiomeDataSets::SprockettTHData() # Matches picante with include.root=TRUE +tse <- microbiomeDataSets::GrieneisenTSData() # All good (minor differences) +tse <- microbiomeDataSets::qa10934() # Matches picante with include.root=TRUE +tse <- microbiomeDataSets::SilvermanAGutData() # All good (minor differences) + +library(biomformat) +biom <- read_biom("/home/grunnar/downloads/con/finrisk.biom") +tree <- ape::read.tree("/home/grunnar/downloads/con/anonymized-finrisk-16S-BL_AGE.tre") +tse <- convertFromBIOM(biom) +rowTree(tse) <- tree +tse <- tse[1:100,1:100] + +x <- addAlpha(tse, index="faith", fast_faith=TRUE) faith <- colData(x)$faith -x2 <- estimateFaith(tse, index="faith", fast_faith=FALSE, only.tips = TRUE) +x2 <- addAlpha(tse, index="faith", fast_faith=FALSE) faith2 <- colData(x2)$faith x3 <- picante::pd(t(assay(tse)), rowTree(tse), include.root = TRUE) faith3 <- as.vector(x3[[1]]) @@ -49,6 +46,7 @@ sum(abs(faith-faith2) > 0.00000001) sum(abs(faith-faith3) > 0.00000001) sum(abs(faith-faith4) > 0.00000001) + ape::write.tree(rowTree(tse), "//utuhome.utu.fi/jealpa/downloads/agut.tre") write.biom() @@ -76,7 +74,7 @@ picante::pd(t(ass), tree, include.root = FALSE)[1] samples <- 10 -obs <- 200 +obs <- 2000 tree <- ape::rtree(obs) obsnames <- tree$tip.label samplenames <- paste0("s", 1:samples) @@ -93,9 +91,9 @@ rownames(testtree) <- obsnames colnames(testtree) <- samplenames rowTree(testtree) <- tree -z <- estimateFaith(testtree, index="faith", fast_faith=TRUE) +z <- addAlpha(testtree, index="faith", fast_faith=TRUE) faith_r <- colData(z)$faith -z2 <- estimateFaith(testtree, index="faith", fast_faith=FALSE) +z2 <- addAlpha(testtree, index="faith", fast_faith=FALSE) faith_r2 <- colData(z2)$faith z3 <- picante::pd(t(assay(testtree)), rowTree(testtree), include.root = TRUE) faith_r3 <- as.vector(z3[[1]]) @@ -222,4 +220,5 @@ nodes <- present[[7]] return(temp) }) faiths_for_taxa_present <- unlist(faiths_for_taxa_present) - faiths[ind] <- faiths_for_taxa_present \ No newline at end of file + faiths[ind] <- faiths_for_taxa_present + \ No newline at end of file diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index 61c7092bb..2c66ba80f 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -22,9 +22,21 @@ BEGIN_RCPP return rcpp_result_gen; END_RCPP } +// sumrowt +double sumrowt(const Rcpp::List& rowTree); +RcppExport SEXP _mia_sumrowt(SEXP rowTreeSEXP) { +BEGIN_RCPP + Rcpp::RObject rcpp_result_gen; + Rcpp::RNGScope rcpp_rngScope_gen; + Rcpp::traits::input_parameter< const Rcpp::List& >::type rowTree(rowTreeSEXP); + rcpp_result_gen = Rcpp::wrap(sumrowt(rowTree)); + return rcpp_result_gen; +END_RCPP +} static const R_CallMethodDef CallEntries[] = { {"_mia_faith_cpp", (DL_FUNC) &_mia_faith_cpp, 2}, + {"_mia_sumrowt", (DL_FUNC) &_mia_sumrowt, 1}, {NULL, NULL, 0} }; diff --git a/src/assay.cpp b/src/assay.cpp index b18dc5fc2..e6f1e365a 100644 --- a/src/assay.cpp +++ b/src/assay.cpp @@ -18,7 +18,7 @@ using namespace su; -Assay::Assay(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree) { +Assay::Assay(const Rcpp::NumericMatrix & assay) { table = assay; sample_ids = std::vector(); @@ -27,9 +27,9 @@ Assay::Assay(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree) { Rcpp::StringVector colnames = Rcpp::colnames(table); sample_ids = Rcpp::as>(colnames); - Rcpp::StringVector tip_label = rowTree["tip.label"]; - obs_ids = Rcpp::as>(tip_label); - + Rcpp::StringVector rownames = Rcpp::rownames(table); + obs_ids = Rcpp::as>(rownames); + n_samples = sample_ids.size(); n_obs = obs_ids.size(); diff --git a/src/assay.hpp b/src/assay.hpp index 8dd8c03e2..c0199e5e1 100644 --- a/src/assay.hpp +++ b/src/assay.hpp @@ -30,7 +30,7 @@ namespace su { * * @param treeSE An R TreeSummarizedExperiment object */ - Assay(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree); + Assay(const Rcpp::NumericMatrix & assay); /* default destructor * diff --git a/src/faith_R.cpp b/src/faith_R.cpp index 9def651de..8eb43d7b1 100644 --- a/src/faith_R.cpp +++ b/src/faith_R.cpp @@ -27,8 +27,12 @@ // [[Rcpp::export]] Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree){ + su::BPTree tree = su::BPTree(rowTree); - su::Assay table = su::Assay(assay, rowTree); + su::Assay table = su::Assay(assay); + + std::unordered_set to_keep(table.obs_ids.begin(),table.obs_ids.end()); + su::BPTree tree_sheared = tree.shear(to_keep).collapse(); su::PropStack propstack(table.n_samples); @@ -39,19 +43,19 @@ Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::Lis std::vector results = std::vector(table.n_samples, 0.0); // for node in postorderselect - for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { - node = tree.postorderselect(k); + const unsigned int max_k = (tree_sheared.nparens>1) ? ((tree_sheared.nparens / 2) - 1) : 0; + for(unsigned int k = 0; k < max_k; k++) { + node = tree_sheared.postorderselect(k); // get branch length - length = tree.lengths[node]; + length = tree_sheared.lengths[node]; // get node proportions and set intermediate scores - node_proportions = set_proportions(tree, node, table, propstack); // this would probably be the most likely culprit for something going wrong + node_proportions = set_proportions(tree_sheared, node, table, propstack, false); // this would probably be the most likely culprit for something going wrong for (unsigned int sample = 0; sample < table.n_samples; sample++){ // calculate contribution of node to score // Is it possible to somehow set the proportions to 0 if we're dealing with the root in a include.root=FALSE scenario? - //if(sample == 0) std::cout << k << " " << node_proportions[sample] << " " << (node_proportions[sample] > 0) << " " << length << "\n"; results[sample] += (node_proportions[sample] > 0) * length; } } @@ -63,4 +67,21 @@ Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::Lis } return faith; +} + +// [[Rcpp::export]] +double sumrowt(const Rcpp::List & rowTree){ + + Rcpp::NumericVector edgelength = rowTree["edge.length"]; + + const uint32_t n_edges = edgelength.size(); + + //Used to find the correct lengths for the nodes - Includes the root + double edge_v = 0.0; + + for(unsigned int i = 0; i < n_edges; i++){ + edge_v += edgelength[i]; + } + + return edge_v; } \ No newline at end of file diff --git a/src/propstack.cpp b/src/propstack.cpp index bbd197b6b..5881010dd 100644 --- a/src/propstack.cpp +++ b/src/propstack.cpp @@ -57,8 +57,13 @@ std::vector su::set_proportions(const BPTree &tree, bool normalize) { std::vector props = std::vector(); + + //propstack.clear(node); the current node is popped from propstack at every loop, replacing the vector with an empty one that then gets filled... + + if(tree.isleaf(node)) { std::string leaf = tree.names[node]; + props = table.get_obs_data(leaf); // Here we basically just need the row for the specified node if (normalize) { for(unsigned int i = 0; i < table.n_samples; i++) { @@ -72,6 +77,7 @@ std::vector su::set_proportions(const BPTree &tree, for(unsigned int i = 0; i < table.n_samples; i++){ props.push_back(0); } + ps.update(node, props); while(current <= right && current != 0) { std::vector vec = ps.get(current); // pull from prop map @@ -79,12 +85,11 @@ std::vector su::set_proportions(const BPTree &tree, for(unsigned int i = 0; i < table.n_samples; i++) props[i] = props[i] + vec[i]; + ps.update(node, props); current = tree.rightsibling(current); } - //std::cout << "n " << props[0] << " " << props[1] << " " << props[2] << "\n"; - } ps.update(node, props); return(props); From f62c853f162c9ed556b28599fedcc36317ffb03f Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Fri, 21 Feb 2025 11:56:41 +0200 Subject: [PATCH 10/48] Miscellaneous cleaning up --- NAMESPACE | 1 + R/RcppExports.R | 4 - R/estimateDiversity.R | 8 +- src/R_interface/rapi_test.R | 224 ----------------------------- src/RcppExports.cpp | 12 -- src/assay.hpp | 1 - src/faith_R.cpp | 23 +-- src/{propstack.cpp => propmap.cpp} | 25 ++-- src/{propstack.hpp => propmap.hpp} | 14 +- src/tree.cpp | 61 +++----- src/tree.hpp | 1 - 11 files changed, 41 insertions(+), 333 deletions(-) delete mode 100644 src/R_interface/rapi_test.R rename src/{propstack.cpp => propmap.cpp} (78%) rename src/{propstack.hpp => propmap.hpp} (77%) diff --git a/NAMESPACE b/NAMESPACE index 4c2d6c1c7..6d3389d19 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -392,6 +392,7 @@ importFrom(ape,has.singles) importFrom(ape,is.binary) importFrom(ape,is.rooted) importFrom(ape,read.tree) +importFrom(ape,reorder.phylo) importFrom(bluster,clusterRows) importFrom(decontam,isContaminant) importFrom(decontam,isNotContaminant) diff --git a/R/RcppExports.R b/R/RcppExports.R index 92cb60774..f73465e76 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -5,7 +5,3 @@ faith_cpp <- function(assay, rowTree) { .Call('_mia_faith_cpp', PACKAGE = 'mia', assay, rowTree) } -sumrowt <- function(rowTree) { - .Call('_mia_sumrowt', PACKAGE = 'mia', rowTree) -} - diff --git a/R/estimateDiversity.R b/R/estimateDiversity.R index 31ca81c92..4f501e3d8 100644 --- a/R/estimateDiversity.R +++ b/R/estimateDiversity.R @@ -121,13 +121,13 @@ NULL #' @importFrom ape reorder.phylo -.calc_faith <- function(mat, tree, only.tips = FALSE, fast_faith = TRUE, ...){ +.calc_faith <- function(mat, tree, only.tips = FALSE, fast.faith = TRUE, ...){ # Input check if( !.is_a_bool(only.tips) ){ stop("'only.tips' must be TRUE or FALSE.", call. = FALSE) } - if( !.is_a_bool(fast_faith) ){ - stop("'fast_faith' must be TRUE or FALSE.", call. = FALSE) + if( !.is_a_bool(fast.faith) ){ + stop("'fast.faith' must be TRUE or FALSE.", call. = FALSE) } # Remove internal nodes if specified if( only.tips ){ @@ -138,7 +138,7 @@ NULL mat[ is.na(mat) ] <- 0 # Use fast algorithm if requested - if( fast_faith ){ + if( fast.faith ){ # The tree must be in cladewise order for the algorithm to work correctly temp <- reorder.phylo(tree, "cladewise") return(faith_cpp(mat, temp)) diff --git a/src/R_interface/rapi_test.R b/src/R_interface/rapi_test.R deleted file mode 100644 index 04d6168d2..000000000 --- a/src/R_interface/rapi_test.R +++ /dev/null @@ -1,224 +0,0 @@ -library(Rcpp) -library(mia) -library(miaSim) -library(ape) -library(picante) - -source = "src/faith_R.cpp" -sourceCpp(source) - -data(GlobalPatterns, package = "mia") # Matches flawlessly, no difference between picantes -data(esophagus, package = "mia") # Matches flawlessly, no difference between picantes -data(Tengeler2020, package = "mia") # Seems to work fine now, despite the tree being unrooted -tse <- GlobalPatterns - -tse <- microbiomeDataSets::artificialgut() # All good (minor differences) -tse <- microbiomeDataSets::baboongut() # All good (minor differences) -tse <- microbiomeDataSets::SprockettTHData() # Matches picante with include.root=TRUE -tse <- microbiomeDataSets::GrieneisenTSData() # All good (minor differences) -tse <- microbiomeDataSets::qa10934() # Matches picante with include.root=TRUE -tse <- microbiomeDataSets::SilvermanAGutData() # All good (minor differences) - -library(biomformat) -biom <- read_biom("/home/grunnar/downloads/con/finrisk.biom") -tree <- ape::read.tree("/home/grunnar/downloads/con/anonymized-finrisk-16S-BL_AGE.tre") -tse <- convertFromBIOM(biom) -rowTree(tse) <- tree -tse <- tse[1:100,1:100] - -x <- addAlpha(tse, index="faith", fast_faith=TRUE) -faith <- colData(x)$faith -x2 <- addAlpha(tse, index="faith", fast_faith=FALSE) -faith2 <- colData(x2)$faith -x3 <- picante::pd(t(assay(tse)), rowTree(tse), include.root = TRUE) -faith3 <- as.vector(x3[[1]]) -x4 <- picante::pd(t(assay(tse)), rowTree(tse), include.root = FALSE) -faith4 <- as.vector(x4[[1]]) - -faith - faith2 -faith - faith3 -faith - faith4 - -faith2 - faith3 -faith3 - faith4 - -sum(abs(faith-faith2) > 0.00000001) -sum(abs(faith-faith3) > 0.00000001) -sum(abs(faith-faith4) > 0.00000001) - - -ape::write.tree(rowTree(tse), "//utuhome.utu.fi/jealpa/downloads/agut.tre") -write.biom() - - -#Checks -#This functions assumes that the tree representation is in cladewise order - Ensure this with ape's reorder.phylo() function -#Ensure that the tree is non-empty, etc -#Ensure that the tree doesn't get modified at any point -#Which assays are normally used for the calculations? - handled in the r functions - -# The algorithm only calculates include.root = true values - These can differ significantly from the opposite case! -# Possibly correctable in the c++ code? - -y <- assay(tse) -t2 <- rowTree(tse) -f4 <- picante::pd(t(y), t2)[1] - -faith - f4 - -rowTree(tse) <- ape::reorder.phylo(rowTree(tse), "cladewise") - - - -picante::pd(t(ass), tree, include.root = FALSE)[1] - - -samples <- 10 -obs <- 2000 -tree <- ape::rtree(obs) -obsnames <- tree$tip.label -samplenames <- paste0("s", 1:samples) - -#v <- rbinom(samples*obs, 1, 0.2) * rgeom(samples*obs, 0.05) -v <- rgeom(samples*obs, 0.05) - -ass <- matrix(v, nrow=obs, ncol = samples) -colnames(ass) <- samplenames -rownames(ass) <- obsnames - -testtree <- TreeSummarizedExperiment(assays=SimpleList(counts=ass)) -rownames(testtree) <- obsnames -colnames(testtree) <- samplenames -rowTree(testtree) <- tree - -z <- addAlpha(testtree, index="faith", fast_faith=TRUE) -faith_r <- colData(z)$faith -z2 <- addAlpha(testtree, index="faith", fast_faith=FALSE) -faith_r2 <- colData(z2)$faith -z3 <- picante::pd(t(assay(testtree)), rowTree(testtree), include.root = TRUE) -faith_r3 <- as.vector(z3[[1]]) -z4 <- picante::pd(t(assay(testtree)), rowTree(testtree), include.root = FALSE) -faith_r4 <- as.vector(z4[[1]]) - - -faith_r - faith_r2 -faith_r - faith_r3 -faith_r - faith_r4 - -faith_r2 - faith_r3 -faith_r2 - faith_r4 -faith_r3 - faith_r4 - -# So far, every randomly generated sample matches picante with include.root=TRUE -# in maybe 90% of cases the match the other methods, as well - -# good samples for examining -#partiallyworks <- testtree # Matches picante with include.root=TRUE -#oldcodefails<- testtree # Seems like the old code fails with single-species samples in some cases? - #Specifically when ape::drop.tip tries to reduce a two-edge tree to a single edge... - #Otherwise matches picante with include.root=TRUE - - -edges <- c(1,2,3,4,5,7,8,10,11,12,14,19,20,21,23,24) -sum(tree$edge.length[edges]) - -# The current version of the code doesn't include a way to choose whether to include root or not -# Manually calculating the sum of edges for this tree produces a result that matches fastfaith! -# What is the old version doing differently? -# This seems to happen when the tree is simplified at the end of the process - collapse.singles eliminates the root node -# faith - -.prune_tree <- function(treent, nodes){ - # Get those tips that can not be found from provided nodes - remove_tips <- treent$tip.label[!treent$tip.label %in% nodes] - # As long as there are tips to be dropped, run the loop - while( length(remove_tips) > 0 ){ - # Drop tips that cannot be found. Drop only one layer at the time. Some - # dataset might have taxa that are not in tip layer but they are in - # higher rank. If we delete more than one layer at the time, we might - # loose the node for those taxa. --> The result of pruning is a tree - # whose all tips can be found provided nodes i.e., rows of TreeSE. Some - # taxa might be higher rank meaning that all rows might not be in tips - # even after pruning; these rows have still child-nodes that represent - # other rows. - # Suppress warning: drop all tips of the tree: returning NULL - suppressWarnings( - treent <- ape::drop.tip( - treent, remove_tips, - trim.internal = FALSE, - collapse.singles = FALSE) - ) - # If all tips were dropped, the result is NULL --> stop loop - if( is.null(treent) ){ - warning("Pruning resulted to empty tree.", call. = FALSE) - break - } - # Again, get those tips of updated tree that cannot be found from - # provided nodes - remove_tips <- treent$tip.label[!treent$tip.label %in% nodes] - } - # Simplify the tree structure. Remove nodes that have only single - # descendant. - if( !is.null(treent) && length(treent$tip.label) > 1 && ape::has.singles(treent) ){ - treent <- ape::collapse.singles(treent) - } - return(treent) -} -treent <- tree -nodes <- present[[7]] -.prune_tree(treent, nodes ) - - - # Gets vector where number represent nth sample - ss <- seq_len(ncol(ass)) - - # Repeats taxa as many times there are samples, i.e. get all the - # taxa that are analyzed in each sample. - taxa <- rep(rownames(ass), length(ss)) - - # Gets those taxa that are present/absent in each sample. - # Gets one big list that combines - # taxa from all the samples. - present_combined <- taxa[ ass[, ss] > 0 ] - - # Gets how many taxa there are in each sample. - # After that, determines indices of samples' first taxa with cumsum. - split_present <- as.vector(cumsum(colSums(ass > 0))) - - # Determines which taxa belongs to which sample by first determining - # the splitting points, - # and after that giving every taxa number which tells their sample. - split_present <- as.factor(cumsum((seq_along(present_combined)-1) %in% - split_present)) - - # Assigns taxa to right samples based on their number that they got from - # previous step, and deletes unnecessary names. - present <- unname(split(present_combined, split_present)) - - # If there were samples without any taxa present/absent, the length of the - # list is not the number of samples since these empty samples are missing. - # Add empty samples as NULL. - names(present) <- names(which(colSums2(ass) > 0)) - present[names(which(colSums2(ass) == 0))] <- list(NULL) - present <- present[colnames(ass)] - - # Assign NA to all samples - faiths <- rep(NA,length(ss)) - - # If there are no taxa present, then faith is 0 - ind <- lengths(present) == 0 - faiths[ind] <- 0 - - # If there are taxa present - ind <- lengths(present) > 0 - # Loop through taxa that were found from each sample - faiths_for_taxa_present <- lapply(present[ind], function(x){ - # Trim the tree - temp <- .prune_tree(tree, x) - # Sum up all the lengths of edges - temp <- sum(temp$edge.length) - return(temp) - }) - faiths_for_taxa_present <- unlist(faiths_for_taxa_present) - faiths[ind] <- faiths_for_taxa_present - \ No newline at end of file diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index 2c66ba80f..61c7092bb 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -22,21 +22,9 @@ BEGIN_RCPP return rcpp_result_gen; END_RCPP } -// sumrowt -double sumrowt(const Rcpp::List& rowTree); -RcppExport SEXP _mia_sumrowt(SEXP rowTreeSEXP) { -BEGIN_RCPP - Rcpp::RObject rcpp_result_gen; - Rcpp::RNGScope rcpp_rngScope_gen; - Rcpp::traits::input_parameter< const Rcpp::List& >::type rowTree(rowTreeSEXP); - rcpp_result_gen = Rcpp::wrap(sumrowt(rowTree)); - return rcpp_result_gen; -END_RCPP -} static const R_CallMethodDef CallEntries[] = { {"_mia_faith_cpp", (DL_FUNC) &_mia_faith_cpp, 2}, - {"_mia_sumrowt", (DL_FUNC) &_mia_sumrowt, 1}, {NULL, NULL, 0} }; diff --git a/src/assay.hpp b/src/assay.hpp index c0199e5e1..6b19d619e 100644 --- a/src/assay.hpp +++ b/src/assay.hpp @@ -70,4 +70,3 @@ namespace su { } #endif /* __FAITH_ASSAY_H */ - diff --git a/src/faith_R.cpp b/src/faith_R.cpp index 8eb43d7b1..87f98c50a 100644 --- a/src/faith_R.cpp +++ b/src/faith_R.cpp @@ -5,7 +5,7 @@ #include "assay.hpp" #include "tree.hpp" -#include "propstack.hpp" +#include "propmap.hpp" /* Access the C++ implementation of the fast Faith's PD algorithm from R @@ -34,7 +34,7 @@ Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::Lis std::unordered_set to_keep(table.obs_ids.begin(),table.obs_ids.end()); su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - su::PropStack propstack(table.n_samples); + su::PropMap propmap(table.n_samples); uint32_t node; std::vector node_proportions; @@ -51,7 +51,7 @@ Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::Lis length = tree_sheared.lengths[node]; // get node proportions and set intermediate scores - node_proportions = set_proportions(tree_sheared, node, table, propstack, false); // this would probably be the most likely culprit for something going wrong + node_proportions = set_proportions(tree_sheared, node, table, propmap, false); // this would probably be the most likely culprit for something going wrong for (unsigned int sample = 0; sample < table.n_samples; sample++){ // calculate contribution of node to score @@ -68,20 +68,3 @@ Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::Lis return faith; } - -// [[Rcpp::export]] -double sumrowt(const Rcpp::List & rowTree){ - - Rcpp::NumericVector edgelength = rowTree["edge.length"]; - - const uint32_t n_edges = edgelength.size(); - - //Used to find the correct lengths for the nodes - Includes the root - double edge_v = 0.0; - - for(unsigned int i = 0; i < n_edges; i++){ - edge_v += edgelength[i]; - } - - return edge_v; -} \ No newline at end of file diff --git a/src/propstack.cpp b/src/propmap.cpp similarity index 78% rename from src/propstack.cpp rename to src/propmap.cpp index 5881010dd..9501a5efa 100644 --- a/src/propstack.cpp +++ b/src/propmap.cpp @@ -9,7 +9,7 @@ #include "tree.hpp" #include "assay.hpp" -#include "propstack.hpp" +#include "propmap.hpp" #include #include @@ -23,17 +23,17 @@ using namespace su; -PropStack::PropStack(uint32_t vecsize) +PropMap::PropMap(uint32_t vecsize) : prop_map() , defaultsize(vecsize) { prop_map.reserve(1000); } -PropStack::~PropStack() { +PropMap::~PropMap() { } -std::vector PropStack::get(uint32_t i) { +std::vector PropMap::get(uint32_t i) { if(prop_map.count(i) > 0){ return prop_map.at(i); } @@ -42,28 +42,23 @@ std::vector PropStack::get(uint32_t i) { } } -void PropStack::clear(uint32_t i) { +void PropMap::clear(uint32_t i) { prop_map[i] = std::vector(); } -void PropStack::update(uint32_t node, std::vector vec) { +void PropMap::update(uint32_t node, std::vector vec) { prop_map[node] = vec; } std::vector su::set_proportions(const BPTree &tree, uint32_t node, const Assay &table, - PropStack &ps, + PropMap &ps, bool normalize) { std::vector props = std::vector(); - - //propstack.clear(node); the current node is popped from propstack at every loop, replacing the vector with an empty one that then gets filled... - - if(tree.isleaf(node)) { std::string leaf = tree.names[node]; - props = table.get_obs_data(leaf); // Here we basically just need the row for the specified node if (normalize) { for(unsigned int i = 0; i < table.n_samples; i++) { @@ -77,7 +72,6 @@ std::vector su::set_proportions(const BPTree &tree, for(unsigned int i = 0; i < table.n_samples; i++){ props.push_back(0); } - ps.update(node, props); while(current <= right && current != 0) { std::vector vec = ps.get(current); // pull from prop map @@ -85,12 +79,13 @@ std::vector su::set_proportions(const BPTree &tree, for(unsigned int i = 0; i < table.n_samples; i++) props[i] = props[i] + vec[i]; - ps.update(node, props); current = tree.rightsibling(current); } + //std::cout << "n " << props[0] << " " << props[1] << " " << props[2] << "\n"; + } ps.update(node, props); return(props); -} \ No newline at end of file +} diff --git a/src/propstack.hpp b/src/propmap.hpp similarity index 77% rename from src/propstack.hpp rename to src/propmap.hpp index 60480b1ba..a2862412e 100644 --- a/src/propstack.hpp +++ b/src/propmap.hpp @@ -7,8 +7,8 @@ * See LICENSE file for more details */ -#ifndef __FAITH_PROPSTACK -#define __FAITH_PROPSTACK 1 +#ifndef __FAITH_PROPMAP +#define __FAITH_PROPMAP 1 #include #include @@ -19,13 +19,13 @@ namespace su { - class PropStack { + class PropMap { private: std::unordered_map> prop_map; uint32_t defaultsize; public: - PropStack(uint32_t vecsize); - virtual ~PropStack(); + PropMap(uint32_t vecsize); + virtual ~PropMap(); void clear(uint32_t i); void update(uint32_t i, std::vector vec); std::vector get(uint32_t i); @@ -33,8 +33,8 @@ namespace su { std::vector set_proportions(const BPTree &tree, uint32_t node, const Assay &table, - PropStack &ps, + PropMap &ps, bool normalize = true); } -#endif /* __FAITH_PROPSTACK */ +#endif /* __FAITH_PROPMAP */ diff --git a/src/tree.cpp b/src/tree.cpp index 185738884..64f109c5c 100644 --- a/src/tree.cpp +++ b/src/tree.cpp @@ -154,37 +154,6 @@ BPTree BPTree::collapse() { return this->mask(collapsemask, new_lengths); } - /* - mask = bit_array_create(self.B.size) - bit_array_set_bit(mask, self.root()) - bit_array_set_bit(mask, self.close(self.root())) - - new_lengths = self._lengths.copy() - new_lengths_ptr = new_lengths.data - - with nogil: - for i in range(n): - current = self.preorderselect(i) - - if self.isleaf(current): - bit_array_set_bit(mask, current) - bit_array_set_bit(mask, self.close(current)) - else: - first = self.fchild(current) - last = self.lchild(current) - - if first == last: - new_lengths_ptr[first] = new_lengths_ptr[first] + \ - new_lengths_ptr[current] - else: - bit_array_set_bit(mask, current) - bit_array_set_bit(mask, self.close(current)) - - new_bp = self._mask_from_self(mask, new_lengths) - bit_array_free(mask) - return new_bp -*/ - BPTree::~BPTree() { } @@ -278,18 +247,20 @@ int32_t BPTree::bwd(uint32_t i, int d) const { } // The algorithms that this class uses need the tree to be stored in a binary format -// In terms of the Newick format, an opening bracket corresponds to a TRUE, a closing bracket to a FALSE, and a tip to a TRUE FALSE -// This functions assumes that the tree representation is in cladewise order - Ensure this with ape's reorder.phylo() function -// Need to check whether tree being rooted or not affects construction -// If rooted, root is by definition ntips+1 -// If unrooted, root is chosen arbitrarily? +// In terms of the Newick format, an opening bracket corresponds to a TRUE, +// a closing bracket to a FALSE, and a tip to a TRUE FALSE +// This function assumes that the tree representation is in cladewise order - +// Ensure this with ape's reorder.phylo() function +// Seems to work whether or not the tree is rooted void BPTree::rowTree_to_bp(const Rcpp::List & phylo) { Rcpp::NumericMatrix edge = phylo["edge"]; Rcpp::StringVector tips = phylo["tip.label"]; - uint32_t ntips = tips.size(); // phylo tips are always numbered from 1 to number of tips; + // phylo tips are always numbered from 1 to number of tips; + uint32_t ntips = tips.size(); - std::stack nodes; // Keeps track of the branch's internal nodes + // Keeps track of the branch's internal nodes + std::stack nodes; unsigned int currentNode = 0; unsigned int nextNode = 0; @@ -310,7 +281,6 @@ void BPTree::rowTree_to_bp(const Rcpp::List & phylo) { if(nodes.size() == 0 || currentNode > nodes.top() ) { // We are either at the root, or entering a new node - // What if the tree is unrooted? nodes.push(currentNode); structure.push_back(true); @@ -350,11 +320,13 @@ void BPTree::structure_to_openclose() { } } -//Add metadata (lengths and names) to the tree representation -//I think we can just iterate through the structure, and whenever we hit a true decide if it's a leaf or not, and then add the corresponding label/length -//edge.length has (nodes + tips) elements - leaves at the start, nodes at the end -//tip.label has (tips) elements -//root.edge and node.labels are optional, giving the length of the root and the internal node (including root) labels, respectively +// Add metadata (lengths and names) to the tree representation +// Iterate through the structure, and whenever we hit a true decide if +// it's a leaf or not, and then add the corresponding label/length +// edge.length has (nodes + tips) elements - leaves at the start, nodes at the end +// tip.label has (tips) elements +// root.edge and node.labels are optional, giving the length of the root and +// the internal node (including root) labels, respectively void BPTree::rowTree_to_metadata(const Rcpp::List & phylo) { Rcpp::NumericVector edgelength = phylo["edge.length"]; Rcpp::NumericMatrix edges = phylo["edge"]; @@ -424,4 +396,3 @@ std::vector BPTree::get_structure() { std::vector BPTree::get_openclose() { return openclose; } - diff --git a/src/tree.hpp b/src/tree.hpp index 48678236b..331d47069 100644 --- a/src/tree.hpp +++ b/src/tree.hpp @@ -138,4 +138,3 @@ namespace su { } #endif /* __FAITH_TREE_H */ - From 2b4cc75e5b305113b43618e5ec1240013b0e2048 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Fri, 21 Feb 2025 11:57:45 +0200 Subject: [PATCH 11/48] Update documentation --- R/addAlpha.R | 5 +++++ man/addAlpha.Rd | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/R/addAlpha.R b/R/addAlpha.R index d67afa7d2..f9a4f5b6c 100644 --- a/R/addAlpha.R +++ b/R/addAlpha.R @@ -40,6 +40,11 @@ #' When \code{only.tips=TRUE}, those rows that are not tips of tree are #' removed. (Default: \code{FALSE}) #' +#' \item \code{fast.faith}: (Faith's index). \code{Logical scalar}. Specifies +#' whether to use a C++ implementation of the Stacked Faith algorithm for +#' calculating Faith's index. This can speed up calculation by several orders +#' of magnitude for large datasets. (Default: \code{TRUE}) +#' #' \item \code{threshold}: (Coverage and all evenness indices). #' \code{Numeric scalar}. #' From \code{0 to 1}, determines the threshold for coverage and evenness diff --git a/man/addAlpha.Rd b/man/addAlpha.Rd index 93b5b9185..5eebc49ab 100644 --- a/man/addAlpha.Rd +++ b/man/addAlpha.Rd @@ -52,6 +52,11 @@ whether to remove internal nodes when Faith's index is calculated. When \code{only.tips=TRUE}, those rows that are not tips of tree are removed. (Default: \code{FALSE}) +\item \code{fast.faith}: (Faith's index). \code{Logical scalar}. Specifies +whether to use a C++ implementation of the Stacked Faith algorithm for +calculating Faith's index. This can speed up calculation by several orders +of magnitude for large datasets. (Default: \code{TRUE}) + \item \code{threshold}: (Coverage and all evenness indices). \code{Numeric scalar}. From \code{0 to 1}, determines the threshold for coverage and evenness From 985a87314b3c1d65f1f88af8ad0fb425aeede3a8 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Fri, 21 Feb 2025 13:35:25 +0200 Subject: [PATCH 12/48] Add all object files to .gitignore --- src/.gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/.gitignore b/src/.gitignore index f7bb13d44..214d1b231 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -10,3 +10,8 @@ unifrac.o unifrac_internal.o unifrac_internal_s.o unifrac_s.o +RcppExports.o +assay.o +faith_R.o +propmap.o +mia.dll From a37163744229d8bd88becc127832757047a394ec Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Fri, 21 Feb 2025 16:24:14 +0200 Subject: [PATCH 13/48] Rename header files to get rid of warnings --- src/assay.cpp | 2 +- src/{assay.hpp => assay.h} | 0 src/faith_R.cpp | 6 +++--- src/propmap.cpp | 6 +++--- src/{propmap.hpp => propmap.h} | 4 ++-- src/tree.cpp | 2 +- src/{tree.hpp => tree.h} | 0 7 files changed, 10 insertions(+), 10 deletions(-) rename src/{assay.hpp => assay.h} (100%) rename src/{propmap.hpp => propmap.h} (95%) rename src/{tree.hpp => tree.h} (100%) diff --git a/src/assay.cpp b/src/assay.cpp index e6f1e365a..165a34f34 100644 --- a/src/assay.cpp +++ b/src/assay.cpp @@ -12,7 +12,7 @@ #include #include -#include "assay.hpp" +#include "assay.h" #include diff --git a/src/assay.hpp b/src/assay.h similarity index 100% rename from src/assay.hpp rename to src/assay.h diff --git a/src/faith_R.cpp b/src/faith_R.cpp index 87f98c50a..8c0f23232 100644 --- a/src/faith_R.cpp +++ b/src/faith_R.cpp @@ -3,9 +3,9 @@ #include -#include "assay.hpp" -#include "tree.hpp" -#include "propmap.hpp" +#include "assay.h" +#include "tree.h" +#include "propmap.h" /* Access the C++ implementation of the fast Faith's PD algorithm from R diff --git a/src/propmap.cpp b/src/propmap.cpp index 9501a5efa..c94e291e7 100644 --- a/src/propmap.cpp +++ b/src/propmap.cpp @@ -7,9 +7,9 @@ * See LICENSE file for more details */ -#include "tree.hpp" -#include "assay.hpp" -#include "propmap.hpp" +#include "tree.h" +#include "assay.h" +#include "propmap.h" #include #include diff --git a/src/propmap.hpp b/src/propmap.h similarity index 95% rename from src/propmap.hpp rename to src/propmap.h index a2862412e..c6ee7c1e7 100644 --- a/src/propmap.hpp +++ b/src/propmap.h @@ -14,8 +14,8 @@ #include #include -#include "tree.hpp" -#include "assay.hpp" +#include "tree.h" +#include "assay.h" namespace su { diff --git a/src/tree.cpp b/src/tree.cpp index 64f109c5c..b607f3ef5 100644 --- a/src/tree.cpp +++ b/src/tree.cpp @@ -1,4 +1,4 @@ -#include "tree.hpp" +#include "tree.h" #include #include diff --git a/src/tree.hpp b/src/tree.h similarity index 100% rename from src/tree.hpp rename to src/tree.h From 5d728ad77281d2d76502ed32cc1554b54c1206a6 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Tue, 25 Feb 2025 12:35:55 +0200 Subject: [PATCH 14/48] Remove old Faith's PD implementation --- R/addAlpha.R | 7 +---- R/estimateDiversity.R | 71 +++++-------------------------------------- 2 files changed, 8 insertions(+), 70 deletions(-) diff --git a/R/addAlpha.R b/R/addAlpha.R index f9a4f5b6c..12f133c11 100644 --- a/R/addAlpha.R +++ b/R/addAlpha.R @@ -39,12 +39,7 @@ #' whether to remove internal nodes when Faith's index is calculated. #' When \code{only.tips=TRUE}, those rows that are not tips of tree are #' removed. (Default: \code{FALSE}) -#' -#' \item \code{fast.faith}: (Faith's index). \code{Logical scalar}. Specifies -#' whether to use a C++ implementation of the Stacked Faith algorithm for -#' calculating Faith's index. This can speed up calculation by several orders -#' of magnitude for large datasets. (Default: \code{TRUE}) -#' +#' #' \item \code{threshold}: (Coverage and all evenness indices). #' \code{Numeric scalar}. #' From \code{0 to 1}, determines the threshold for coverage and evenness diff --git a/R/estimateDiversity.R b/R/estimateDiversity.R index 4f501e3d8..87f5897f9 100644 --- a/R/estimateDiversity.R +++ b/R/estimateDiversity.R @@ -121,82 +121,25 @@ NULL #' @importFrom ape reorder.phylo -.calc_faith <- function(mat, tree, only.tips = FALSE, fast.faith = TRUE, ...){ +.calc_faith <- function(mat, tree, only.tips = FALSE, ...){ # Input check if( !.is_a_bool(only.tips) ){ stop("'only.tips' must be TRUE or FALSE.", call. = FALSE) } - if( !.is_a_bool(fast.faith) ){ - stop("'fast.faith' must be TRUE or FALSE.", call. = FALSE) - } # Remove internal nodes if specified if( only.tips ){ mat <- mat[ rownames(mat) %in% tree$tip.label, ] } + # To ensure that the function works with NA also, convert NAs to 0. # Zero means that the taxon is not present --> same as NA (no information) mat[ is.na(mat) ] <- 0 - # Use fast algorithm if requested - if( fast.faith ){ - # The tree must be in cladewise order for the algorithm to work correctly - temp <- reorder.phylo(tree, "cladewise") - return(faith_cpp(mat, temp)) - } - - # Gets vector where number represent nth sample - samples <- seq_len(ncol(mat)) - - # Repeats taxa as many times there are samples, i.e. get all the - # taxa that are analyzed in each sample. - taxa <- rep(rownames(mat), length(samples)) - - # Gets those taxa that are present/absent in each sample. - # Gets one big list that combines - # taxa from all the samples. - present_combined <- taxa[ mat[, samples] > 0 ] - - # Gets how many taxa there are in each sample. - # After that, determines indices of samples' first taxa with cumsum. - split_present <- as.vector(cumsum(colSums(mat > 0))) - - # Determines which taxa belongs to which sample by first determining - # the splitting points, - # and after that giving every taxa number which tells their sample. - split_present <- as.factor(cumsum((seq_along(present_combined)-1) %in% - split_present)) - - # Assigns taxa to right samples based on their number that they got from - # previous step, and deletes unnecessary names. - present <- unname(split(present_combined, split_present)) - - # If there were samples without any taxa present/absent, the length of the - # list is not the number of samples since these empty samples are missing. - # Add empty samples as NULL. - names(present) <- names(which(colSums2(mat) > 0)) - present[names(which(colSums2(mat) == 0))] <- list(NULL) - present <- present[colnames(mat)] - - # Assign NA to all samples - faiths <- rep(NA,length(samples)) - - # If there are no taxa present, then faith is 0 - ind <- lengths(present) == 0 - faiths[ind] <- 0 - - # If there are taxa present - ind <- lengths(present) > 0 - # Loop through taxa that were found from each sample - faiths_for_taxa_present <- lapply(present[ind], function(x){ - # Trim the tree - temp <- .prune_tree(tree, x, ...) - # Sum up all the lengths of edges - temp <- sum(temp$edge.length) - return(temp) - }) - faiths_for_taxa_present <- unlist(faiths_for_taxa_present) - faiths[ind] <- faiths_for_taxa_present - return(faiths) + # The tree must be in cladewise order for the algorithm to work correctly + temp <- reorder.phylo(tree, "cladewise") + + # Call the C++ code + return(faith_cpp(mat, temp)) } .calc_log_modulo_skewness <- function(mat, quantile = 0.5, From e61b74ff18d1b4db8b3b8baf73631b1b43be03ba Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 3 Mar 2025 11:56:33 +0200 Subject: [PATCH 15/48] Update documentation --- NAMESPACE | 1 + R/RcppExports.R | 33 +++++++++++++++++++- R/addAlpha.R | 14 +++++++-- R/estimateDiversity.R | 2 +- man/addAlpha.Rd | 19 +++++++----- man/dot-faith_cpp.Rd | 43 ++++++++++++++++++++++++++ src/faith_R.cpp | 70 +++++++++++++++++++++++++++++-------------- src/tree.cpp | 9 ++++++ 8 files changed, 155 insertions(+), 36 deletions(-) create mode 100644 man/dot-faith_cpp.Rd diff --git a/NAMESPACE b/NAMESPACE index 6d3389d19..9700c061f 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,6 +1,7 @@ # Generated by roxygen2: do not edit by hand export("relabundance<-") +export(.faith_cpp) export(IdTaxaToDataFrame) export(ZTransform) export(addAbundanceClass) diff --git a/R/RcppExports.R b/R/RcppExports.R index f73465e76..dd7802c0f 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -1,7 +1,38 @@ # Generated by using Rcpp::compileAttributes() -> do not edit by hand # Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 -faith_cpp <- function(assay, rowTree) { +#' Calculate Faith's PD +#' +#' This function calculates Faith's phylogenetic diversity for a given assay +#' and rowTree, using a C++ implementation of the Stacked Faith's Phylogenetic +#' Diversity (SFPhD) algorithm. +#' +#' @details +#' This function makes several assumptions about the contents of +#' \code{assay} and \code{rowTree}, namely that: +#' \itemize{ +#' \item \code{assay} and \code{rowTree} are both non-empty. +#' \item \code{assay} has row and column names. +#' \item \code{rowTree}'s nodes are arranged in cladewise order. +#' } +#' These checks should all be handled in the surrounding R code. +#' +#' The values returned by this function are equivalent to the values returned +#' by \code{picante::pd()} with the parameter \code{include.root=TRUE}. +#' +#' The C++ code was adapted from an implementation by the Unifrac team (see +#' \url{https://genome.cshlp.org/content/31/11/2131} or +#' \url{https://github.com/biocore/unifrac}), which is licensed under the BSD +#' 3-Clause license. +#' +#' @param assay An R numeric matrix containing the assay of a \code{TreeSE} +#' object. +#' @param rowTree An \code{ape::phylo} object containing the rowTree of a +#' \code{TreeSE} object. +#' @return A vector containing Faith's PD values. +#' +#' @export +.faith_cpp <- function(assay, rowTree) { .Call('_mia_faith_cpp', PACKAGE = 'mia', assay, rowTree) } diff --git a/R/addAlpha.R b/R/addAlpha.R index 12f133c11..4940debe2 100644 --- a/R/addAlpha.R +++ b/R/addAlpha.R @@ -91,9 +91,12 @@ #' argument. #' #' \item 'faith': Faith's phylogenetic alpha diversity index measures how -#' long the taxonomic distance is between taxa that are present in the sample. -#' Larger values represent higher diversity. Using this index requires -#' rowTree. (Faith 1992) +#' long the taxonomic distance is between taxa that are present in the sample +#' (Faith 1992). Larger values represent higher diversity. The current +#' implementation is based on the Stacked Faith's Phylogenetic Diversity (SFPhD) +#' algorithm (Armstrong et al. 2021), which produces values equivalent to +#' \code{\link[picante:pd]{picante::pd}} with the parameter +#' \code{include.root=TRUE}. Using this index requires a rowTree. #' #' If the data includes features that are not in tree's tips but in #' internal nodes, there are two options. First, you can keep those features, @@ -341,6 +344,11 @@ #' Refer to Schloss (2024) for more details on rarefaction. #' #' @references +#' +#' Armstrong G. et al. (2021) +#' Efficient computation of Faith's phylogenetic diversity with applications +#' in characterizing microbiomes. +#' _Genome Res._ 31(11):2131-2137. doi: 10.1101/gr.275777.121 #' #' Beisel J-N. et al. (2003) #' A Comparative Analysis of Diversity Index Sensitivity. diff --git a/R/estimateDiversity.R b/R/estimateDiversity.R index 87f5897f9..7d99c97e6 100644 --- a/R/estimateDiversity.R +++ b/R/estimateDiversity.R @@ -139,7 +139,7 @@ NULL temp <- reorder.phylo(tree, "cladewise") # Call the C++ code - return(faith_cpp(mat, temp)) + return(.faith_cpp(mat, temp)) } .calc_log_modulo_skewness <- function(mat, quantile = 0.5, diff --git a/man/addAlpha.Rd b/man/addAlpha.Rd index 5eebc49ab..2ebac7cbf 100644 --- a/man/addAlpha.Rd +++ b/man/addAlpha.Rd @@ -52,11 +52,6 @@ whether to remove internal nodes when Faith's index is calculated. When \code{only.tips=TRUE}, those rows that are not tips of tree are removed. (Default: \code{FALSE}) -\item \code{fast.faith}: (Faith's index). \code{Logical scalar}. Specifies -whether to use a C++ implementation of the Stacked Faith algorithm for -calculating Faith's index. This can speed up calculation by several orders -of magnitude for large datasets. (Default: \code{TRUE}) - \item \code{threshold}: (Coverage and all evenness indices). \code{Numeric scalar}. From \code{0 to 1}, determines the threshold for coverage and evenness @@ -128,9 +123,12 @@ the ecosystem (50 percent by default). Tune this with the threshold argument. \item 'faith': Faith's phylogenetic alpha diversity index measures how -long the taxonomic distance is between taxa that are present in the sample. -Larger values represent higher diversity. Using this index requires -rowTree. (Faith 1992) +long the taxonomic distance is between taxa that are present in the sample +(Faith 1992). Larger values represent higher diversity. The current +implementation is based on the Stacked Faith's Phylogenetic Diversity (SFPhD) +algorithm (Armstrong et al. 2021), which produces values equivalent to +\code{\link[picante:pd]{picante::pd}} with the parameter +\code{include.root=TRUE}. Using this index requires a rowTree. If the data includes features that are not in tree's tips but in internal nodes, there are two options. First, you can keep those features, @@ -410,6 +408,11 @@ res |> head() } \references{ +Armstrong G. et al. (2021) +Efficient computation of Faith's phylogenetic diversity with applications +in characterizing microbiomes. +\emph{Genome Res.} 31(11):2131-2137. doi: 10.1101/gr.275777.121 + Beisel J-N. et al. (2003) A Comparative Analysis of Diversity Index Sensitivity. \emph{Internal Rev. Hydrobiol.} 88(1):3-15. diff --git a/man/dot-faith_cpp.Rd b/man/dot-faith_cpp.Rd new file mode 100644 index 000000000..df7d0a9f9 --- /dev/null +++ b/man/dot-faith_cpp.Rd @@ -0,0 +1,43 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/RcppExports.R +\name{.faith_cpp} +\alias{.faith_cpp} +\title{Calculate Faith's PD} +\usage{ +.faith_cpp(assay, rowTree) +} +\arguments{ +\item{assay}{An R numeric matrix containing the assay of a \code{TreeSE} +object.} + +\item{rowTree}{An \code{ape::phylo} object containing the rowTree of a +\code{TreeSE} object.} +} +\value{ +A vector containing Faith's PD values. +} +\description{ +This function calculates Faith's phylogenetic diversity for a given assay +and rowTree, using a C++ implementation of the Stacked Faith's Phylogenetic +Diversity (SFPhD) algorithm. +} +\details{ +This function makes several assumptions about the contents of +\code{assay} and \code{rowTree}, namely that: +\itemize{ +\item \code{assay} and \code{rowTree} are both non-empty. +\item \code{assay} has row and column names. +\item \code{assay}'s rownames can all be found in \code{rowTree}'s +metadata. +\item \code{rowTree}'s nodes are arranged in cladewise order. +} +These checks should all be handled in the surrounding R code. + +The values returned by this function are equivalent to the values returned +by \code{picante::pd()} with the parameter \code{include.root=TRUE}. + +The C++ code was adapted from an implementation by the Unifrac team (see +\url{https://genome.cshlp.org/content/31/11/2131} or +\url{https://github.com/biocore/unifrac}), which is licensed under the BSD +3-Clause license. +} diff --git a/src/faith_R.cpp b/src/faith_R.cpp index 8c0f23232..c0ba38730 100644 --- a/src/faith_R.cpp +++ b/src/faith_R.cpp @@ -1,3 +1,12 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + #include #include @@ -7,31 +16,47 @@ #include "tree.h" #include "propmap.h" - -/* Access the C++ implementation of the fast Faith's PD algorithm from R - * - * treeSE an R TreeSummarizedExperiment object. - * faith the resulting vector of computed Faith PD values - * - * This functions makes several assumptions about treeSE: - * - * - It must contain a non-empty counts assay and a non-empty RowTree - * - The RowTree must be sorted in cladewise order - * - The RowTree must be rooted - Unrooted trees can be passed without error, but don't produce correct results - * - // Check that tree and table are non-empty and match before calling the c++ code - // shear the tree (to contain only the obs in the table?) - Also should be done before the call? - // Assure that tree does not contain ids that are not in table - * - */ -// [[Rcpp::export]] -Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree){ - +//' Calculate Faith's PD +//' +//' This function calculates Faith's phylogenetic diversity for a given assay +//' and rowTree, using a C++ implementation of the Stacked Faith's Phylogenetic +//' Diversity (SFPhD) algorithm. +//' +//' @details +//' This function makes several assumptions about the contents of +//' \code{assay} and \code{rowTree}, namely that: +//' \itemize{ +//' \item \code{assay} and \code{rowTree} are both non-empty. +//' \item \code{assay} has row and column names. +//' \item \code{rowTree}'s nodes are arranged in cladewise order. +//' } +//' These checks should all be handled in the surrounding R code. +//' +//' The values returned by this function are equivalent to the values returned +//' by \code{picante::pd()} with the parameter \code{include.root=TRUE}. +//' +//' The C++ code was adapted from an implementation by the Unifrac team (see +//' \url{https://genome.cshlp.org/content/31/11/2131} or +//' \url{https://github.com/biocore/unifrac}), which is licensed under the BSD +//' 3-Clause license. +//' +//' @param assay An R numeric matrix containing the assay of a \code{TreeSE} +//' object. +//' @param rowTree An \code{ape::phylo} object containing the rowTree of a +//' \code{TreeSE} object. +//' @return A vector containing Faith's PD values. +//' +//' @export +// [[Rcpp::export(.faith_cpp)]] +Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, + const Rcpp::List & rowTree){ su::BPTree tree = su::BPTree(rowTree); su::Assay table = su::Assay(assay); - std::unordered_set to_keep(table.obs_ids.begin(),table.obs_ids.end()); + std::unordered_set to_keep(table.obs_ids.begin(), + table.obs_ids.end()); + su::BPTree tree_sheared = tree.shear(to_keep).collapse(); su::PropMap propmap(table.n_samples); @@ -51,11 +76,10 @@ Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::Lis length = tree_sheared.lengths[node]; // get node proportions and set intermediate scores - node_proportions = set_proportions(tree_sheared, node, table, propmap, false); // this would probably be the most likely culprit for something going wrong + node_proportions = set_proportions(tree_sheared, node, table, propmap, false); for (unsigned int sample = 0; sample < table.n_samples; sample++){ // calculate contribution of node to score - // Is it possible to somehow set the proportions to 0 if we're dealing with the root in a include.root=FALSE scenario? results[sample] += (node_proportions[sample] > 0) * length; } } diff --git a/src/tree.cpp b/src/tree.cpp index b607f3ef5..d516660e3 100644 --- a/src/tree.cpp +++ b/src/tree.cpp @@ -1,3 +1,12 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + #include "tree.h" #include From d5d0288be4d107788ba7bbfdab48d5202a0fa1c6 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Wed, 12 Mar 2025 11:01:18 +0200 Subject: [PATCH 16/48] Make C++ documentation internal --- R/RcppExports.R | 1 + man/dot-faith_cpp.Rd | 3 +-- src/faith_R.cpp | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/R/RcppExports.R b/R/RcppExports.R index dd7802c0f..315b206e0 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -31,6 +31,7 @@ #' \code{TreeSE} object. #' @return A vector containing Faith's PD values. #' +#' @keywords internal #' @export .faith_cpp <- function(assay, rowTree) { .Call('_mia_faith_cpp', PACKAGE = 'mia', assay, rowTree) diff --git a/man/dot-faith_cpp.Rd b/man/dot-faith_cpp.Rd index df7d0a9f9..d32a8c996 100644 --- a/man/dot-faith_cpp.Rd +++ b/man/dot-faith_cpp.Rd @@ -27,8 +27,6 @@ This function makes several assumptions about the contents of \itemize{ \item \code{assay} and \code{rowTree} are both non-empty. \item \code{assay} has row and column names. -\item \code{assay}'s rownames can all be found in \code{rowTree}'s -metadata. \item \code{rowTree}'s nodes are arranged in cladewise order. } These checks should all be handled in the surrounding R code. @@ -41,3 +39,4 @@ The C++ code was adapted from an implementation by the Unifrac team (see \url{https://github.com/biocore/unifrac}), which is licensed under the BSD 3-Clause license. } +\keyword{internal} diff --git a/src/faith_R.cpp b/src/faith_R.cpp index c0ba38730..f7120df8f 100644 --- a/src/faith_R.cpp +++ b/src/faith_R.cpp @@ -46,6 +46,7 @@ //' \code{TreeSE} object. //' @return A vector containing Faith's PD values. //' +//' @keywords internal //' @export // [[Rcpp::export(.faith_cpp)]] Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, From 6f3e370650088f8d993cc3fea29ee8c21f859ee2 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Thu, 13 Mar 2025 12:23:42 +0200 Subject: [PATCH 17/48] Remove export keyword --- NAMESPACE | 1 - R/RcppExports.R | 1 - src/faith_R.cpp | 1 - 3 files changed, 3 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 16abb01e6..f33e12450 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,7 +1,6 @@ # Generated by roxygen2: do not edit by hand export("relabundance<-") -export(.faith_cpp) export(IdTaxaToDataFrame) export(ZTransform) export(addAbundanceClass) diff --git a/R/RcppExports.R b/R/RcppExports.R index 315b206e0..33b32c811 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -32,7 +32,6 @@ #' @return A vector containing Faith's PD values. #' #' @keywords internal -#' @export .faith_cpp <- function(assay, rowTree) { .Call('_mia_faith_cpp', PACKAGE = 'mia', assay, rowTree) } diff --git a/src/faith_R.cpp b/src/faith_R.cpp index f7120df8f..babc12ff4 100644 --- a/src/faith_R.cpp +++ b/src/faith_R.cpp @@ -47,7 +47,6 @@ //' @return A vector containing Faith's PD values. //' //' @keywords internal -//' @export // [[Rcpp::export(.faith_cpp)]] Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree){ From cb85a3b34d69243c88e498b2a118d93f3f5f1ad4 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Thu, 13 Mar 2025 12:39:11 +0200 Subject: [PATCH 18/48] Update reference style --- R/RcppExports.R | 6 ++---- man/dot-faith_cpp.Rd | 6 ++---- src/faith_R.cpp | 6 ++---- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/R/RcppExports.R b/R/RcppExports.R index 33b32c811..a26903934 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -20,10 +20,8 @@ #' The values returned by this function are equivalent to the values returned #' by \code{picante::pd()} with the parameter \code{include.root=TRUE}. #' -#' The C++ code was adapted from an implementation by the Unifrac team (see -#' \url{https://genome.cshlp.org/content/31/11/2131} or -#' \url{https://github.com/biocore/unifrac}), which is licensed under the BSD -#' 3-Clause license. +#' The C++ code was adapted from an implementation by the Unifrac team +#' (Armstrong et al. 2021), which is licensed under the BSD 3-Clause license. #' #' @param assay An R numeric matrix containing the assay of a \code{TreeSE} #' object. diff --git a/man/dot-faith_cpp.Rd b/man/dot-faith_cpp.Rd index d32a8c996..da3323ed2 100644 --- a/man/dot-faith_cpp.Rd +++ b/man/dot-faith_cpp.Rd @@ -34,9 +34,7 @@ These checks should all be handled in the surrounding R code. The values returned by this function are equivalent to the values returned by \code{picante::pd()} with the parameter \code{include.root=TRUE}. -The C++ code was adapted from an implementation by the Unifrac team (see -\url{https://genome.cshlp.org/content/31/11/2131} or -\url{https://github.com/biocore/unifrac}), which is licensed under the BSD -3-Clause license. +The C++ code was adapted from an implementation by the Unifrac team +(Armstrong et al. 2021), which is licensed under the BSD 3-Clause license. } \keyword{internal} diff --git a/src/faith_R.cpp b/src/faith_R.cpp index babc12ff4..7f54d3bfe 100644 --- a/src/faith_R.cpp +++ b/src/faith_R.cpp @@ -35,10 +35,8 @@ //' The values returned by this function are equivalent to the values returned //' by \code{picante::pd()} with the parameter \code{include.root=TRUE}. //' -//' The C++ code was adapted from an implementation by the Unifrac team (see -//' \url{https://genome.cshlp.org/content/31/11/2131} or -//' \url{https://github.com/biocore/unifrac}), which is licensed under the BSD -//' 3-Clause license. +//' The C++ code was adapted from an implementation by the Unifrac team +//' (Armstrong et al. 2021), which is licensed under the BSD 3-Clause license. //' //' @param assay An R numeric matrix containing the assay of a \code{TreeSE} //' object. From e446c8f48efe2bc63256eb173434b2c8ae579a48 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 17 Mar 2025 12:08:08 +0200 Subject: [PATCH 19/48] Clean up C++ code --- src/assay.cpp | 44 +++++---- src/assay.h | 117 +++++++++++------------ src/faith_R.cpp | 38 ++++---- src/propmap.cpp | 67 +++++++------ src/propmap.h | 46 ++++----- src/tree.cpp | 246 +++++++++++++++++++++++++----------------------- src/tree.h | 244 +++++++++++++++++++++++------------------------ 7 files changed, 407 insertions(+), 395 deletions(-) diff --git a/src/assay.cpp b/src/assay.cpp index 165a34f34..bf8254290 100644 --- a/src/assay.cpp +++ b/src/assay.cpp @@ -1,11 +1,11 @@ /* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ +* BSD 3-Clause License +* +* Copyright (c) 2016-2021, UniFrac development team. +* All rights reserved. +* +* See LICENSE file for more details +*/ #include #include @@ -18,7 +18,7 @@ using namespace su; -Assay::Assay(const Rcpp::NumericMatrix & assay) { +Assay::Assay(const Rcpp::NumericMatrix & assay){ table = assay; sample_ids = std::vector(); @@ -29,53 +29,51 @@ Assay::Assay(const Rcpp::NumericMatrix & assay) { Rcpp::StringVector rownames = Rcpp::rownames(table); obs_ids = Rcpp::as>(rownames); - + n_samples = sample_ids.size(); n_obs = obs_ids.size(); - - /* define a mapping between an ID and its corresponding offset */ + + /* Define a mapping between an ID and its corresponding offset */ obs_id_index = std::unordered_map(); sample_id_index = std::unordered_map(); - + create_id_index(obs_ids, obs_id_index); create_id_index(sample_ids, sample_id_index); - + sample_counts = get_sample_counts(); - } -Assay::~Assay() { - +Assay::~Assay(){ } void Assay::create_id_index(std::vector &ids, - std::unordered_map &map) { + std::unordered_map &map){ uint32_t count = 0; map.reserve(ids.size()); - for(auto i = ids.begin(); i != ids.end(); i++, count++) { + for( auto i = ids.begin(); i != ids.end(); i++, count++ ){ map[*i] = count; } } - std::vector Assay::get_obs_data(const std::string &id) const { std::vector out = std::vector(); uint32_t idx = obs_id_index.at(id); - for(unsigned int i = 0; i < n_samples; i++) { + for( unsigned int i = 0; i < n_samples; i++ ){ out.push_back(table(idx, i)); } return out; } -std::vector Assay::get_sample_counts() { +std::vector Assay::get_sample_counts(){ std::vector sample_counts = std::vector(); - for(unsigned int i = 0; i < n_samples; i++) { + for( unsigned int i = 0; i < n_samples; i++ ){ unsigned int sum = 0; - for(unsigned int j = 0; j < n_obs; j++){ + for( unsigned int j = 0; j < n_obs; j++ ){ sum += table(j, i); } sample_counts.push_back(sum); } + return(sample_counts); } diff --git a/src/assay.h b/src/assay.h index 6b19d619e..36ae632d9 100644 --- a/src/assay.h +++ b/src/assay.h @@ -1,11 +1,11 @@ /* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ +* BSD 3-Clause License +* +* Copyright (c) 2016-2021, UniFrac development team. +* All rights reserved. +* +* See LICENSE file for more details +*/ #ifndef __FAITH_ASSAY_H #define __FAITH_ASSAY_H 1 @@ -16,57 +16,58 @@ #include namespace su { - class Assay { - public: - // cache the IDs contained within the table - std::vector sample_ids; - std::vector obs_ids; - - uint32_t n_samples; // the number of samples - uint32_t n_obs; // the number of observations - std::vector sample_counts; // Counts summed per sample - - /* default constructor - * - * @param treeSE An R TreeSummarizedExperiment object - */ - Assay(const Rcpp::NumericMatrix & assay); - - /* default destructor - * - * Temporary arrays are freed - */ - ~Assay(); - - /* get a dense vector of observation data - * - * @param id The observation ID to fetch - * @param out An allocated array of at least size n_samples. - * Values of an index position [0, n_samples) which do not - * have data will be zero'd. - */ - std::vector get_obs_data(const std::string &id) const; - - private: - Rcpp::NumericMatrix table; // Access to the raw sample counts in R's memory - - std::vector get_sample_counts(); - - /* At construction, lookups mapping IDs -> index position within an - * axis are defined - */ - std::unordered_map obs_id_index; - std::unordered_map sample_id_index; - - /* create an index mapping an ID to its corresponding index - * position. - * - * @param ids A vector of IDs to index - * @param map A hash table to populate - */ - void create_id_index(std::vector &ids, - std::unordered_map &map); - }; +class Assay { + public: + // Cache the IDs contained within the table + std::vector sample_ids; + std::vector obs_ids; + + uint32_t n_samples; // The number of samples + uint32_t n_obs; // The number of observations + std::vector sample_counts; // Counts summed per sample + + /* Default constructor + * + * @param treeSE An R TreeSummarizedExperiment object + */ + Assay(const Rcpp::NumericMatrix & assay); + + /* Default destructor + * + * Temporary arrays are freed + */ + ~Assay(); + + /* Get a dense vector of observation data + * + * @param id The observation ID to fetch + * @param out An allocated array of at least size n_samples. + * Values of an index position [0, n_samples) which do not + * have data will be zero'd. + */ + std::vector get_obs_data(const std::string &id) const; + + private: + Rcpp::NumericMatrix table; // Access to raw sample counts in R's memory + + std::vector get_sample_counts(); + + /* At construction, lookups mapping IDs -> index position within an + * axis are defined + */ + std::unordered_map obs_id_index; + std::unordered_map sample_id_index; + + /* Create an index mapping an ID to its corresponding index + * position. + * + * @param ids A vector of IDs to index + * @param map A hash table to populate + */ + void create_id_index(std::vector &ids, + std::unordered_map &map); + }; } #endif /* __FAITH_ASSAY_H */ diff --git a/src/faith_R.cpp b/src/faith_R.cpp index 7f54d3bfe..0d6835757 100644 --- a/src/faith_R.cpp +++ b/src/faith_R.cpp @@ -1,11 +1,11 @@ /* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ +* BSD 3-Clause License +* +* Copyright (c) 2016-2021, UniFrac development team. +* All rights reserved. +* +* See LICENSE file for more details +*/ #include #include @@ -47,8 +47,7 @@ //' @keywords internal // [[Rcpp::export(.faith_cpp)]] Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, - const Rcpp::List & rowTree){ - + const Rcpp::List & rowTree){ su::BPTree tree = su::BPTree(rowTree); su::Assay table = su::Assay(assay); @@ -65,26 +64,29 @@ Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, std::vector results = std::vector(table.n_samples, 0.0); - // for node in postorderselect - const unsigned int max_k = (tree_sheared.nparens>1) ? ((tree_sheared.nparens / 2) - 1) : 0; - for(unsigned int k = 0; k < max_k; k++) { + // For node in postorderselect + const unsigned int max_k = (tree_sheared.nparens>1) ? + ((tree_sheared.nparens / 2) - 1) : 0; + + for( unsigned int k = 0; k < max_k; k++ ){ node = tree_sheared.postorderselect(k); - // get branch length + // Get branch length length = tree_sheared.lengths[node]; - // get node proportions and set intermediate scores - node_proportions = set_proportions(tree_sheared, node, table, propmap, false); + // Get node proportions and set intermediate scores + node_proportions = set_proportions(tree_sheared, node, table, propmap, + false); - for (unsigned int sample = 0; sample < table.n_samples; sample++){ - // calculate contribution of node to score + for( unsigned int sample = 0; sample < table.n_samples; sample++ ){ + // Calculate contribution of node to score results[sample] += (node_proportions[sample] > 0) * length; } } Rcpp::NumericVector faith = Rcpp::NumericVector(results.size()); - for(unsigned int i = 0; i < results.size(); i++){ + for( unsigned int i = 0; i < results.size(); i++ ){ faith[i] = results[i]; } diff --git a/src/propmap.cpp b/src/propmap.cpp index c94e291e7..ce02cfa1e 100644 --- a/src/propmap.cpp +++ b/src/propmap.cpp @@ -1,11 +1,11 @@ /* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ +* BSD 3-Clause License +* +* Copyright (c) 2016-2021, UniFrac development team. +* All rights reserved. +* +* See LICENSE file for more details +*/ #include "tree.h" #include "assay.h" @@ -24,8 +24,8 @@ using namespace su; PropMap::PropMap(uint32_t vecsize) -: prop_map() -, defaultsize(vecsize) + : prop_map() + , defaultsize(vecsize) { prop_map.reserve(1000); } @@ -33,59 +33,56 @@ PropMap::PropMap(uint32_t vecsize) PropMap::~PropMap() { } -std::vector PropMap::get(uint32_t i) { - if(prop_map.count(i) > 0){ +std::vector PropMap::get(uint32_t i){ + if( prop_map.count(i) > 0 ){ return prop_map.at(i); - } - else { + } else { return(std::vector()); } } -void PropMap::clear(uint32_t i) { +void PropMap::clear(uint32_t i){ prop_map[i] = std::vector(); } -void PropMap::update(uint32_t node, std::vector vec) { +void PropMap::update(uint32_t node, std::vector vec){ prop_map[node] = vec; } std::vector su::set_proportions(const BPTree &tree, - uint32_t node, - const Assay &table, - PropMap &ps, - bool normalize) { - + uint32_t node, + const Assay &table, + PropMap &ps, + bool normalize){ std::vector props = std::vector(); - if(tree.isleaf(node)) { + if( tree.isleaf(node) ){ std::string leaf = tree.names[node]; - props = table.get_obs_data(leaf); // Here we basically just need the row for the specified node - if (normalize) { - for(unsigned int i = 0; i < table.n_samples; i++) { - props[i] /= table.sample_counts[i]; + props = table.get_obs_data(leaf); // get row for the specified node + if( normalize ){ + for( unsigned int i = 0; i < table.n_samples; i++ ){ + props[i] /= table.sample_counts[i]; } - } + } } else { unsigned int current = tree.leftchild(node); unsigned int right = tree.rightchild(node); - - for(unsigned int i = 0; i < table.n_samples; i++){ + + for( unsigned int i = 0; i < table.n_samples; i++ ){ props.push_back(0); } - while(current <= right && current != 0) { - std::vector vec = ps.get(current); // pull from prop map - ps.clear(current); // remove from prop map, place back on stack + while( current <= right && current != 0 ){ + std::vector vec = ps.get(current); // Pull from prop map + ps.clear(current); // Remove from prop map - for(unsigned int i = 0; i < table.n_samples; i++) + for( unsigned int i = 0; i < table.n_samples; i++ ){ props[i] = props[i] + vec[i]; + } current = tree.rightsibling(current); } - - //std::cout << "n " << props[0] << " " << props[1] << " " << props[2] << "\n"; - } + ps.update(node, props); return(props); } diff --git a/src/propmap.h b/src/propmap.h index c6ee7c1e7..ec936d194 100644 --- a/src/propmap.h +++ b/src/propmap.h @@ -1,11 +1,11 @@ /* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ +* BSD 3-Clause License +* +* Copyright (c) 2016-2021, UniFrac development team. +* All rights reserved. +* +* See LICENSE file for more details +*/ #ifndef __FAITH_PROPMAP #define __FAITH_PROPMAP 1 @@ -18,23 +18,23 @@ #include "assay.h" namespace su { +class PropMap { + public: + PropMap(uint32_t vecsize); + virtual ~PropMap(); + void clear(uint32_t i); + void update(uint32_t i, std::vector vec); + std::vector get(uint32_t i); + + private: + std::unordered_map> prop_map; + uint32_t defaultsize; +}; - class PropMap { - private: - std::unordered_map> prop_map; - uint32_t defaultsize; - public: - PropMap(uint32_t vecsize); - virtual ~PropMap(); - void clear(uint32_t i); - void update(uint32_t i, std::vector vec); - std::vector get(uint32_t i); - }; - - std::vector set_proportions(const BPTree &tree, uint32_t node, - const Assay &table, - PropMap &ps, - bool normalize = true); +std::vector set_proportions(const BPTree &tree, uint32_t node, + const Assay &table, + PropMap &ps, + bool normalize = true); } #endif /* __FAITH_PROPMAP */ diff --git a/src/tree.cpp b/src/tree.cpp index d516660e3..480d92cfd 100644 --- a/src/tree.cpp +++ b/src/tree.cpp @@ -1,11 +1,11 @@ /* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ +* BSD 3-Clause License +* +* Copyright (c) 2016-2021, UniFrac development team. +* All rights reserved. +* +* See LICENSE file for more details +*/ #include "tree.h" @@ -16,14 +16,15 @@ using namespace su; -BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names) { - +BPTree::BPTree(std::vector input_structure, + std::vector input_lengths, + std::vector input_names){ structure = input_structure; lengths = input_lengths; names = input_names; nparens = structure.size(); - + openclose = std::vector(); select_0_index = std::vector(); select_1_index = std::vector(); @@ -31,14 +32,14 @@ BPTree::BPTree(std::vector input_structure, std::vector input_leng select_0_index.resize(nparens / 2); select_1_index.resize(nparens / 2); excess.resize(nparens); - + structure_to_openclose(); index_and_cache(); } -BPTree::BPTree(const Rcpp::List & rowTree) { +BPTree::BPTree(const Rcpp::List & rowTree){ - //Initialize vectors + // Initialize vectors openclose = std::vector(); lengths = std::vector(); names = std::vector(); @@ -47,13 +48,13 @@ BPTree::BPTree(const Rcpp::List & rowTree) { select_0_index = std::vector(); select_1_index = std::vector(); - //Load the tree structure + // Load the tree structure structure = std::vector(); - structure.reserve(500000); // a fair sized tree... avoid reallocs, and its not _that_ much waste if this is wrong - rowTree_to_bp(rowTree); //Also sets the size of nparens + structure.reserve(500000); // A reasonably large initial allocation + rowTree_to_bp(rowTree); // Also sets the size of nparens - //Resize vectors - // resize is correct here as we are not performing a push_back + // Resize vectors + // Resize is correct here as we are not performing a push_back openclose.resize(nparens); lengths.resize(nparens); names.resize(nparens); @@ -62,38 +63,39 @@ BPTree::BPTree(const Rcpp::List & rowTree) { select_0_index.resize(nparens / 2); select_1_index.resize(nparens / 2); - //Builds a vector that lets us find the corresponding indices for each true/false pair + // Build a vector that lets us find the corresponding indices for each + // true/false pair structure_to_openclose(); - //Get metadata + // Get metadata rowTree_to_metadata(rowTree); - //Finalize + // Finalize index_and_cache(); } -BPTree BPTree::mask(std::vector topology_mask, std::vector in_lengths) { +BPTree BPTree::mask(std::vector topology_mask, + std::vector in_lengths){ std::vector new_structure = std::vector(); std::vector new_lengths = std::vector(); std::vector new_names = std::vector(); - + uint32_t count = 0; - for(auto i = topology_mask.begin(); i != topology_mask.end(); i++) { - if(*i) - count++; + for( auto i = topology_mask.begin(); i != topology_mask.end(); i++ ){ + if( *i ) count++; } - + new_structure.resize(count); new_lengths.resize(count); new_names.resize(count); - + auto mask_it = topology_mask.begin(); auto base_it = this->structure.begin(); uint32_t new_idx = 0; uint32_t old_idx = 0; - for(; mask_it != topology_mask.end(); mask_it++, base_it++, old_idx++) { - if(*mask_it) { + for( ; mask_it != topology_mask.end(); mask_it++, base_it++, old_idx++ ){ + if( *mask_it ){ new_structure[new_idx] = this->structure[old_idx]; new_lengths[new_idx] = in_lengths[old_idx]; new_names[new_idx] = this->names[old_idx]; @@ -104,55 +106,56 @@ BPTree BPTree::mask(std::vector topology_mask, std::vector in_leng return BPTree(new_structure, new_lengths, new_names); } -std::unordered_set BPTree::get_tip_names() { +std::unordered_set BPTree::get_tip_names(){ std::unordered_set observed; - - for(unsigned int i = 0; i < this->nparens; i++) { - if(this->isleaf(i)) { + + for( unsigned int i = 0; i < this->nparens; i++ ){ + if( this->isleaf(i) ){ observed.insert(this->names[i]); } } - + return observed; } -BPTree BPTree::shear(std::unordered_set to_keep) { +BPTree BPTree::shear(std::unordered_set to_keep){ std::vector shearmask = std::vector(this->nparens); int32_t p; - - for(unsigned int i = 0; i < this->nparens; i++) { - if(this->isleaf(i) && to_keep.count(this->names[i]) > 0) { + + for( unsigned int i = 0; i < this->nparens; i++ ){ + if( this->isleaf(i) && to_keep.count(this->names[i]) > 0 ){ shearmask[i] = true; shearmask[i+1] = true; - + p = this->parent(i); - while(p != -1 && !shearmask[p]) { + while( p != -1 && !shearmask[p] ){ shearmask[p] = true; shearmask[this->close(p)] = true; p = this->parent(p); } } } + return this->mask(shearmask, this->lengths); } BPTree BPTree::collapse() { std::vector collapsemask = std::vector(this->nparens); std::vector new_lengths = std::vector(this->lengths); - + uint32_t current, first, last; - - for(uint32_t i = 0; i < this->nparens / 2; i++) { + + for( uint32_t i = 0; i < this->nparens / 2; i++ ){ current = this->preorderselect(i); - - if(this->isleaf(current) or (current == 0)) { // 0 == root + + if( this->isleaf(current) or (current == 0) ){ // 0 == root collapsemask[current] = true; collapsemask[this->close(current)] = true; } else { first = this->leftchild(current); last = this->rightchild(current); - - if(first == last) { + + if( first == last ) { new_lengths[first] = new_lengths[first] + new_lengths[current]; } else { collapsemask[current] = true; @@ -160,15 +163,15 @@ BPTree BPTree::collapse() { } } } - + return this->mask(collapsemask, new_lengths); } -BPTree::~BPTree() { +BPTree::~BPTree(){ } -void BPTree::index_and_cache() { - // should probably do the open/close in here too +void BPTree::index_and_cache(){ + // Should probably do the open/close in here too unsigned int idx = 0; auto i = structure.begin(); auto k0 = select_0_index.begin(); @@ -176,12 +179,11 @@ void BPTree::index_and_cache() { auto e_it = excess.begin(); unsigned int e = 0; - for(; i != structure.end(); i++, idx++ ) { - if(*i) { + for( ; i != structure.end(); i++, idx++ ){ + if( *i ){ *(k1++) = idx; *(e_it++) = ++e; - } - else { + } else { *(k0++) = idx; *(e_it++) = --e; } @@ -209,30 +211,36 @@ bool BPTree::isleaf(unsigned int idx) const { } uint32_t BPTree::leftchild(uint32_t i) const { - // aka fchild - if(isleaf(i)) - return 0; // this is awkward, using 0 which is root, but a root cannot be a child. edge case - else + // Aka fchild + if( isleaf(i) ){ + return 0; + } else { return i + 1; + } } uint32_t BPTree::rightchild(uint32_t i) const { - // aka lchild - if(isleaf(i)) - return 0; // this is awkward, using 0 which is root, but a root cannot be a child. edge case - else + // Aka lchild + if( isleaf(i) ){ + // This is awkward, using 0 which is root, but a root cannot be a child. + // Edge case. + return 0; + } else { return open(close(i) - 1); + } } uint32_t BPTree::rightsibling(uint32_t i) const { - // aka nsibling + // Aka nsibling uint32_t position = close(i) + 1; - if(position >= nparens) - return 0; // will return 0 if no sibling as root cannot have a sibling - else if(structure[position]) + if( position >= nparens ){ + // Will return 0 if no siblings, as root cannot have a sibling + return 0; + } else if(structure[position]){ return position; - else + } else { return 0; + } } int32_t BPTree::parent(uint32_t i) const { @@ -240,28 +248,30 @@ int32_t BPTree::parent(uint32_t i) const { } int32_t BPTree::enclose(uint32_t i) const { - if(structure[i]) + if(structure[i]){ return bwd(i, -2) + 1; - else - return bwd(i - 1, -2) + 1; + } else { + return bwd(i - 1, -2) + 1; + } } int32_t BPTree::bwd(uint32_t i, int d) const { uint32_t target_excess = excess[i] + d; - for(int current_idx = i - 1; current_idx >= 0; current_idx--) { - if(excess[current_idx] == target_excess) + for( int current_idx = i - 1; current_idx >= 0; current_idx-- ){ + if( excess[current_idx] == target_excess ){ return current_idx; + } } return -1; } -// The algorithms that this class uses need the tree to be stored in a binary format +// This class needs the tree to be stored in a binary format // In terms of the Newick format, an opening bracket corresponds to a TRUE, // a closing bracket to a FALSE, and a tip to a TRUE FALSE // This function assumes that the tree representation is in cladewise order - // Ensure this with ape's reorder.phylo() function // Seems to work whether or not the tree is rooted -void BPTree::rowTree_to_bp(const Rcpp::List & phylo) { +void BPTree::rowTree_to_bp(const Rcpp::List & phylo){ Rcpp::NumericMatrix edge = phylo["edge"]; Rcpp::StringVector tips = phylo["tip.label"]; @@ -274,51 +284,50 @@ void BPTree::rowTree_to_bp(const Rcpp::List & phylo) { unsigned int currentNode = 0; unsigned int nextNode = 0; - // Goal: Insert true when a branch starts, a false when it closes, and a true-false for each tip. - - for (int i = 0; i < edge.nrow(); i++){ + for( int i = 0; i < edge.nrow(); i++ ){ currentNode = edge(i, 0); nextNode = edge(i, 1); - if(nodes.size() > 0 && currentNode < nodes.top()) { + if( nodes.size() > 0 && currentNode < nodes.top() ){ // We've exhausted the branch and moved backwards in the tree do { nodes.pop(); structure.push_back(false); - } while(currentNode != nodes.top()); + } while( currentNode != nodes.top() ); } - if(nodes.size() == 0 || currentNode > nodes.top() ) { + if( nodes.size() == 0 || currentNode > nodes.top() ){ // We are either at the root, or entering a new node nodes.push(currentNode); structure.push_back(true); } - if(nextNode <= ntips) { + if( nextNode <= ntips ){ // We've found a tip structure.push_back(true); structure.push_back(false); } - if(i == edge.nrow() - 1) { + if( i == edge.nrow() - 1 ){ // We've reached the end of the tree do { nodes.pop(); structure.push_back(false); - } while(nodes.size() > 0); + } while( nodes.size() > 0 ); } } + nparens = structure.size(); } -void BPTree::structure_to_openclose() { +void BPTree::structure_to_openclose(){ std::stack oc; unsigned int open_idx; unsigned int i = 0; - - for(auto it = structure.begin(); it != structure.end(); it++, i++) { - if(*it) { + + for( auto it = structure.begin(); it != structure.end(); it++, i++ ){ + if( *it ) { oc.push(i); } else { open_idx = oc.top(); @@ -330,59 +339,63 @@ void BPTree::structure_to_openclose() { } // Add metadata (lengths and names) to the tree representation -// Iterate through the structure, and whenever we hit a true decide if -// it's a leaf or not, and then add the corresponding label/length -// edge.length has (nodes + tips) elements - leaves at the start, nodes at the end -// tip.label has (tips) elements +// Iterate through the structure, and whenever we hit a true decide if it's a +// leaf or not, and then add the corresponding label/length + +// edge.length has (nodes + tips) elements - +// leaves at the start, nodes at the end. +// tip.label has (tips) elements. // root.edge and node.labels are optional, giving the length of the root and -// the internal node (including root) labels, respectively -void BPTree::rowTree_to_metadata(const Rcpp::List & phylo) { +// the internal node (including root) labels, respectively. +void BPTree::rowTree_to_metadata(const Rcpp::List & phylo){ Rcpp::NumericVector edgelength = phylo["edge.length"]; Rcpp::NumericMatrix edges = phylo["edge"]; Rcpp::StringVector tips = phylo["tip.label"]; const uint32_t n_edges = edgelength.size(); uint32_t ntips = tips.size(); - - //Used to find the correct lengths for the nodes - Includes the root + + // Used to find the correct lengths for the nodes - Includes the root std::vector edge_v(n_edges + 1, 0.0); - for(unsigned int i = 0; i < n_edges; i++){ + for( unsigned int i = 0; i < n_edges; i++ ){ edge_v.at(edges(i,1) - 1) = edgelength[i]; } - if(phylo.containsElementNamed("root.edge")) { + if( phylo.containsElementNamed("root.edge") ){ edge_v.at(ntips) = phylo["root.edge"]; } bool hasNodeLabels = false; Rcpp::StringVector nodes; - if(phylo.containsElementNamed("node.labels")) { + if( phylo.containsElementNamed("node.labels") ){ hasNodeLabels = true; nodes = phylo["node.labels"]; } - unsigned int tip_idx = 0; // tip indices run from 0 to ntips-1 - unsigned int node_idx = 0; // node indices run from ntips to ntips + nnodes - 1 + // Tip indices run from 0 to ntips-1 + unsigned int tip_idx = 0; + // Node indices run from ntips to ntips + nnodes - 1 + unsigned int node_idx = 0; - for(unsigned int i = 0; i < structure.size(); i++) { - if(structure[i]){ + for( unsigned int i = 0; i < structure.size(); i++ ){ + if( structure[i] ){ std::string label = std::string(); double length = 0.0; - if(isleaf(i)){ - //Tips can be expected to have both a length and a label + if( isleaf(i) ){ + // Tips can be expected to have both a length and a label label = Rcpp::as(tips[tip_idx]); length = edge_v[tip_idx]; tip_idx++; - } - - else{ - //Nodes always have lengths (except the root, which may have it optionally, but defaults to 0.0) - //Nodes may also optionally have labels (which includes the root label) + } else { + // Nodes always have lengths (except the root, which may have it + // optionally, but defaults to 0.0) + // Nodes may also optionally have labels (which includes the + // root label) length = edge_v[ntips + node_idx]; - if(hasNodeLabels){ + if( hasNodeLabels ){ label = Rcpp::as(nodes[node_idx]); } node_idx++; @@ -392,16 +405,17 @@ void BPTree::rowTree_to_metadata(const Rcpp::List & phylo) { } } -//This takes a label and a length and assigns them to the correct places -void BPTree::set_node_metadata(unsigned int open_idx, std::string name, double length) { +// This takes a label and a length and assigns them to the correct places +void BPTree::set_node_metadata(unsigned int open_idx, std::string name, + double length){ names[open_idx] = name; lengths[open_idx] = length; } -std::vector BPTree::get_structure() { +std::vector BPTree::get_structure(){ return structure; } -std::vector BPTree::get_openclose() { +std::vector BPTree::get_openclose(){ return openclose; } diff --git a/src/tree.h b/src/tree.h index 331d47069..05b116e61 100644 --- a/src/tree.h +++ b/src/tree.h @@ -1,11 +1,11 @@ /* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ +* BSD 3-Clause License +* +* Copyright (c) 2016-2021, UniFrac development team. +* All rights reserved. +* +* See LICENSE file for more details +*/ #ifndef __FAITH_TREE_H #define __FAITH_TREE_H 1 @@ -19,121 +19,121 @@ #include namespace su { - class BPTree { - public: - /* tracked attributes */ - std::vector lengths; - std::vector names; - - /* total number of parentheses */ - uint32_t nparens; - - /* constructor from a defined topology - * - * @param input_structure A boolean vector defining the topology - * @param input_lengths A vector of double of the branch lengths - * @param input_names A vector of str of the vertex names - */ - BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names); - - /* constructor from a TreeSummarizedExperiment - * - * @param treeSE An R treeSE object - */ - BPTree(const Rcpp::List & rowTree); - - ~BPTree(); - - /* postorder tree traversal - * - * Get the index position of the ith node in a postorder tree - * traversal. - * - * @param i The ith node in a postorder traversal - */ - uint32_t postorderselect(uint32_t i)const ; - - /* preorder tree traversal - * - * Get the index position of the ith node in a preorder tree - * traversal. - * - * @param i The ith node in a preorder traversal - */ - uint32_t preorderselect(uint32_t i) const; - - /* Test if the node at an index position is a leaf - * - * @param i The node to evaluate - */ - bool isleaf(uint32_t i) const; - - /* Get the left child of a node - * - * @param i The node to obtain the left child from - */ - uint32_t leftchild(uint32_t i) const ; - - /* Get the right child of a node - * - * @param i The node to obtain the right child from - */ - uint32_t rightchild(uint32_t i) const; - - /* Get the right sibling of a node - * - * @param i The node to obtain the right sibling from - */ - uint32_t rightsibling(uint32_t i) const; - - /* Get the parent of a node - * - * @param i The node to obtain the parent of - */ - int32_t parent(uint32_t i) const; - - /* get the names at the tips of the tree */ - std::unordered_set get_tip_names(); - - /* public getters */ - std::vector get_structure(); - std::vector get_openclose(); - - /* serialize the structure as a sequence of 1s and 0s */ - void print() { - for(auto c = structure.begin(); c != structure.end(); c++) { - if(*c) - std::cout << "1"; - else - std::cout << "0"; - } - std::cout << std::endl; - } - - BPTree mask(std::vector topology_mask, std::vector in_lengths); // mask self - - BPTree shear(std::unordered_set to_keep); - - BPTree collapse(); - - private: - std::vector structure; // the topology - std::vector openclose; // cache'd mapping between parentheses - std::vector select_0_index; // cache of select 0 - std::vector select_1_index; // cache of select 1 - std::vector excess; - - void index_and_cache(); // construct the select caches - void rowTree_to_bp(const Rcpp::List & phylo); // convert ape tree structure to boolean structure - void rowTree_to_metadata(const Rcpp::List & phylo); // assign attributes - void newick_to_metadata(std::string newick); // convert newick to attributes - void structure_to_openclose(); // set the cache mapping between parentheses pairs - void set_node_metadata(unsigned int open_idx, std::string label, double length); // set attributes for a node - inline uint32_t open(uint32_t i) const; // obtain the index of the opening for a given parenthesis - inline uint32_t close(uint32_t i) const; // obtain the index of the closing for a given parenthesis - - int32_t bwd(uint32_t i, int32_t d) const; - int32_t enclose(uint32_t i) const; +class BPTree { + public: + /* Tracked attributes */ + std::vector lengths; + std::vector names; + + /* Total number of parentheses */ + uint32_t nparens; + + /* constructor from a defined topology + * + * @param input_structure A boolean vector defining the topology + * @param input_lengths A vector of double of the branch lengths + * @param input_names A vector of str of the vertex names + */ + BPTree(std::vector input_structure, + std::vector input_lengths, + std::vector input_names); + + /* Constructor from a TreeSummarizedExperiment + * + * @param treeSE An R treeSE object + */ + BPTree(const Rcpp::List & rowTree); + + ~BPTree(); + + /* Postorder tree traversal + * + * Get the index position of the ith node in a postorder tree + * traversal. + * + * @param i The ith node in a postorder traversal + */ + uint32_t postorderselect(uint32_t i)const ; + + /* Preorder tree traversal + * + * Get the index position of the ith node in a preorder tree + * traversal. + * + * @param i The ith node in a preorder traversal + */ + uint32_t preorderselect(uint32_t i) const; + + /* Test if the node at an index position is a leaf + * + * @param i The node to evaluate + */ + bool isleaf(uint32_t i) const; + + /* Get the left child of a node + * + * @param i The node to obtain the left child from + */ + uint32_t leftchild(uint32_t i) const ; + + /* Get the right child of a node + * + * @param i The node to obtain the right child from + */ + uint32_t rightchild(uint32_t i) const; + + /* Get the right sibling of a node + * + * @param i The node to obtain the right sibling from + */ + uint32_t rightsibling(uint32_t i) const; + + /* Get the parent of a node + * + * @param i The node to obtain the parent of + */ + int32_t parent(uint32_t i) const; + + /* Get the names at the tips of the tree */ + std::unordered_set get_tip_names(); + + /* Public getters */ + std::vector get_structure(); + std::vector get_openclose(); + + // Mask self + BPTree mask(std::vector topology_mask, + std::vector in_lengths); + + BPTree shear(std::unordered_set to_keep); + + BPTree collapse(); + + private: + std::vector structure; // The topology + std::vector openclose; // Cache'd mapping b/w parentheses + std::vector select_0_index; // Cache of select 0 + std::vector select_1_index; // Cache of select 1 + std::vector excess; + + /* Construct the select caches */ + void index_and_cache(); + /* Convert ape tree structure to boolean structure */ + void rowTree_to_bp(const Rcpp::List & phylo); + /* Assign attributes */ + void rowTree_to_metadata(const Rcpp::List & phylo); + /* Set the cache mapping between parentheses pairs */ + void structure_to_openclose(); + /* Set attributes for a node */ + void set_node_metadata(unsigned int open_idx, + std::string label, double length); + /* Obtain the index of the opening for a given parenthesis */ + inline uint32_t open(uint32_t i) const; + /* Obtain the index of the closing for a given parenthesis */ + inline uint32_t close(uint32_t i) const; + + int32_t bwd(uint32_t i, int32_t d) const; + int32_t enclose(uint32_t i) const; }; } From 76ed7f2e0a06084f035e7bcaebb8729f13d64d24 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 17 Mar 2025 12:08:52 +0200 Subject: [PATCH 20/48] Save re-sorted edges directly to tree --- R/estimateDiversity.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/estimateDiversity.R b/R/estimateDiversity.R index 7d99c97e6..b6c38a087 100644 --- a/R/estimateDiversity.R +++ b/R/estimateDiversity.R @@ -136,10 +136,10 @@ NULL mat[ is.na(mat) ] <- 0 # The tree must be in cladewise order for the algorithm to work correctly - temp <- reorder.phylo(tree, "cladewise") + tree <- reorder.phylo(tree, "cladewise") # Call the C++ code - return(.faith_cpp(mat, temp)) + return(.faith_cpp(mat, tree)) } .calc_log_modulo_skewness <- function(mat, quantile = 0.5, From 43526d9fb6a24cdad771486ba636edd2f212b3f7 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 17 Mar 2025 13:31:32 +0200 Subject: [PATCH 21/48] Modify C++ code to work without colnames --- src/assay.cpp | 7 +------ src/assay.h | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/assay.cpp b/src/assay.cpp index bf8254290..c98dd0efe 100644 --- a/src/assay.cpp +++ b/src/assay.cpp @@ -24,21 +24,16 @@ Assay::Assay(const Rcpp::NumericMatrix & assay){ sample_ids = std::vector(); obs_ids = std::vector(); - Rcpp::StringVector colnames = Rcpp::colnames(table); - sample_ids = Rcpp::as>(colnames); - Rcpp::StringVector rownames = Rcpp::rownames(table); obs_ids = Rcpp::as>(rownames); - n_samples = sample_ids.size(); + n_samples = table.ncol(); n_obs = obs_ids.size(); /* Define a mapping between an ID and its corresponding offset */ obs_id_index = std::unordered_map(); - sample_id_index = std::unordered_map(); create_id_index(obs_ids, obs_id_index); - create_id_index(sample_ids, sample_id_index); sample_counts = get_sample_counts(); } diff --git a/src/assay.h b/src/assay.h index 36ae632d9..89b50c262 100644 --- a/src/assay.h +++ b/src/assay.h @@ -56,7 +56,6 @@ class Assay { * axis are defined */ std::unordered_map obs_id_index; - std::unordered_map sample_id_index; /* Create an index mapping an ID to its corresponding index * position. From 991e3afcb40ea9391fa8beb488e0c5763b2b8c3d Mon Sep 17 00:00:00 2001 From: TuomasBorman Date: Mon, 17 Mar 2025 16:13:37 +0200 Subject: [PATCH 22/48] Update docs --- DESCRIPTION | 2 +- R/RcppExports.R | 29 --------------- man/dot-faith_cpp.Rd | 40 --------------------- src/faith_R.cpp | 86 ++++++++++++++++++++++---------------------- 4 files changed, 44 insertions(+), 113 deletions(-) delete mode 100644 man/dot-faith_cpp.Rd diff --git a/DESCRIPTION b/DESCRIPTION index dc76c4907..4a204a3ad 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: mia Type: Package -Version: 1.15.32 +Version: 1.15.33 Authors@R: c(person(given = "Tuomas", family = "Borman", role = c("aut", "cre"), email = "tuomas.v.borman@utu.fi", diff --git a/R/RcppExports.R b/R/RcppExports.R index a26903934..250858021 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -1,35 +1,6 @@ # Generated by using Rcpp::compileAttributes() -> do not edit by hand # Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 -#' Calculate Faith's PD -#' -#' This function calculates Faith's phylogenetic diversity for a given assay -#' and rowTree, using a C++ implementation of the Stacked Faith's Phylogenetic -#' Diversity (SFPhD) algorithm. -#' -#' @details -#' This function makes several assumptions about the contents of -#' \code{assay} and \code{rowTree}, namely that: -#' \itemize{ -#' \item \code{assay} and \code{rowTree} are both non-empty. -#' \item \code{assay} has row and column names. -#' \item \code{rowTree}'s nodes are arranged in cladewise order. -#' } -#' These checks should all be handled in the surrounding R code. -#' -#' The values returned by this function are equivalent to the values returned -#' by \code{picante::pd()} with the parameter \code{include.root=TRUE}. -#' -#' The C++ code was adapted from an implementation by the Unifrac team -#' (Armstrong et al. 2021), which is licensed under the BSD 3-Clause license. -#' -#' @param assay An R numeric matrix containing the assay of a \code{TreeSE} -#' object. -#' @param rowTree An \code{ape::phylo} object containing the rowTree of a -#' \code{TreeSE} object. -#' @return A vector containing Faith's PD values. -#' -#' @keywords internal .faith_cpp <- function(assay, rowTree) { .Call('_mia_faith_cpp', PACKAGE = 'mia', assay, rowTree) } diff --git a/man/dot-faith_cpp.Rd b/man/dot-faith_cpp.Rd deleted file mode 100644 index da3323ed2..000000000 --- a/man/dot-faith_cpp.Rd +++ /dev/null @@ -1,40 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/RcppExports.R -\name{.faith_cpp} -\alias{.faith_cpp} -\title{Calculate Faith's PD} -\usage{ -.faith_cpp(assay, rowTree) -} -\arguments{ -\item{assay}{An R numeric matrix containing the assay of a \code{TreeSE} -object.} - -\item{rowTree}{An \code{ape::phylo} object containing the rowTree of a -\code{TreeSE} object.} -} -\value{ -A vector containing Faith's PD values. -} -\description{ -This function calculates Faith's phylogenetic diversity for a given assay -and rowTree, using a C++ implementation of the Stacked Faith's Phylogenetic -Diversity (SFPhD) algorithm. -} -\details{ -This function makes several assumptions about the contents of -\code{assay} and \code{rowTree}, namely that: -\itemize{ -\item \code{assay} and \code{rowTree} are both non-empty. -\item \code{assay} has row and column names. -\item \code{rowTree}'s nodes are arranged in cladewise order. -} -These checks should all be handled in the surrounding R code. - -The values returned by this function are equivalent to the values returned -by \code{picante::pd()} with the parameter \code{include.root=TRUE}. - -The C++ code was adapted from an implementation by the Unifrac team -(Armstrong et al. 2021), which is licensed under the BSD 3-Clause license. -} -\keyword{internal} diff --git a/src/faith_R.cpp b/src/faith_R.cpp index 0d6835757..8640ea14b 100644 --- a/src/faith_R.cpp +++ b/src/faith_R.cpp @@ -16,79 +16,79 @@ #include "tree.h" #include "propmap.h" -//' Calculate Faith's PD -//' -//' This function calculates Faith's phylogenetic diversity for a given assay -//' and rowTree, using a C++ implementation of the Stacked Faith's Phylogenetic -//' Diversity (SFPhD) algorithm. -//' -//' @details -//' This function makes several assumptions about the contents of -//' \code{assay} and \code{rowTree}, namely that: -//' \itemize{ -//' \item \code{assay} and \code{rowTree} are both non-empty. -//' \item \code{assay} has row and column names. -//' \item \code{rowTree}'s nodes are arranged in cladewise order. -//' } -//' These checks should all be handled in the surrounding R code. -//' -//' The values returned by this function are equivalent to the values returned -//' by \code{picante::pd()} with the parameter \code{include.root=TRUE}. -//' -//' The C++ code was adapted from an implementation by the Unifrac team -//' (Armstrong et al. 2021), which is licensed under the BSD 3-Clause license. -//' -//' @param assay An R numeric matrix containing the assay of a \code{TreeSE} -//' object. -//' @param rowTree An \code{ape::phylo} object containing the rowTree of a -//' \code{TreeSE} object. -//' @return A vector containing Faith's PD values. -//' -//' @keywords internal +// Calculate Faith's PD +// +// This function calculates Faith's phylogenetic diversity for a given assay +// and rowTree, using a C++ implementation of the Stacked Faith's Phylogenetic +// Diversity (SFPhD) algorithm. +// +// @details +// This function makes several assumptions about the contents of +// \code{assay} and \code{rowTree}, namely that: +// \itemize{ +// \item \code{assay} and \code{rowTree} are both non-empty. +// \item \code{assay} has row and column names. +// \item \code{rowTree}'s nodes are arranged in cladewise order. +// } +// These checks should all be handled in the surrounding R code. +// +// The values returned by this function are equivalent to the values returned +// by \code{picante::pd()} with the parameter \code{include.root=TRUE}. +// +// The C++ code was adapted from an implementation by the Unifrac team +// (Armstrong et al. 2021), which is licensed under the BSD 3-Clause license. +// +// @param assay An R numeric matrix containing the assay of a \code{TreeSE} +// object. +// @param rowTree An \code{ape::phylo} object containing the rowTree of a +// \code{TreeSE} object. +// @return A vector containing Faith's PD values. +// +// @keywords internal // [[Rcpp::export(.faith_cpp)]] Rcpp::NumericVector faith_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree){ - su::BPTree tree = su::BPTree(rowTree); + su::BPTree tree = su::BPTree(rowTree); su::Assay table = su::Assay(assay); - + std::unordered_set to_keep(table.obs_ids.begin(), table.obs_ids.end()); - + su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - + su::PropMap propmap(table.n_samples); - + uint32_t node; std::vector node_proportions; double length; - + std::vector results = std::vector(table.n_samples, 0.0); - + // For node in postorderselect const unsigned int max_k = (tree_sheared.nparens>1) ? ((tree_sheared.nparens / 2) - 1) : 0; - + for( unsigned int k = 0; k < max_k; k++ ){ node = tree_sheared.postorderselect(k); - + // Get branch length length = tree_sheared.lengths[node]; - + // Get node proportions and set intermediate scores node_proportions = set_proportions(tree_sheared, node, table, propmap, false); - + for( unsigned int sample = 0; sample < table.n_samples; sample++ ){ // Calculate contribution of node to score results[sample] += (node_proportions[sample] > 0) * length; } } - + Rcpp::NumericVector faith = Rcpp::NumericVector(results.size()); - + for( unsigned int i = 0; i < results.size(); i++ ){ faith[i] = results[i]; } - + return faith; } From 82149ee09e63025bbd19305cfc95b8f4004dceff Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 17 Mar 2025 16:42:31 +0200 Subject: [PATCH 23/48] Add name to DESCRIPTION --- DESCRIPTION | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index dc76c4907..e2e8a77cb 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -42,7 +42,8 @@ Authors@R: person(given = "Rajesh", family = "Shigdel", role=c("ctb")), person(given = "Katariina", family = "Pärnänen", role=c("ctb")), person(given = "Pande", family = "Erawijantari", role=c("ctb")), - person(given = "Danielle", family = "Callan", role=c("ctb"))) + person(given = "Danielle", family = "Callan", role=c("ctb")), + person(given = "Jesse", family = "Pasanen", role=c("ctb"))) Title: Microbiome analysis Description: mia implements tools for microbiome analysis based on the From 24af8101076dadd5d84b1df28d4944af7a3ab798 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Fri, 26 Sep 2025 05:12:11 +0300 Subject: [PATCH 24/48] WIP --- src/propmap.cpp | 2 +- src/stripemap.cpp | 49 +++ src/stripemap.h | 35 ++ src/unifrac.cpp | 648 ++++++++++++++++++++++++++++++++++++ src/unifrac.hpp | 142 ++++++++ src/unifrac_R.cpp | 168 ++++++++++ src/unifrac_task.cpp | 770 +++++++++++++++++++++++++++++++++++++++++++ src/unifrac_task.hpp | 573 ++++++++++++++++++++++++++++++++ 8 files changed, 2386 insertions(+), 1 deletion(-) create mode 100644 src/stripemap.cpp create mode 100644 src/stripemap.h create mode 100644 src/unifrac.cpp create mode 100644 src/unifrac.hpp create mode 100644 src/unifrac_R.cpp create mode 100644 src/unifrac_task.cpp create mode 100644 src/unifrac_task.hpp diff --git a/src/propmap.cpp b/src/propmap.cpp index ce02cfa1e..d08e75f03 100644 --- a/src/propmap.cpp +++ b/src/propmap.cpp @@ -85,4 +85,4 @@ std::vector su::set_proportions(const BPTree &tree, ps.update(node, props); return(props); -} +} \ No newline at end of file diff --git a/src/stripemap.cpp b/src/stripemap.cpp new file mode 100644 index 000000000..940fd3682 --- /dev/null +++ b/src/stripemap.cpp @@ -0,0 +1,49 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include "tree.h" +#include "assay.h" +#include "stripemap.h" + +#include + +using namespace su; + +StripeMap::StripeMap(uint32_t n_samples) + : stripe_map(), + vecsize(n_samples) +{ + n_stripes = (n_samples + 1) / 2; + for( unsigned int i = 0; i < n_stripes; i++ ){ + this->update(i, std::vector(vecsize, 0.0)); + } +} + +StripeMap::~StripeMap() { +} + +std::vector StripeMap::get(uint32_t i){ + if( stripe_map.count(i) > 0 ){ + return stripe_map.at(i); + } else { + return(std::vector()); + } +} + +void StripeMap::clear(uint32_t i){ + stripe_map[i] = std::vector(); +} + +void StripeMap::update(uint32_t node, std::vector vec){ + stripe_map[node] = vec; +} + +bool StripeMap::is_empty(uint32_t i){ + return get(i).empty(); +} \ No newline at end of file diff --git a/src/stripemap.h b/src/stripemap.h new file mode 100644 index 000000000..1ebc647e0 --- /dev/null +++ b/src/stripemap.h @@ -0,0 +1,35 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#ifndef __FAITH_PROPMAP +#define __FAITH_PROPMAP 1 + +#include +#include +#include + +namespace su { +class StripeMap { +public: + StripeMap(uint32_t n_samples); + virtual ~StripeMap(); + void clear(uint32_t i); + void update(uint32_t i, std::vector vec); + std::vector get(uint32_t i); + bool is_empty(uint32_t i); + +private: + std::unordered_map> stripe_map; + uint32_t vecsize; // Size of stripe vectors is always the number of samples + uint32_t n_stripes; +}; + +} + +#endif /* __FAITH_PROPMAP */ diff --git a/src/unifrac.cpp b/src/unifrac.cpp new file mode 100644 index 000000000..9a5c8e70b --- /dev/null +++ b/src/unifrac.cpp @@ -0,0 +1,648 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include "unifrac.hpp" +#include "propmap.hpp" +#include "stripemap.hpp" +#include "tree.hpp" + +#include "biom_interface.hpp" +#include "affinity.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "unifrac_internal.hpp" + +void su::process_stripes(const su::Assay & table, + const su::BPTree & tree_sheared, + Method method, + bool variance_adjust, + su::StripeMap dm_stripes, + su::StripeMap dm_stripes_total, + su::task_parameters task) { + if(variance_adjust) + su::unifrac_vaw( + std::ref(table), + std::ref(tree_sheared), + method, + std::ref(dm_stripes), + std::ref(dm_stripes_total), + &tasks[tid]); + else + switch(method) { + case su::unweighted: + //A templated function... + unifracTT(table, tree, true, dm_stripes, dm_stripes_total, task); + break; + default: + fprintf(stderr, "Unknown unifrac task\n"); + exit(1); + break; + } +} + + +template +inline void unifracTT(const su::Assay & table, + const su::BPTree & tree, + const bool want_total, + su::StripeMap dm_stripes, + su::StripeMap dm_stripes_total, + const su::task_parameters & task_p) { + + const unsigned int n_samples = task_p->n_samples; + + //What is this used for? + //UNIFRAC_BLOCK = 16 + //const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up + const uint64_t n_samples_r = ((n_samples + 15)/16)*16; // round up + + + //How does this differ from normal propstack? + //Basically a vector of several fixed-size propstacks? + //Can probably be replaced with our current impl. + + //su::PropStackMulti propstack_multi(table.n_samples); + su::PropMap propmap(table.n_samples); + + su::StripeMap stripemap(table.n_samples); + su::StripeMap stripemap_total(table.n_samples); + + //Not sure how relevant these are, but let's keep them for now + const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; + + //Initialize a task object with the given memory locations and parameters + //PResumably we want to pass the stripes by reference? + //Eh, just add a function to get them + TaskT taskObj(dm_stripes, dm_stripes_total, max_emb, task_p); + + std::vector lengths(max_emb) + + //Algorithm portion follows + + /* + * The values in the example vectors correspond to index positions of an + * element in the resulting distance matrix. So, in the example below, + * the following can be interpreted: + * + * [0 1 2] + * [1 2 3] + * + * As comparing the sample for row 0 against the sample for col 1, the + * sample for row 1 against the sample for col 2, the sample for row 2 + * against the sample for col 3. + * + * In other words, we're computing stripes of a distance matrix. In the + * following example, we're computing over 6 samples requiring 3 + * stripes. + * + * A; stripe == 0 + * [0 1 2 3 4 5] + * [1 2 3 4 5 0] + * + * B; stripe == 1 + * [0 1 2 3 4 5] + * [2 3 4 5 0 1] + * + * C; stripe == 2 + * [0 1 2 3 4 5] + * [3 4 5 0 1 2] + * + * The stripes end up computing the following positions in the distance + * matrix. + * + * x A B C x x + * x x A B C x + * x x x A B C + * C x x x A B + * B C x x x A + * A B C x x x + * + * However, we store those stripes as vectors, ie + * [ A A A A A A ] + * + * We end up performing N / 2 redundant calculations on the last stripe + * (see C) but that is small over large N. + */ + + unsigned int k = 0; // index in tree + const unsigned int max_k = (tree.nparens / 2) - 1; + + while (k node_proportions = propmap.get(node); + + node_proportions = su::set_proportions(tree, node, table, propmap); + + if(task_p.bypass_tips && tree.isleaf(node)) + continue; + + lengths[filled_emb] = tree.lengths[node]; + filled_emb++; + + taskObj.embed_proportions(node_proportions, my_filled_emb); + my_filled_emb++; + } + + k=my_k; + + taskObj._run(filled_emb,lengths); + filled_emb=0; + } + + //I suppose want_total is used if you want the results as a percentage of the total? + if(want_total) { + const uint64_t start_idx = task_p->start; + const uint64_t stop_idx = task_p->stop; + + double * const dm_stripes_buf = taskObj.dm_stripes.buf; + const double * const dm_stripes_total_buf = taskObj.dm_stripes_total.buf; + + for(uint64_t i = start_idx; i < stop_idx; i++) + for(uint64_t j = 0; j < n_samples; j++) { + uint64_t idx = (i-start_idx)*n_samples_r+j; + dm_stripes_buf[idx]=dm_stripes_buf[idx]/dm_stripes_total_buf[idx]; + } + + } +} + +su::mat_t su::stripes_to_condensed_form(su::StripeMap stripes, + uint32_t n, + su::mat_t result, + unsigned int start, + unsigned int stop) { + // n must be >= 2, but that should be enforced upstream as that would imply + // computing unifrac on a single sample. + + std::vector cf = result.condensed_form; + + uint64_t comb_N = comb_2(n); + for(unsigned int stripe = start; stripe < stop; stripe++) { + // compute the (i, j) position of each element in each stripe + uint64_t i = 0; + uint64_t j = stripe + 1; + for(uint64_t k = 0; k < n; k++, i++, j++) { + if(j == n) { + i = 0; + j = n - (stripe + 1); + } + // determine the position in the condensed form vector for a given (i, j) + // based off of + // https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.squareform.html + uint64_t comb_N_minus_i = comb_2(n - i); + cf[comb_N - comb_N_minus_i + (j - i - 1)] = stripes.get(stripe)[k]; + } + } + result.condensed_form = cf; + return result; +} + +double** su::deconvolute_stripes(std::vector &stripes, uint32_t n) { + // would be better to just do striped_to_condensed_form + double **dm; + dm = (double**)malloc(sizeof(double*) * n); + if(dm == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(double*) * n, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + for(unsigned int i = 0; i < n; i++) { + dm[i] = (double*)malloc(sizeof(double) * n); + if(dm[i] == NULL) { + fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", + sizeof(double) * n, __FILE__, __LINE__); + exit(EXIT_FAILURE); + } + dm[i][i] = 0; + } + + for(unsigned int i = 0; i < stripes.size(); i++) { + double *vec = stripes[i]; + unsigned int k = 0; + for(unsigned int row = 0, col = i + 1; row < n; row++, col++) { + if(col < n) { + dm[row][col] = vec[k]; + dm[col][row] = vec[k]; + } else { + dm[col % n][row] = vec[k]; + dm[row][col % n] = vec[k]; + } + k++; + } + } + return dm; +} + + + +// write in a 2D matrix +// also suitable for writing to disk +template +void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d) { + const uint64_t comb_N = su::comb_2(n); + for(uint64_t i = 0; i < n; i++) { + for(uint64_t j = 0; j < n; j++) { + TReal v; + if(i < j) { // upper triangle + const uint64_t comb_N_minus = su::comb_2(n - i); + v = cf[comb_N - comb_N_minus + (j - i - 1)]; + } else if (i > j) { // lower triangle + const uint64_t comb_N_minus = su::comb_2(n - j); + v = cf[comb_N - comb_N_minus + (i - j - 1)]; + } else { + v = 0.0; + } + buf2d[i*n+j] = v; + } + } +} + + +// make sure it is instantiated +template void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); +template void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); + +void su::condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d) { + su::condensed_form_to_matrix_T(cf,n,buf2d); +} + +void su::condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d) { + su::condensed_form_to_matrix_T(cf,n,buf2d); +} + +/* + * The stripes end up computing the following positions in the distance + * matrix. + * + * x A B C x x + * x x A B C x + * x x x A B C + * C x x x A B + * B C x x x A + * A B C x x x + * + * However, we store those stripes as vectors, ie + * [ A A A A A A ] + */ + + +// Helper class +// Will cache pointers and automatically release stripes when all elements are used +class OnceManagedStripes { + private: + const uint32_t n_samples; + const uint32_t n_stripes; + const ManagedStripes &stripes; + std::vector stripe_ptr; + std::vector stripe_accessed; + + const double *get_stripe(const uint32_t stripe) { + if (stripe_ptr[stripe]==0) stripe_ptr[stripe]=stripes.get_stripe(stripe); + return stripe_ptr[stripe]; + } + + void release_stripe(const uint32_t stripe) { + stripes.release_stripe(stripe); + stripe_ptr[stripe]=0; + } + + public: + OnceManagedStripes(const ManagedStripes &_stripes, const uint32_t _n_samples, const uint32_t _n_stripes) + : n_samples(_n_samples), n_stripes(_n_stripes) + , stripes(_stripes) + , stripe_ptr(n_stripes) + , stripe_accessed(n_stripes) + {} + + ~OnceManagedStripes() + { + for(uint32_t i = 0; i < n_stripes; i++) { + if (stripe_ptr[i]!=0) { + release_stripe(i); + } + } + } + + double get_val(const uint32_t stripe, const uint32_t el) + { + if (stripe_ptr[stripe]==0) stripe_ptr[stripe]=stripes.get_stripe(stripe); + const double *mystripe = stripe_ptr[stripe]; + double val = mystripe[el]; + + stripe_accessed[stripe]++; + if (stripe_accessed[stripe]==n_samples) release_stripe(stripe); // we will not use this stripe anymore + + return val; + } + + +}; + +// write in a 2D matrix +// also suitable for writing to disk +template +void su::stripes_to_matrix_T(const ManagedStripes &_stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size) { + // n_samples must be >= 2, but that should be enforced upstream as that would imply + // computing unifrac on a single sample. + + // tile for for better memory access pattern + const uint32_t TILE = (tile_size>0) ? tile_size : (128/sizeof(TReal)); + const uint32_t n_samples_tup = (n_samples+(TILE-1))/TILE; // round up + + OnceManagedStripes stripes(_stripes, n_samples, n_stripes); + + + for(uint32_t oi = 0; oi < n_samples_tup; oi++) { // off diagonal + // alternate between inner and outer off-diagonal, due to wrap around in stripes + const uint32_t o = ((oi%2)==0) ? \ + (oi/2)*TILE : /* close to diagonal */ \ + (n_samples_tup-(oi/2)-1)*TILE; /* far from diagonal */ + + for(uint32_t d = 0; d < (n_samples-o); d+=TILE) { // diagonal + + uint32_t iOut = d; + uint32_t jOut = d+o; + + uint32_t iMax = std::min(iOut+TILE,n_samples); + uint32_t jMax = std::min(jOut+TILE,n_samples); + + + if (iOut==jOut) { + // on diagonal + for(uint64_t i = iOut; i < iMax; i++) { + buf2d[i*n_samples+i] = 0.0; + + int64_t stripe=0; + + uint64_t j = i+1; + for(; (stripen_stripes) { + // ops, we overshoot... roll back + j-=(stripe-n_stripes); + stripe=n_stripes; + } + for(; (stripe(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size); +template void su::stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size); + +void su::stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size) { + return su::stripes_to_matrix_T(stripes, n_samples, n_stripes, buf2d, tile_size); +} + +void su::stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size) { + return su::stripes_to_matrix_T(stripes, n_samples, n_stripes, buf2d, tile_size); +} + + +void progressbar(float progress) { + // from http://stackoverflow.com/a/14539953 + // + // could encapsulate into a classs for displaying time elapsed etc + int barWidth = 70; + std::cout << "["; + int pos = barWidth * progress; + for (int i = 0; i < barWidth; ++i) { + if (i < pos) std::cout << "="; + else if (i == pos) std::cout << ">"; + else std::cout << " "; + } + std::cout << "] " << int(progress * 100.0) << " %\r"; + std::cout.flush(); +} + +// Computes Faith's PD for the samples in `table` over the phylogenetic +// tree given by `tree`. +// Assure that tree does not contain ids that are not in table +void su::faith_pd(biom_interface &table, + BPTree &tree, + double* result) { + PropStack propstack(table.n_samples); + + uint32_t node; + double *node_proportions; + double length; + + // for node in postorderselect + for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { + node = tree.postorderselect(k); + // get branch length + length = tree.lengths[node]; + + // get node proportions and set intermediate scores + node_proportions = propstack.pop(node); + set_proportions(node_proportions, tree, node, table, propstack); + + for (unsigned int sample = 0; sample < table.n_samples; sample++){ + // calculate contribution of node to score + result[sample] += (node_proportions[sample] > 0) * length; + } + } +} + + +#ifdef UNIFRAC_ENABLE_ACC + +// test only once, then use persistent value +static int proc_use_acc = -1; + +inline bool use_acc() { + if (proc_use_acc!=-1) return (proc_use_acc!=0); + int has_nvidia_gpu_rc = access("/proc/driver/nvidia/gpus", F_OK); + + bool print_info = false; + + if (const char* env_p = std::getenv("UNIFRAC_GPU_INFO")) { + print_info = true; + std::string env_s(env_p); + if ((env_s=="NO") || (env_s=="N") || (env_s=="no") || (env_s=="n") || + (env_s=="NEVER") || (env_s=="never")) { + print_info = false; + } + } + + + if (has_nvidia_gpu_rc != 0) { + if (print_info) printf("INFO (unifrac): GPU not found, using CPU\n"); + proc_use_acc=0; + return false; + } + + if (const char* env_p = std::getenv("UNIFRAC_USE_GPU")) { + std::string env_s(env_p); + if ((env_s=="NO") || (env_s=="N") || (env_s=="no") || (env_s=="n") || + (env_s=="NEVER") || (env_s=="never")) { + if (print_info) printf("INFO (unifrac): Use of GPU explicitly disabled, using CPU\n"); + proc_use_acc=0; + return false; + } + } + + if (print_info) printf("INFO (unifrac): Using GPU\n"); + proc_use_acc=1; + return true; +} +#endif + +void su::unifrac(biom_interface &table, + BPTree &tree, + Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const su::task_parameters* task_p) { +#ifdef UNIFRAC_ENABLE_ACC + if (use_acc()) { + su_acc::unifrac(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); + } else { +#else + if (true) { +#endif + su_cpu::unifrac(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); + } +} + + +void su::unifrac_vaw(biom_interface &table, + BPTree &tree, + Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const su::task_parameters* task_p) { +#ifdef UNIFRAC_ENABLE_ACC + if (use_acc()) { + su_acc::unifrac_vaw(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); + } else { +#else + if (true) { +#endif + su_cpu::unifrac_vaw(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); + } +} + + +void su::process_stripes(biom_interface &table, + BPTree &tree_sheared, + Method method, + bool variance_adjust, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + std::vector &threads, + std::vector &tasks) { + + // register a signal handler so we can ask the master thread for its + // progress + register_report_status(); + + // cannot use threading with openacc or openmp + for(unsigned int tid = 0; tid < threads.size(); tid++) { + if(variance_adjust) + su::unifrac_vaw( + std::ref(table), + std::ref(tree_sheared), + method, + std::ref(dm_stripes), + std::ref(dm_stripes_total), + &tasks[tid]); + else + su::unifrac( + std::ref(table), + std::ref(tree_sheared), + method, + std::ref(dm_stripes), + std::ref(dm_stripes_total), + &tasks[tid]); + } + + remove_report_status(); +} diff --git a/src/unifrac.hpp b/src/unifrac.hpp new file mode 100644 index 000000000..83167d8d8 --- /dev/null +++ b/src/unifrac.hpp @@ -0,0 +1,142 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include +#include +#include +#include +#include + +#ifndef __UNIFRAC + +#include "task_parameters.hpp" +#include "biom_interface.hpp" + + namespace su { + + typedef struct mat { + unsigned int n_samples; + unsigned int cf_size; + bool is_upper_triangle; + std::vector condensed_form; + std::vector sample_ids; + } mat_t; + + // process the stripes described by tasks + void process_stripes(const su::Assay & table, + const su::BPTree & tree_sheared, + Method method, + bool variance_adjust, + su::StripeMap dm_stripes, + su::StripeMap dm_stripes_total, + su::task_parameters task); + + // Stripes to condensed form for the results + su::mat_t stripes_to_condensed_form(su::StripeMap stripes, + uint32_t n, + su::mat_t result, + unsigned int start, + unsigned int stop); + + // Works the vectors + template + inline void unifracTT(const su::Assay & table, + const su::BPTree & tree, + const bool want_total, + su::StripeMap dm_stripes, + su::StripeMap dm_stripes_total, + const su::task_parameters & task_p); + + inline uint64_t comb_2(uint64_t N) { + // based off of _comb_int_long + // https://github.com/scipy/scipy/blob/v0.19.1/scipy/special/_comb.pyx + + // Compute binom(N, k) for integers. + // + // we're disregarding overflow as that practically should not + // happen unless the number of samples processed is in excess + // of 4 billion + uint64_t val, j, M, nterms; + uint64_t k = 2; + + M = N + 1; + nterms = k < (N - k) ? k : N - k; + + val = 1; + + for(j = 1; j < nterms + 1; j++) { + val *= M - j; + val /= j; + } + return val; + } + + template + inline void unifracTT(const su::biom_interface &table, + const su::BPTree &tree, + const bool want_total, + su::StripeMap dm_stripes, + su::StripeMap dm_stripes_total, + const su::task_parameters & task_p) + + enum Method {unweighted, + weighted_normalized, + weighted_unnormalized, + generalized}; + + void unifrac(biom_interface &table, + BPTree &tree, + Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const task_parameters* task_p); + + void unifrac_vaw(biom_interface &table, + BPTree &tree, + Method unifrac_method, + std::vector &dm_stripes, + std::vector &dm_stripes_total, + const task_parameters* task_p); + + double** deconvolute_stripes(std::vector &stripes, uint32_t n); + + class ManagedStripes { + public: + virtual ~ManagedStripes() {} + virtual const double *get_stripe(uint32_t stripe) const = 0; + virtual void release_stripe(uint32_t stripe) const = 0; + }; + + class MemoryStripes : public ManagedStripes { + private: + const double * const * stripes; // just a pointer, not owned + public: + MemoryStripes(const double * const * _stripes) : stripes(_stripes) {} + MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} + MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} + MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} + MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} + + virtual const double *get_stripe(uint32_t stripe) const {return stripes[stripe];} + virtual void release_stripe(uint32_t stripe) const {}; + }; + + // tile_size==0 means memory optimized + template void stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size=0); + void stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size=0); + void stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size=0); + + + template void condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d); + void condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); + void condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); + + } +#define __UNIFRAC 1 +#endif diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp new file mode 100644 index 000000000..3e4ef89ce --- /dev/null +++ b/src/unifrac_R.cpp @@ -0,0 +1,168 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include +#include + +#include + +#include "assay.h" +#include "tree.h" +#include "propmap.h" +#include "stripemap.h" +#include "unifrac.hpp" +#include "unifrac_task.hpp" + +// Calculate Unifrac +// +// @keywords internal +// [[Rcpp::export(.unifrac_cpp)]] +Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, + const Rcpp::List & rowTree){ + // + // std::unordered_set to_keep(table.obs_ids.begin(), + // table.obs_ids.end()); + // + // su::BPTree tree_sheared = tree.shear(to_keep).collapse(); + // + // su::PropMap propmap(table.n_samples); + // + // uint32_t node; + // std::vector node_proportions; + // double length; + // + // std::vector results = std::vector(table.n_samples, 0.0); + // + // + // // For node in postorderselect + // const unsigned int max_k = (tree_sheared.nparens>1) ? + // ((tree_sheared.nparens / 2) - 1) : 0; + // + // for( unsigned int k = 0; k < max_k; k++ ){ + // node = tree_sheared.postorderselect(k); + // + // // Get branch length + // length = tree_sheared.lengths[node]; + // + // // Get node proportions and set intermediate scores + // node_proportions = set_proportions(tree_sheared, node, table, propmap, + // false); + // + // for( unsigned int sample = 0; sample < table.n_samples; sample++ ){ + // // Calculate contribution of node to score + // results[sample] += (node_proportions[sample] > 0) * length; + // } + // } + + su::BPTree tree = su::BPTree(rowTree); + su::Assay table = su::Assay(assay); + std::string method = "unweighted"; + + su::mat_t results = one_off(table, tree, method, false, 1.0, false); + + //condensed_form is the main values, returned in result + //Sample_ids can be handled with a map? + //n_samples, cf_size, is_upper_triangle are single values that can be passed in some other way? + + /* + Rcpp::NumericVector unifrac = Rcpp::NumericVector(results.size()); + + for( unsigned int i = 0; i < results.cf_size; i++ ){ + unifrac[i] = results.condensed_form[i]; + } + */ + + return Rcpp::List::create(Rcpp::Named("n_samples") = results.n_samples, + Rcpp::Named("is_upper_triangle") = results.is_upper_triangle, + Rcpp::Named("cf_size") = results.cf_size, + Rcpp::Named("c_form") = results.condensed_form); +} + + + + + + +/* Compute UniFrac - condensed form + * + * biom_filename the filename to the biom table. + * tree_filename the filename to the correspodning tree. + * unifrac_method the requested unifrac method. + * variance_adjust whether to apply variance adjustment. + * alpha GUniFrac alpha, only relevant if method == generalized. + * bypass_tips disregard tips, reduces compute by about 50% + * threads the number of threads to use. + * result the resulting distance matrix in condensed form, this is initialized within the method so using ** + * + * one_off returns the following error codes: + * + * okay : no problems encountered + * table_missing : the filename for the table does not exist + * tree_missing : the filename for the tree does not exist + * unknown_method : the requested method is unknown. + * table_empty : the table does not have any entries + */ + +su::mat_t one_off(const su::Assay & table, + const su::BPTree & tree, + std::string unifrac_method, + double alpha, + bool variance_adjust, + bool bypass_tips) { + + //Check that method is valid - pass it as something other than string? + //SET_METHOD(unifrac_method, unknown_method) + + //Stripes relate to matrix calculations + const unsigned int stripe_stop = (table.n_samples + 1) / 2; + + //Originally std::vector of double pointers - this is where the data travels? + su::StripeMap dm_stripes(table.n_samples); + su::StripeMap dm_stripes_total(table.n_samples); + + su::task_parameters task; + + task.tid = 0; + task.start = 0; + task.stop = stripe_stop; + task.bypass_tips = bypass_tips; + task.n_samples = n_samples; + task.g_unifrac_alpha = alpha; + + //Main action + //Calls either unifrac or _vaw depending on variance_adjust + //makes use of std::ref? + //Versions for accelerated and cpu - let's go with cpu for now + //method is "unweighted" by default, let's start with that and see what else may be needed + + //Threads are passed here after being created with a vector + //This does nothing except pass the number of threads, however + su::process_stripes(table, tree_sheared, method, variance_adjust, + dm_stripes, dm_stripes_total, task); + + + //Only use of threading in this version of code was for stripes to condensed form + //Basically each thread calls stripes_to_condensed_form + //Which is just a bunch of binomial calculations + + su::mat_t result; + result->n_samples = table.n_samples; + result->cf_size = su::comb_2(table.n_samples); + result->sample_ids = table.sample_ids; + result->condensed_form = std::vector(su::comb_2(table.n_samples), + 0.0); + result->is_upper_triangle = true; + + return su::stripes_to_condensed_form(dm_stripes, + table.n_samples, + result, + task.start, + task.stop); +} + diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp new file mode 100644 index 000000000..7cd8188e4 --- /dev/null +++ b/src/unifrac_task.cpp @@ -0,0 +1,770 @@ + +#include "unifrac_task.hpp" + +#include +#include + + +void su::UnifracUnweightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { + + //Parameter finding + + //Task parameters determine stuff + const uint64_t start_idx = this->task_p->start; + const uint64_t stop_idx = this->task_p->stop; + const uint64_t n_samples = this->task_p->n_samples; + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + + + // openacc only works well with local variables + const uint64_t * const __restrict__ embedded_proportions = this->embedded_proportions; + TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; + TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; + + TFloat * const __restrict__ sums = this->sums; + + const uint64_t step_size = SUCMP_NM::UnifracUnweightedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + + const uint64_t filled_embs_els = filled_embs/64; + const uint64_t filled_embs_rem = filled_embs%64; + + const uint64_t filled_embs_els_round = (filled_embs+63)/64; + + + + // pre-compute sums of length elements, since they are likely to be accessed many times + // We will use a 8-bit map, to keep it small enough to keep in L1 cache + for (uint64_t emb_el=0; emb_el psum = &(sums[emb8<<8]); + const TFloat * __restrict__ pl = &(lengths[emb8*8]); + + // compute all the combinations for this block (8-bits total) + // psum[0] = 0.0 // +0*pl[0]+0*pl[1]+0*pl[2]+... + // psum[1] = pl[0] // +0*pl[1]+0*pl[2]+... + // psum[2] = pl[1] // +0*pl[0]+0*pl[2]+ + // psum[2] = pl[0] + pl[1] + // ... + // psum[255] = pl[1] +.. + pl[7] // + 0*pl[0] + // psum[255] = pl[0] +pl[1] +.. + pl[7] + for (uint64_t b8_i=0; b8_i<0x100; b8_i++) { + psum[b8_i] = (((b8_i >> 0) & 1) * pl[0]) + (((b8_i >> 1) & 1) * pl[1]) + + (((b8_i >> 2) & 1) * pl[2]) + (((b8_i >> 3) & 1) * pl[3]) + + (((b8_i >> 4) & 1) * pl[4]) + (((b8_i >> 5) & 1) * pl[5]) + + (((b8_i >> 6) & 1) * pl[6]) + (((b8_i >> 7) & 1) * pl[7]); + } + } + } + + + + if (filled_embs_rem>0) { // add also the overflow elements + const uint64_t emb_el=filled_embs_els; + for (uint64_t sub8=0; sub8<8; sub8++) { + // we are summing we have enough buffer in sums + const uint64_t emb8 = emb_el*8+sub8; + TFloat * __restrict__ psum = &(sums[emb8<<8]); + + // compute all the combinations for this block, set to 0 any past the limit + // as above + for (uint64_t b8_i=0; b8_i<0x100; b8_i++) { + TFloat val= 0; + for (uint64_t li=(emb8*8); li> (li-(emb8*8))) & 1) * lengths[li]; + } + psum[b8_i] = val; + } + } + } + + // point of thread + for(uint64_t sk = 0; sk < sample_steps ; sk++) { + for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t ik = 0; ik < step_size ; ik++) { + const uint64_t k = sk*step_size + ik; + const uint64_t idx = (stripe-start_idx) * n_samples_r; + TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; + TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; + //TFloat *dm_stripe = dm_stripes[stripe]; + //TFloat *dm_stripe_total = dm_stripes_total[stripe]; + + if (k>=n_samples) continue; // past the limit + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + bool did_update = false; + TFloat my_stripe = 0.0; + TFloat my_stripe_total = 0.0; + + + //This is the main calculation phase + + for (uint64_t emb_el=0; emb_el> 8) & 0xff)] + + psum[0x200+((x1 >> 16) & 0xff)] + + psum[0x300+((x1 >> 24) & 0xff)] + + psum[0x400+((x1 >> 32) & 0xff)] + + psum[0x500+((x1 >> 40) & 0xff)] + + psum[0x600+((x1 >> 48) & 0xff)] + + psum[0x700+((x1 >> 56) )]; + my_stripe_total += psum[ (o1 & 0xff)] + + psum[0x100+((o1 >> 8) & 0xff)] + + psum[0x200+((o1 >> 16) & 0xff)] + + psum[0x300+((o1 >> 24) & 0xff)] + + psum[0x400+((o1 >> 32) & 0xff)] + + psum[0x500+((o1 >> 40) & 0xff)] + + psum[0x600+((o1 >> 48) & 0xff)] + + psum[0x700+((o1 >> 56) )]; + } + } + + if (did_update) { + dm_stripe[k] += my_stripe; + dm_stripe_total[k] += my_stripe_total; + } + } + } + } +} + +// +// +// template +// void SUCMP_NM::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { +// const uint64_t start_idx = this->task_p->start; +// const uint64_t stop_idx = this->task_p->stop; +// const uint64_t n_samples = this->task_p->n_samples; +// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; +// +// // openacc only works well with local variables +// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; +// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; +// +// bool * const __restrict__ zcheck = this->zcheck; +// TFloat * const __restrict__ sums = this->sums; +// +// const uint64_t step_size = SUCMP_NM::UnifracUnnormalizedWeightedTask::step_size; +// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up +// +// // check for zero values and pre-compute single column sums +// #ifdef _OPENACC +// #pragma acc parallel loop present(embedded_proportions,lengths,zcheck,sums) +// #else +// #pragma omp parallel for default(shared) +// #endif +// for(uint64_t k=0; k::acc_vector_size; +// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,lengths,zcheck,sums) async +// #else +// // use dynamic scheduling due to non-homogeneity in the loop +// #pragma omp parallel for default(shared) schedule(dynamic,1) +// #endif +// for(uint64_t sk = 0; sk < sample_steps ; sk++) { +// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { +// for(uint64_t ik = 0; ik < step_size ; ik++) { +// const uint64_t k = sk*step_size + ik; +// +// if (k>=n_samples) continue; // past the limit +// +// const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here +// +// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound +// +// const bool allzero_k = zcheck[k]; +// const bool allzero_l1 = zcheck[l1]; +// +// if (allzero_k && allzero_l1) { +// // nothing to do, would have to add 0 +// } else { +// TFloat my_stripe; +// +// if (allzero_k || allzero_l1) { +// // one side has all zeros +// // we can use the distributed property, and use the pre-computed values +// +// const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 +// k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 +// +// // keep reads in the same place to maximize GPU warp performance +// my_stripe = sums[ridx]; +// +// } else { +// // both sides non zero, use the explicit but slow approach +// my_stripe = 0.0; +// +// #pragma acc loop seq +// for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); +// #endif +// } +// +// template +// void SUCMP_NM::UnifracVawUnnormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { +// const uint64_t start_idx = this->task_p->start; +// const uint64_t stop_idx = this->task_p->stop; +// const uint64_t n_samples = this->task_p->n_samples; +// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; +// +// // openacc only works well with local variables +// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; +// const TFloat * const __restrict__ embedded_counts = this->embedded_counts; +// const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; +// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; +// +// const uint64_t step_size = SUCMP_NM::UnifracVawUnnormalizedWeightedTask::step_size; +// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up +// +// // point of thread +// #ifdef _OPENACC +// const unsigned int acc_vector_size = SUCMP_NM::UnifracVawUnnormalizedWeightedTask::acc_vector_size; +// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,lengths) async +// #else +// #pragma omp parallel for default(shared) schedule(dynamic,1) +// #endif +// for(uint64_t sk = 0; sk < sample_steps ; sk++) { +// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { +// for(uint64_t ik = 0; ik < step_size ; ik++) { +// const uint64_t k = sk*step_size + ik; +// const uint64_t idx = (stripe-start_idx) * n_samples_r; +// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; +// //TFloat *dm_stripe = dm_stripes[stripe]; +// +// if (k>=n_samples) continue; // past the limit +// +// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound +// +// const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; +// +// TFloat my_stripe = dm_stripe[k]; +// +// #pragma acc loop seq +// for (uint64_t emb=0; emb 0) { +// TFloat u1 = embedded_proportions[offset + k]; +// TFloat v1 = embedded_proportions[offset + l1]; +// TFloat diff1 = fabs(u1 - v1); +// TFloat length = lengths[emb]; +// +// my_stripe += (diff1 * length) / vaw; +// } +// } +// +// dm_stripe[k] = my_stripe; +// } +// +// } +// } +// +// #ifdef _OPENACC +// // next iteration will use the alternative space +// std::swap(this->embedded_proportions,this->embedded_proportions_alt); +// #endif +// } +// +// template +// void SUCMP_NM::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { +// const uint64_t start_idx = this->task_p->start; +// const uint64_t stop_idx = this->task_p->stop; +// const uint64_t n_samples = this->task_p->n_samples; +// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; +// +// // openacc only works well with local variables +// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; +// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; +// TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; +// +// bool * const __restrict__ zcheck = this->zcheck; +// TFloat * const __restrict__ sums = this->sums; +// +// const uint64_t step_size = SUCMP_NM::UnifracNormalizedWeightedTask::step_size; +// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up +// +// // check for zero values and pre-compute single column sums +// #ifdef _OPENACC +// #pragma acc parallel loop present(embedded_proportions,lengths,zcheck,sums) +// #else +// #pragma omp parallel for default(shared) +// #endif +// for(uint64_t k=0; k::acc_vector_size; +// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,lengths,zcheck,sums) async +// #else +// // use dynamic scheduling due to non-homogeneity in the loop +// #pragma omp parallel for schedule(dynamic,1) default(shared) +// #endif +// for(uint64_t sk = 0; sk < sample_steps ; sk++) { +// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { +// for(uint64_t ik = 0; ik < step_size ; ik++) { +// const uint64_t k = sk*step_size + ik; +// +// if (k>=n_samples) continue; // past the limit +// +// const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here +// +// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound +// +// const bool allzero_k = zcheck[k]; +// const bool allzero_l1 = zcheck[l1]; +// +// if (allzero_k && allzero_l1) { +// // nothing to do, would have to add 0 +// } else { +// const uint64_t idx = (stripe-start_idx) * n_samples_r; +// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; +// TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; +// //TFloat *dm_stripe = dm_stripes[stripe]; +// //TFloat *dm_stripe_total = dm_stripes_total[stripe]; +// +// // the totals can always use the distributed property +// dm_stripe_total[k] += sums[k] + sums[l1]; +// +// TFloat my_stripe; +// +// if (allzero_k || allzero_l1) { +// // one side has all zeros +// // we can use the distributed property, and use the pre-computed values +// +// const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 +// k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 +// +// // keep reads in the same place to maximize GPU warp performance +// my_stripe = sums[ridx]; +// +// } else { +// // both sides non zero, use the explicit but slow approach +// +// my_stripe = 0.0; +// +// #pragma acc loop seq +// for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); +// #endif +// } +// +// template +// void SUCMP_NM::UnifracVawNormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { +// const uint64_t start_idx = this->task_p->start; +// const uint64_t stop_idx = this->task_p->stop; +// const uint64_t n_samples = this->task_p->n_samples; +// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; +// +// // openacc only works well with local variables +// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; +// const TFloat * const __restrict__ embedded_counts = this->embedded_counts; +// const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; +// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; +// TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; +// +// const uint64_t step_size = SUCMP_NM::UnifracVawNormalizedWeightedTask::step_size; +// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up +// +// // point of thread +// #ifdef _OPENACC +// const unsigned int acc_vector_size = SUCMP_NM::UnifracVawNormalizedWeightedTask::acc_vector_size; +// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async +// #else +// #pragma omp parallel for schedule(dynamic,1) default(shared) +// #endif +// for(uint64_t sk = 0; sk < sample_steps ; sk++) { +// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { +// for(uint64_t ik = 0; ik < step_size ; ik++) { +// const uint64_t k = sk*step_size + ik; +// const uint64_t idx = (stripe-start_idx) * n_samples_r; +// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; +// TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; +// //TFloat *dm_stripe = dm_stripes[stripe]; +// //TFloat *dm_stripe_total = dm_stripes_total[stripe]; +// +// if (k>=n_samples) continue; // past the limit +// +// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound +// +// const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; +// +// TFloat my_stripe = dm_stripe[k]; +// TFloat my_stripe_total = dm_stripe_total[k]; +// +// #pragma acc loop seq +// for (uint64_t emb=0; emb 0) { +// TFloat u1 = embedded_proportions[offset + k]; +// TFloat v1 = embedded_proportions[offset + l1]; +// TFloat diff1 = fabs(u1 - v1); +// TFloat length = lengths[emb]; +// +// my_stripe += (diff1 * length) / vaw; +// my_stripe_total += ((u1 + v1) * length) / vaw; +// } +// } +// +// dm_stripe[k] = my_stripe; +// dm_stripe_total[k] = my_stripe_total; +// +// } +// +// } +// } +// +// #ifdef _OPENACC +// // next iteration will use the alternative space +// std::swap(this->embedded_proportions,this->embedded_proportions_alt); +// #endif +// } +// +// template +// void SUCMP_NM::UnifracGeneralizedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { +// const uint64_t start_idx = this->task_p->start; +// const uint64_t stop_idx = this->task_p->stop; +// const uint64_t n_samples = this->task_p->n_samples; +// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; +// +// // openacc only works well with local variables +// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; +// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; +// TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; +// +// const TFloat g_unifrac_alpha = this->task_p->g_unifrac_alpha; +// +// const uint64_t step_size = SUCMP_NM::UnifracGeneralizedTask::step_size; +// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up +// +// // point of thread +// #ifdef _OPENACC +// const unsigned int acc_vector_size = SUCMP_NM::UnifracGeneralizedTask::acc_vector_size; +// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,lengths) async +// #else +// #pragma omp parallel for schedule(dynamic,1) default(shared) +// #endif +// for(uint64_t sk = 0; sk < sample_steps ; sk++) { +// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { +// for(uint64_t ik = 0; ik < step_size ; ik++) { +// const uint64_t k = sk*step_size + ik; +// const uint64_t idx = (stripe-start_idx) * n_samples_r; +// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; +// TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; +// //TFloat *dm_stripe = dm_stripes[stripe]; +// //TFloat *dm_stripe_total = dm_stripes_total[stripe]; +// +// if (k>=n_samples) continue; // past the limit +// +// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound +// +// TFloat my_stripe = dm_stripe[k]; +// TFloat my_stripe_total = dm_stripe_total[k]; +// +// #pragma acc loop seq +// for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); +// #endif +// } +// +// template +// void SUCMP_NM::UnifracVawGeneralizedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { +// const uint64_t start_idx = this->task_p->start; +// const uint64_t stop_idx = this->task_p->stop; +// const uint64_t n_samples = this->task_p->n_samples; +// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; +// +// const TFloat g_unifrac_alpha = this->task_p->g_unifrac_alpha; +// +// // openacc only works well with local variables +// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; +// const TFloat * const __restrict__ embedded_counts = this->embedded_counts; +// const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; +// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; +// TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; +// +// const uint64_t step_size = SUCMP_NM::UnifracVawGeneralizedTask::step_size; +// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up +// // quick hack, to be finished +// +// // point of thread +// #ifdef _OPENACC +// const unsigned int acc_vector_size = SUCMP_NM::UnifracVawGeneralizedTask::acc_vector_size; +// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async +// #else +// #pragma omp parallel for schedule(dynamic,1) default(shared) +// #endif +// for(uint64_t sk = 0; sk < sample_steps ; sk++) { +// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { +// for(uint64_t ik = 0; ik < step_size ; ik++) { +// const uint64_t k = sk*step_size + ik; +// const uint64_t idx = (stripe-start_idx) * n_samples_r; +// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; +// TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; +// //TFloat *dm_stripe = dm_stripes[stripe]; +// //TFloat *dm_stripe_total = dm_stripes_total[stripe]; +// +// if (k>=n_samples) continue; // past the limit +// +// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound +// +// const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; +// +// TFloat my_stripe = dm_stripe[k]; +// TFloat my_stripe_total = dm_stripe_total[k]; +// +// #pragma acc loop seq +// for (uint64_t emb=0; emb 0) { +// TFloat u1 = embedded_proportions[offset + k]; +// TFloat v1 = embedded_proportions[offset + l1]; +// TFloat length = lengths[emb]; +// +// TFloat sum1 = (u1 + v1) / vaw; +// TFloat sub1 = fabs(u1 - v1) / vaw; +// TFloat sum_pow1 = pow(sum1, g_unifrac_alpha) * length; +// +// my_stripe += sum_pow1 * (sub1 / sum1); +// my_stripe_total += sum_pow1; +// } +// } +// +// dm_stripe[k] = my_stripe; +// dm_stripe_total[k] = my_stripe_total; +// +// } +// } +// } +// +// #ifdef _OPENACC +// // next iteration will use the alternative space +// std::swap(this->embedded_proportions,this->embedded_proportions_alt); +// #endif +// } +// +// template +// void SUCMP_NM::UnifracVawUnweightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { +// const uint64_t start_idx = this->task_p->start; +// const uint64_t stop_idx = this->task_p->stop; +// const uint64_t n_samples = this->task_p->n_samples; +// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; +// +// // openacc only works well with local variables +// const uint32_t * const __restrict__ embedded_proportions = this->embedded_proportions; +// const TFloat * const __restrict__ embedded_counts = this->embedded_counts; +// const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; +// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; +// TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; +// +// const uint64_t step_size = SUCMP_NM::UnifracVawUnweightedTask::step_size; +// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up +// +// const uint64_t filled_embs_els = (filled_embs+31)/32; // round up +// +// // point of thread +// #ifdef _OPENACC +// const unsigned int acc_vector_size = SUCMP_NM::UnifracVawUnweightedTask::acc_vector_size; +// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async +// #else +// #pragma omp parallel for schedule(dynamic,1) default(shared) +// #endif +// for(uint64_t sk = 0; sk < sample_steps ; sk++) { +// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { +// for(uint64_t ik = 0; ik < step_size ; ik++) { +// const uint64_t k = sk*step_size + ik; +// const uint64_t idx = (stripe-start_idx) * n_samples_r; +// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; +// TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; +// //TFloat *dm_stripe = dm_stripes[stripe]; +// //TFloat *dm_stripe_total = dm_stripes_total[stripe]; +// +// if (k>=n_samples) continue; // past the limit +// +// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound +// +// TFloat my_stripe = dm_stripe[k]; +// TFloat my_stripe_total = dm_stripe_total[k]; +// +// const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; +// +// #pragma acc loop seq +// for (uint64_t emb_el=0; emb_el 0) { +// TFloat length = lengths[emb]; +// TFloat lv1 = length / vaw; +// +// my_stripe += ((x1 >> ei) & 1)*lv1; +// my_stripe_total += ((o1 >> ei) & 1)*lv1; +// } +// } +// } +// } +// +// dm_stripe[k] = my_stripe; +// dm_stripe_total[k] = my_stripe_total; +// +// } +// +// } +// } +// +// #ifdef _OPENACC +// // next iteration will use the alternative space +// std::swap(this->embedded_proportions,this->embedded_proportions_alt); +// #endif +// } +// diff --git a/src/unifrac_task.hpp b/src/unifrac_task.hpp new file mode 100644 index 000000000..df094171e --- /dev/null +++ b/src/unifrac_task.hpp @@ -0,0 +1,573 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ + +#include "task_parameters.hpp" +#include "stripemap.h" + +/** REMOVE **/ +#include +#include +#include +#include +#include +#include + + +#ifndef __UNIFRAC_TASKS +#define __UNIFRAC_TASKS 1 +// CPUs don't need such a big alignment +#define UNIFRAC_BLOCK 16 +#endif + + +namespace su { + + /* task specific compute parameters + * + * n_samples the number of samples being processed + * start the first stripe to process + * stop the last stripe to process + * tid the thread identifier + * bypass_tips ignore tips on compute, reduces compute by ~50% + * g_unifrac_alpha an alpha value for generalized unifrac + */ + struct task_parameters { + uint32_t n_samples; // number of samples + unsigned int start; // starting stripe + unsigned int stop; // stopping stripe + unsigned int tid; // thread ID + bool bypass_tips; // avoid compute at tips + + // task specific arguments below + double g_unifrac_alpha; // generalized unifrac alpha + }; + + + + /* + Task parameters - struct of parameters + UnifracTaskVector - vector with special things + dm_stripes: Vector of vectors: Replace with stripemap + task_p + + + */ + + // Note: This adds a copy, which is suboptimal + // But was the easiest way to get a contiguous buffer + // And it does allow for fp32 compute, when desired + + //Seems to have a block of unused stuff at the front? + //Accessed via the + class UnifracTaskVector { + private: + su::StripeMap dm_stripes; + const su::task_parameters task_p; + + public: + const unsigned int start_idx; + const unsigned int n_samples; + const uint64_t n_samples_r; + std::vector buf; + + UnifracTaskVector(su::StripeMap _dm_stripes, const su::task_parameters _task_p) + : dm_stripes(_dm_stripes), task_p(_task_p) + , start_idx(task_p->start), n_samples(task_p->n_samples) + , n_samples_r(((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK) // round up + //buf is just a new array with as many stripes as called for in task_p + //n_samples_r tells us how many unifrac_blocks are required for n_samples. + //Originally this was a null comparison, we might need to check what it does specifically + , buf((dm_stripes.is_empty(start_idx)) ? + std::vector() : + std::vector(n_samples_r*(task_p->stop-start_idx), 0.0)) // dm_stripes could be null, in which case keep it null + { + if (!buf.empty()) { + for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { + std::vector dm_stripe = dm_stripes.get(stripe); + //This just returns the specified section of buf, and copies the values from dm_stripe there + std::vector buf_stripe = this->operator[](stripe); + //double * buf_stripe = this->operator[](stripe); + for(unsigned int j=0; j& operator[](unsigned int idx) { return buf+((idx-start_idx)*n_samples_r);} + const double * operator[](unsigned int idx) const { return buf+((idx-start_idx)*n_samples_r);} + + //Destructor copies the buffer values back into dm_stripe + ~UnifracTaskVector() + { + double * const ibuf = buf; + if (ibuf != NULL) { + for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { + std::vector dm_stripe = dm_stripes[stripe]; + double * buf_stripe = this->operator[](stripe); + dm_stripes.update(stripe, vec) + } + } + } + + private: + UnifracTaskVector() = delete; + UnifracTaskVector operator=(const UnifracTaskVector&other) const = delete; + }; + + + /***********************************************/ + + + // Base task class to be shared by all tasks + template + class UnifracTaskBase { + public: + //Two taskvectors for stripes and total + UnifracTaskVector dm_stripes; + UnifracTaskVector dm_stripes_total; + + su::task_parameters task_p; + + const unsigned int max_embs; + std::vector embedded_proportions; + + UnifracTaskBase(su::StripeMap _dm_stripes, su::StripeMap _dm_stripes_total, + unsigned int _max_embs, su::task_parameters _task_p) + : dm_stripes(_dm_stripes,_task_p), dm_stripes_total(_dm_stripes_total,_task_p), task_p(_task_p) + , max_embs(_max_embs)) + { + uint64_t bsize = dm_stripes.n_samples_r * get_emb_els(max_embs); + embedded_proportions = std::vector(bsize, 0.0) + } + + virtual ~UnifracTaskBase() {} + + static unsigned int get_emb_els(unsigned int max_embs); + + //Need to return a vector? + void embed_proportions_range(std::vector in, unsigned int start, unsigned int end, unsigned int emb); + void embed_proportions(std::vector in, unsigned int emb) {embed_proportions_range(in,0,dm_stripes.n_samples,emb);} + + // + // ===== Internal, do not use directly ======= + // + + // Just copy from one buffer to another + // May convert between fp formats in the process (if TOut!=double) + + //out has all the stripes? + //in has just a specific section? + std::vector embed_proportions_range_straight( + std::vector out, + std::vector in, + unsigned int start, + unsigned int end, + unsigned int emb) const + { + const unsigned int n_samples = dm_stripes.n_samples; + const uint64_t n_samples_r = dm_stripes.n_samples_r; + const uint64_t offset = emb * n_samples_r; + + //Copy to stripe indicated by emb + //Stripes are all contained in in/out in one mass + //Start/end aren't necessarily the whole stripe? + for(unsigned int i = start; i < end; i++) { + out[offset + i] = in[i-start]; + } + + if (end==n_samples) { + // avoid NaNs + for(unsigned int i = n_samples; i < n_samples_r; i++) { + out[offset + i] = 0.0; + } + } + return out; + } + + + // packed bool + // Compute (in[:]>0) on each element, and store only the boolean bit. + // The output values are stored in a multi-byte format, one bit per emb index, + // so it will likely take multiple passes to store all the values + // + // Note: assumes we are processing emb in increasing order, starting from 0 + template void embed_proportions_range_bool( + std::vector out, + std::vector in, + unsigned int start, + unsigned int end, + unsigned int emb) const + { + + const unsigned int n_packed = sizeof(TOut)*8;// e.g. 32 for unit32_t + const unsigned int n_samples = dm_stripes.n_samples; + const uint64_t n_samples_r = dm_stripes.n_samples_r; + // The output values are stored in a multi-byte format, one bit per emb index + // Compute the element to store the bit into, as well as whichbit in that element + unsigned int emb_block = emb/n_packed; // beginning of the element block + unsigned int emb_bit = emb%n_packed; // bit inside the elements + const uint64_t offset = emb_block * n_samples_r; + + if (emb_bit==0) { + // assign for emb_bit==0, so it clears the other bits + // assumes we processing emb in increasing order, starting from 0 + for(unsigned int i = start; i < end; i++) { + out[offset + i] = (in[i-start] > 0); + } + + if (end==n_samples) { + // avoid NaNs + for(unsigned int i = n_samples; i < n_samples_r; i++) { + out[offset + i] = 0; + } + } + } else { + // just update my bit + for(unsigned int i = start; i < end; i++) { + out[offset + i] |= (TOut(in[i-start] > 0) << emb_bit); + } + + // the rest of the els are already OK + } + } + }; + + // straight embeded_proportions + template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_straight(embedded_proportions,in,start,end,emb);} + template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return max_embs;} + + //packed bool embeded_proportions + template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} + template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+31)/32;} + + template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} + template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+63)/64;} + + + + + + + /***********************************************/ + + /* void unifrac tasks + * + * all methods utilize the same function signature. that signature is as follows: + * + * dm_stripes vector the stripes of the distance matrix being accumulated + * into for unique branch length + * dm_stripes vector the stripes of the distance matrix being accumulated + * into for total branch length (e.g., to normalize unweighted unifrac) + * embedded_proportions the proportions vector for a sample, or rather + * the counts vector normalized to 1. this vector is embedded as it is + * duplicated: if A, B and C are proportions for features A, B, and C, the + * vector will look like [A B C A B C]. + * length the branch length of the current node to its parent. + * task_p task specific parameters. + */ + + template + class UnifracTask : public UnifracTaskBase { + protected: + // Use one cache line on CPU + // On GPU, sharing a cache line is actually a good thing + static const unsigned int step_size = 16*4/sizeof(double); + + public: + + UnifracTask(su::StripeMap _dm_stripes, su::StripeMap _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) + : UnifracTaskBase(_dm_stripes, _dm_stripes_total, _max_embs, _task_p) {} + + virtual ~UnifracTask() {} + + //Probably should return a vector? + virtual void run(unsigned int filled_embs, std::vector length) = 0; + + protected: + static const unsigned int RECOMMENDED_MAX_EMBS_STRAIGHT = 128-16; // a little less to leave a bit of space of maxed-out L1 + // packed uses 32x less memory,so this should be 32x larger than straight... but there are additional structures, so use half of that + static const unsigned int RECOMMENDED_MAX_EMBS_BOOL = 64*32; + + }; + + /***********************************************/ + + + //Simplify all template stuff into doubles + + class UnifracUnweightedTask : public UnifracTask { + public: + static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_BOOL; + + // Note: _max_emb MUST be multiple of 64 + UnifracUnweightedTask(su::StripeMap _dm_stripes, su::StripeMap _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) + : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) + { + const unsigned int bsize = _max_embs*32; + sums = std::vector(bsize, 0.0); + } + + virtual ~UnifracUnweightedTask() {} + + virtual void run(unsigned int filled_embs, std::vector length) {_run(filled_embs, length);} + + void _run(unsigned int filled_embs, std::vector length); + private: + std::vector sums; // temp buffer + }; + + + + + + + + + /***********************************************/ + + +// template +// class UnifracUnnormalizedWeightedTask : public UnifracTask { +// public: +// static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; +// +// UnifracUnnormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) +// : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) +// { +// const unsigned int n_samples = this->task_p->n_samples; +// +// zcheck = NULL; +// sums = NULL; +// posix_memalign((void **)&zcheck, 4096, sizeof(bool) * n_samples); +// posix_memalign((void **)&sums , 4096, sizeof(TFloat) * n_samples); +// #pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) +// } +// +// virtual ~UnifracUnnormalizedWeightedTask() +// { +// free(sums); +// free(zcheck); +// } +// +// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} +// +// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); +// protected: +// // temp buffers +// bool *zcheck; +// TFloat *sums; +// }; +// +// /***********************************************/ +// +// template +// class UnifracNormalizedWeightedTask : public UnifracTask { +// public: +// static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; +// +// UnifracNormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) +// : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) +// { +// const unsigned int n_samples = this->task_p->n_samples; +// +// zcheck = NULL; +// sums = NULL; +// posix_memalign((void **)&zcheck, 4096, sizeof(bool) * n_samples); +// posix_memalign((void **)&sums , 4096, sizeof(TFloat) * n_samples); +// #pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) +// } +// +// virtual ~UnifracNormalizedWeightedTask() +// { +// free(sums); +// free(zcheck); +// } +// +// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} +// +// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); +// protected: +// // temp buffers +// bool *zcheck; +// TFloat *sums; +// }; +// +// +// /***********************************************/ +// +// +// template +// class UnifracGeneralizedTask : public UnifracTask { +// public: +// static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; +// +// UnifracGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) +// : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) {} +// +// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} +// +// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); +// }; +// +// /* void unifrac_vaw tasks +// * +// * all methods utilize the same function signature. that signature is as follows: +// * +// * dm_stripes vector the stripes of the distance matrix being accumulated +// * into for unique branch length +// * dm_stripes vector the stripes of the distance matrix being accumulated +// * into for total branch length (e.g., to normalize unweighted unifrac) +// * embedded_proportions the proportions vector for a sample, or rather +// * the counts vector normalized to 1. this vector is embedded as it is +// * duplicated: if A, B and C are proportions for features A, B, and C, the +// * vector will look like [A B C A B C]. +// * embedded_counts the counts vector embedded in the same way and order as +// * embedded_proportions. the values of this array are unnormalized feature +// * counts for the subtree. +// * sample_total_counts the total unnormalized feature counts for all samples +// * embedded in the same way and order as embedded_proportions. +// * length the branch length of the current node to its parent. +// * task_p task specific parameters. +// */ +// template +// class UnifracVawTask : public UnifracTaskBase { +// protected: +// #ifdef _OPENACC +// // The parallel nature of GPUs needs a largish step +// #ifndef SMALLGPU +// // default to larger step, which makes a big difference for bigger GPUs like V100 +// static const unsigned int step_size = 32; +// // keep the vector size just big enough to keep the used emb array inside the 32k buffer +// static const unsigned int acc_vector_size = 32*32*8/sizeof(TFloat); +// #else +// // smaller GPUs prefer a slightly smaller step +// static const unsigned int step_size = 16; +// // keep the vector size just big enough to keep the used emb array inside the 32k buffer +// static const unsigned int acc_vector_size = 32*32*8/sizeof(TFloat); +// #endif +// #else +// // The serial nature of CPU cores prefers a small step +// static const unsigned int step_size = 4; +// #endif +// +// public: +// TFloat * const embedded_counts; +// const TFloat * const sample_total_counts; +// +// static const unsigned int RECOMMENDED_MAX_EMBS = 128; +// +// UnifracVawTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, +// const TFloat * _sample_total_counts, +// unsigned int _max_embs, const su::task_parameters* _task_p) +// : UnifracTaskBase(_dm_stripes, _dm_stripes_total, _max_embs, _task_p) +// , embedded_counts(UnifracTaskBase::initialize_embedded(this->dm_stripes.n_samples_r,_max_embs)), sample_total_counts(_sample_total_counts) {} +// +// +// /* delete +// UnifracVawTask(UnifracTaskBase &baseObj, +// const TEmb * _embedded_proportions, const TFloat * _sample_total_counts, unsigned int _max_embs) +// : UnifracTaskBase(baseObj) +// , embedded_proportions(_embedded_proportions), embedded_counts(initialize_embedded()), sample_total_counts(_sample_total_counts), max_embs(_max_embs) {} +// */ +// +// +// virtual ~UnifracVawTask() {} +// +// void sync_embedded_counts(unsigned int filled_embs) +// { +// #ifdef _OPENACC +// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; +// const uint64_t bsize = n_samples_r * filled_embs; +// #pragma acc update device(embedded_counts[:bsize]) +// #endif +// } +// +// void sync_embedded(unsigned int filled_embs) { this->sync_embedded_proportions(filled_embs); this->sync_embedded_counts(filled_embs);} +// +// void embed_range(const TFloat* __restrict__ in_proportions, const TFloat* __restrict__ in_counts, unsigned int start, unsigned int end, unsigned int emb) { +// this->embed_proportions_range(in_proportions,start,end,emb); +// this->embed_proportions_range_straight(this->embedded_counts,in_counts,start,end,emb); +// } +// void embed(const TFloat* __restrict__ in_proportions, const double* __restrict__ in_counts, unsigned int emb) { embed_range(in_proportions,in_counts,0,this->dm_stripes.n_samples,emb);} +// +// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) = 0; +// }; +// +// /***********************************************/ +// +// +// template +// class UnifracVawUnnormalizedWeightedTask : public UnifracVawTask { +// public: +// UnifracVawUnnormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, +// const TFloat * _sample_total_counts, +// unsigned int _max_embs, const su::task_parameters* _task_p) +// : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} +// +// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} +// +// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); +// }; +// +// /***********************************************/ +// +// template +// class UnifracVawNormalizedWeightedTask : public UnifracVawTask { +// public: +// UnifracVawNormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, +// const TFloat * _sample_total_counts, +// unsigned int _max_embs, const su::task_parameters* _task_p) +// : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} +// +// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} +// +// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); +// }; +// +// /***********************************************/ +// +// template +// class UnifracVawUnweightedTask : public UnifracVawTask { +// public: +// UnifracVawUnweightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, +// const TFloat * _sample_total_counts, +// unsigned int _max_embs, const su::task_parameters* _task_p) +// : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} +// +// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} +// +// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); +// }; +// +// /***********************************************/ +// +// template +// class UnifracVawGeneralizedTask : public UnifracVawTask { + public: + UnifracVawGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, + const TFloat * _sample_total_counts, + unsigned int _max_embs, const su::task_parameters* _task_p) + : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} + + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} + + void _run(unsigned int filled_embs, const TFloat * __restrict__ length); + }; + + +} + +#endif From d3b3fbdade959a1804afa670c3d835eeaaeb4907 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 22 Apr 2026 16:42:10 +0300 Subject: [PATCH 25/48] Initialise exporters to raw, qiime and mothur --- NAMESPACE | 5 + R/AllGenerics.R | 15 +++ R/exporters.R | 251 ++++++++++++++++++++++++++++++++++++++++++ man/export-methods.Rd | 98 +++++++++++++++++ 4 files changed, 369 insertions(+) create mode 100644 R/exporters.R create mode 100644 man/export-methods.Rd diff --git a/NAMESPACE b/NAMESPACE index b0f050187..b0e9e6210 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -57,6 +57,9 @@ export(estimateDominance) export(estimateEvenness) export(estimateFaith) export(estimateRichness) +export(exportMothur) +export(exportQIIME2) +export(exportRaw) export(full_join) export(getAbundanceClass) export(getAbundant) @@ -413,6 +416,8 @@ importFrom(ape,is.binary) importFrom(ape,is.rooted) importFrom(ape,read.tree) importFrom(ape,reorder.phylo) +importFrom(ape,write.FASTA) +importFrom(ape,write.tree) importFrom(bluster,clusterRows) importFrom(decontam,isContaminant) importFrom(decontam,isNotContaminant) diff --git a/R/AllGenerics.R b/R/AllGenerics.R index 241789a42..5ea880f5d 100644 --- a/R/AllGenerics.R +++ b/R/AllGenerics.R @@ -105,6 +105,21 @@ setGeneric("convertToBIOM", signature = c("x"), setGeneric("convertToPhyloseq", signature = c("x"), function(x, ...) standardGeneric("convertToPhyloseq")) +#' @rdname export-methods +#' @export +setGeneric("exportRaw", signature = c("x"), function(x, ...) + standardGeneric("exportRaw")) + +#' @rdname export-methods +#' @export +setGeneric("exportQIIME2", signature = c("x"), function(x, ...) + standardGeneric("exportQIIME2")) + +#' @rdname export-methods +#' @export +setGeneric("exportMothur", signature = c("x"), function(x, ...) + standardGeneric("exportMothur")) + #' @rdname isContaminant #' @export setGeneric("addContaminantQC", signature = c("x"), diff --git a/R/exporters.R b/R/exporters.R new file mode 100644 index 000000000..321e256c3 --- /dev/null +++ b/R/exporters.R @@ -0,0 +1,251 @@ +#' Exporters to common formats for microbiome data outside of R +#' +#' @description +#' There are a few very popular external tools for microbiome analysis, +#' including QIIME2 and mothur. However, R does not currently provide any class +#' to accommodate those data formats. When exporting data from mia to external +#' tools, the best approach is therefore to break a data container into its +#' building blocks (assays, side information, trees, etc.). +#' +#' Thanks to \code{exportRaw}, \code{exportQIIME2} and \code{exportMothur}, +#' it is now possible to export a +#' \code{\link[TreeSummarizedExperiment]{TreeSummarizedExperiment}} object as +#' raw elements or near-ready QIIME2 and mothur formats, respectively. This way, +#' migrating from mia to an external system is still a bad idea, but at least it +#' is fairly straightforward. +#' +#' @param x a \code{\link[TreeSummarizedExperiment]{TreeSummarizedExperiment}} object. +#' +#' @param dpath \code{Character scalar}. +#' +#' @param assay.type \code{Character scalar}. (Default: \code{"counts"}) +#' +#' @param rowdata \code{Character scalar}. (Default: \code{"rowdata"}) +#' +#' @param coldata \code{Character scalar}. (Default: \code{"coldata"}) +#' +#' @param assay.dir \code{Character scalar}. (Default: \code{"assays"}) +#' +#' @param rowtree.dir \code{Character scalar}. (Default: \code{"row_trees"}) +#' +#' @param coltree.dir \code{Character scalar}. (Default: \code{"col_trees"}) +#' +#' @param dimred.dir \code{Character scalar}. (Default: \code{"dim_reds"}) +#' +#' @param altexp.dir \code{Character scalar}. (Default: \code{"alt_exps"}) +#' +#' @param ... Unused. +#' +#' @returns Directory at \code{dpath} with components of \code{x} each stored +#' as a file in the proper format. +#' +#' @details The output directory contains the elements of \code{x}. For +#' \code{exportQIIME2} and \code{exportMothur}, data will need some more +#' processing using the target tool. For some tips, check the rbiom package +#' vignettes on converting data: +#' \url{https://cmmr.github.io/rbiom/articles/convert.html} +#' +#' @examples +#' library(TreeSummarizedExperiment) +#' +#' tse <- makeTSE() +#' assayNames(tse) <- "counts" +#' +#' # Export raw TreeSE components in custom directory +#' exportRaw(tse, "out") +#' +#' # Export TreeSE components in near-ready QIIME2 format +#' exportQIIME2(tse, "qiime2_dir") +#' +#' # Export TreeSE components in near-ready mothur format +#' exportMothur(tse, "mothur_dir") +#' +#' @name export-methods +#' @aliases exportRaw exportQIIME2 exportMothur +NULL + + +#' @rdname export-methods +#' @importFrom ape write.tree +setMethod("exportRaw", signature = c(x = "TreeSummarizedExperiment"), + function(x, dpath, rowdata.file = "rowdata", coldata.file = "coldata", + assay.dir = "assays", rowtree.dir = "row_trees", coltree.dir = "col_trees", + dimred.dir = "dim_reds", altexp.dir = "alt_exps"){ + + if( !endsWith(dpath, "/") ) dpath <- paste0(dpath, "/") + if( !dir.exists(dpath) ) dir.create(dpath) + + write.table(rowData(x), paste0(dpath, rowdata.file, ".tsv"), sep = "\t") + + write.table(colData(x), paste0(dpath, coldata.file, ".tsv"), sep = "\t") + + assay.dir <- .create_slot_dir(x, assays, assay.dir, dpath) + + for( assay_name in assayNames(x) ){ + write.table( + assay(x, assay_name), + paste0(assay.dir, assay_name, ".tsv"), + sep = "\t" + ) + } + + rowtree.dir <- .create_slot_dir(x, rowTreeNames, rowtree.dir, dpath) + + for( tree_name in rowTreeNames(x) ){ + write.tree( + rowTree(x, tree_name), paste0(rowtree.dir, tree_name, ".nwk") + ) + } + + coltree.dir <- .create_slot_dir(x, colTreeNames, coltree.dir, dpath) + + for( tree_name in colTreeNames(x) ){ + write.tree( + colTree(x, tree_name), paste0(coltree.dir, tree_name, ".nwk") + ) + } + + dimred.dir <- .create_slot_dir(x, reducedDims, dimred.dir, dpath) + + for( dimred_name in reducedDimNames(x) ){ + write.table( + reducedDim(x, dimred_name), + paste0(dimred.dir, dimred_name, ".tsv"), + sep = "\t" + ) + } + + altexp.dir <- .create_slot_dir(x, altExps, altexp.dir, dpath) + + for( altexp_name in altExpNames(x) ){ + + altexp_path <- paste0(altexp.dir, altexp_name) + + if( is(altExp(x, altexp_name), "SummarizedExperiment") ){ + + exportToRaw(altExp(x, altexp_name), altexp_path) + + }else{ + + write.table(altExp(x, altexp_name), altexp_path, sep = "\t") + + } + } + invisible(NULL) +}) + + +.create_slot_dir <- function(x, FUN, slot.path, main.path = ""){ + + if( length(FUN(x)) != 0L ){ + slot.path <- paste0(main.path, slot.path) + if( !endsWith(slot.path, "/") ) slot.path <- paste0(slot.path, "/") + dir.create(slot.path) + } + return(slot.path) +} + + +#' @rdname export-methods +#' @importFrom ape write.tree write.FASTA +setMethod("exportQIIME2", signature = c(x = "TreeSummarizedExperiment"), + function(x, dpath, assay.type = "counts", tree.name = "phylo"){ + + if( !endsWith(dpath, "/") ) dpath <- paste0(dpath, "/") + if( !dir.exists(dpath) ) dir.create(dpath) + + row_data <- apply(rowData(x), 1L, paste, collapse = ";_") + row_data <- gsub(";_$", "", row_data) + + row_data <- cbind(names(row_data), row_data, 1L) + colnames(row_data) <- c("Feature ID", "Taxon", "Confidence") + + write.table( + row_data, paste0(dpath, "taxonomy.tsv"), sep = "\t", row.names = FALSE + ) + + col_data <- as.data.frame(colData(x)) + + col_types <- apply( + col_data, 2L, function(col) switch( + type(col), character = "categorical", integer = , double = "numeric") + ) + + col_data <- rbind(col_types, col_data) + col_data <- cbind(`sample-id` = rownames(col_data), col_data) + col_data[1L, "sample-id"] <- "#q2:types" + + write.table( + col_data, paste0(dpath, "metadata.tsv"), sep = "\t", row.names = FALSE + ) + + sel_assay <- assay(x, assay.type) + sel_assay <- cbind(`#OTU ID` = rownames(sel_assay), sel_assay) + + write.table( + sel_assay, paste0(dpath, assay.type, ".tsv"), + sep = "\t", row.names = FALSE + ) + + row_tree <- rowTree(x, tree.name) + + if( !is.null(row_tree) ){ + write.tree(row_tree, paste(dpath, "tree.nwk")) + } + + if( !is.null(referenceSeq(x)) ){ + write.FASTA(referenceSeq(x), paste0(dpath, "seqs.fna")) + } + invisible(NULL) +}) + + +#' @rdname export-methods +#' @importFrom ape write.tree write.FASTA +setMethod("exportMothur", signature = c(x = "TreeSummarizedExperiment"), + function(x, dpath, assay.type = "counts", tree.name = "phylo"){ + + if( !endsWith(dpath, "/") ) dpath <- paste0(dpath, "/") + if( !dir.exists(dpath) ) dir.create(dpath) + + row_data <- apply(rowData(x), 1L, paste, collapse = ";") + row_data <- gsub(";$", "", row_data) + + sel_assay <- assay(x, assay.type) + row_sums <- rowSums(sel_assay) + + row_data <- cbind(names(row_data), row_sums, row_data) + colnames(row_data) <- c("OTU", "Size", "Taxonomy") + + write.table( + row_data, paste0(dpath, "taxonomy.tsv"), sep = "\t", row.names = FALSE + ) + + col_data <- as.data.frame(colData(x)) + col_data <- cbind(group = rownames(col_data), col_data) + + write.table( + col_data, paste0(dpath, "metadata.tsv"), sep = "\t", row.names = FALSE + ) + + sel_assay <- cbind( + `Representative Sequence` = rownames(sel_assay), + total = row_sums, sel_assay + ) + + write.table( + sel_assay, paste0(dpath, assay.type, ".tsv"), + sep = "\t", row.names = FALSE + ) + + row_tree <- rowTree(x, tree.name) + + if( !is.null(row_tree) ){ + write.tree(row_tree, paste(dpath, "tree.nwk")) + } + + if( !is.null(referenceSeq(x)) ){ + write.FASTA(referenceSeq(x), paste0(dpath, "seqs.fna")) + } + invisible(NULL) +}) diff --git a/man/export-methods.Rd b/man/export-methods.Rd new file mode 100644 index 000000000..3d1edd26b --- /dev/null +++ b/man/export-methods.Rd @@ -0,0 +1,98 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/AllGenerics.R, R/exporters.R +\name{exportRaw} +\alias{exportRaw} +\alias{exportQIIME2} +\alias{exportMothur} +\alias{export-methods} +\alias{exportRaw,TreeSummarizedExperiment-method} +\alias{exportQIIME2,TreeSummarizedExperiment-method} +\alias{exportMothur,TreeSummarizedExperiment-method} +\title{Exporters to common formats for microbiome data outside of R} +\usage{ +exportRaw(x, ...) + +exportQIIME2(x, ...) + +exportMothur(x, ...) + +\S4method{exportRaw}{TreeSummarizedExperiment}( + x, + dpath, + rowdata.file = "rowdata", + coldata.file = "coldata", + assay.dir = "assays", + rowtree.dir = "row_trees", + coltree.dir = "col_trees", + dimred.dir = "dim_reds", + altexp.dir = "alt_exps" +) + +\S4method{exportQIIME2}{TreeSummarizedExperiment}(x, dpath, assay.type = "counts", tree.name = "phylo") + +\S4method{exportMothur}{TreeSummarizedExperiment}(x, dpath, assay.type = "counts", tree.name = "phylo") +} +\arguments{ +\item{x}{a \code{\link[TreeSummarizedExperiment]{TreeSummarizedExperiment}} object.} + +\item{...}{Unused.} + +\item{dpath}{\code{Character scalar}.} + +\item{assay.dir}{\code{Character scalar}. (Default: \code{"assays"})} + +\item{rowtree.dir}{\code{Character scalar}. (Default: \code{"row_trees"})} + +\item{coltree.dir}{\code{Character scalar}. (Default: \code{"col_trees"})} + +\item{dimred.dir}{\code{Character scalar}. (Default: \code{"dim_reds"})} + +\item{altexp.dir}{\code{Character scalar}. (Default: \code{"alt_exps"})} + +\item{assay.type}{\code{Character scalar}. (Default: \code{"counts"})} + +\item{rowdata}{\code{Character scalar}. (Default: \code{"rowdata"})} + +\item{coldata}{\code{Character scalar}. (Default: \code{"coldata"})} +} +\value{ +Directory at \code{dpath} with components of \code{x} each stored +as a file in the proper format. +} +\description{ +There are a few very popular external tools for microbiome analysis, +including QIIME2 and mothur. However, R does not currently provide any class +to accommodate those data formats. When exporting data from mia to external +tools, the best approach is therefore to break a data container into its +building blocks (assays, side information, trees, etc.). + +Thanks to \code{exportRaw}, \code{exportQIIME2} and \code{exportMothur}, +it is now possible to export a +\code{\link[TreeSummarizedExperiment]{TreeSummarizedExperiment}} object as +raw elements or near-ready QIIME2 and mothur formats, respectively. This way, +migrating from mia to an external system is still a bad idea, but at least it +is fairly straightforward. +} +\details{ +The output directory contains the elements of \code{x}. For +\code{exportQIIME2} and \code{exportMothur}, data will need some more +processing using the target tool. For some tips, check the rbiom package +vignettes on converting data: +\url{https://cmmr.github.io/rbiom/articles/convert.html} +} +\examples{ +library(TreeSummarizedExperiment) + +tse <- makeTSE() +assayNames(tse) <- "counts" + +# Export raw TreeSE components in custom directory +exportRaw(tse, "out") + +# Export TreeSE components in near-ready QIIME2 format +exportQIIME2(tse, "qiime2_dir") + +# Export TreeSE components in near-ready mothur format +exportMothur(tse, "mothur_dir") + +} From a519e0e96f6c5705f896ceb29705a60b08ef3d64 Mon Sep 17 00:00:00 2001 From: Giulio Benedetti Date: Wed, 22 Apr 2026 20:44:34 +0300 Subject: [PATCH 26/48] Minor fix to exporters --- R/exporters.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/exporters.R b/R/exporters.R index 321e256c3..20abd6c51 100644 --- a/R/exporters.R +++ b/R/exporters.R @@ -190,7 +190,7 @@ setMethod("exportQIIME2", signature = c(x = "TreeSummarizedExperiment"), row_tree <- rowTree(x, tree.name) if( !is.null(row_tree) ){ - write.tree(row_tree, paste(dpath, "tree.nwk")) + write.tree(row_tree, paste0(dpath, "tree.nwk")) } if( !is.null(referenceSeq(x)) ){ @@ -241,7 +241,7 @@ setMethod("exportMothur", signature = c(x = "TreeSummarizedExperiment"), row_tree <- rowTree(x, tree.name) if( !is.null(row_tree) ){ - write.tree(row_tree, paste(dpath, "tree.nwk")) + write.tree(row_tree, paste0(dpath, "tree.nwk")) } if( !is.null(referenceSeq(x)) ){ From b0d11f65b3ad3dba25d66707bc4ff0e1d17a1471 Mon Sep 17 00:00:00 2001 From: Giulio Benedetti Date: Thu, 23 Apr 2026 11:33:35 +0300 Subject: [PATCH 27/48] Fix variable types in exporters --- R/exporters.R | 44 +++++++++++++++++++++++++------------------ man/export-methods.Rd | 12 ++++++++---- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/R/exporters.R b/R/exporters.R index 20abd6c51..cff8f13f9 100644 --- a/R/exporters.R +++ b/R/exporters.R @@ -14,15 +14,18 @@ #' migrating from mia to an external system is still a bad idea, but at least it #' is fairly straightforward. #' -#' @param x a \code{\link[TreeSummarizedExperiment]{TreeSummarizedExperiment}} object. +#' @param x a \code{\link[TreeSummarizedExperiment]{TreeSummarizedExperiment}} +#' object. #' #' @param dpath \code{Character scalar}. #' #' @param assay.type \code{Character scalar}. (Default: \code{"counts"}) #' -#' @param rowdata \code{Character scalar}. (Default: \code{"rowdata"}) +#' @param tree.name \code{Character scalar}. (Default: \code{"phylo"}) #' -#' @param coldata \code{Character scalar}. (Default: \code{"coldata"}) +#' @param rowdata.file \code{Character scalar}. (Default: \code{"rowdata"}) +#' +#' @param coldata.file \code{Character scalar}. (Default: \code{"coldata"}) #' #' @param assay.dir \code{Character scalar}. (Default: \code{"assays"}) #' @@ -50,6 +53,7 @@ #' #' tse <- makeTSE() #' assayNames(tse) <- "counts" +#' names(rowData(tse))[1] <- "Genus" #' #' # Export raw TreeSE components in custom directory #' exportRaw(tse, "out") @@ -154,10 +158,10 @@ setMethod("exportQIIME2", signature = c(x = "TreeSummarizedExperiment"), if( !endsWith(dpath, "/") ) dpath <- paste0(dpath, "/") if( !dir.exists(dpath) ) dir.create(dpath) - row_data <- apply(rowData(x), 1L, paste, collapse = ";_") - row_data <- gsub(";_$", "", row_data) + row_data <- apply(rowData(x)[taxonomyRanks(x)], 1L, paste, collapse = ";_") + row_data <- gsub("(;_|;_NA)+$", "", row_data) - row_data <- cbind(names(row_data), row_data, 1L) + row_data <- data.frame(rownames(x), row_data, 1L, row.names = NULL) colnames(row_data) <- c("Feature ID", "Taxon", "Confidence") write.table( @@ -166,21 +170,24 @@ setMethod("exportQIIME2", signature = c(x = "TreeSummarizedExperiment"), col_data <- as.data.frame(colData(x)) + col_data[] <- lapply( + col_data, function(col) if( is.factor(col) ) as.character(col) else col + ) + col_types <- apply( col_data, 2L, function(col) switch( type(col), character = "categorical", integer = , double = "numeric") ) col_data <- rbind(col_types, col_data) - col_data <- cbind(`sample-id` = rownames(col_data), col_data) - col_data[1L, "sample-id"] <- "#q2:types" + col_data <- cbind(`sample-id` = c("#q2:types", colnames(x)), col_data) write.table( col_data, paste0(dpath, "metadata.tsv"), sep = "\t", row.names = FALSE ) - sel_assay <- assay(x, assay.type) - sel_assay <- cbind(`#OTU ID` = rownames(sel_assay), sel_assay) + sel_assay <- data.frame(rownames(x), assay(x, assay.type), row.names = NULL) + colnames(sel_assay)[1L] <- "#OTU ID" write.table( sel_assay, paste0(dpath, assay.type, ".tsv"), @@ -208,30 +215,31 @@ setMethod("exportMothur", signature = c(x = "TreeSummarizedExperiment"), if( !endsWith(dpath, "/") ) dpath <- paste0(dpath, "/") if( !dir.exists(dpath) ) dir.create(dpath) - row_data <- apply(rowData(x), 1L, paste, collapse = ";") - row_data <- gsub(";$", "", row_data) + row_data <- apply(rowData(x)[taxonomyRanks(x)], 1L, paste, collapse = ";") + row_data <- gsub("(;|;NA)+$", "", row_data) sel_assay <- assay(x, assay.type) row_sums <- rowSums(sel_assay) - row_data <- cbind(names(row_data), row_sums, row_data) + row_data <- data.frame( + names(row_data), row_sums, row_data, row.names = NULL + ) colnames(row_data) <- c("OTU", "Size", "Taxonomy") write.table( row_data, paste0(dpath, "taxonomy.tsv"), sep = "\t", row.names = FALSE ) - col_data <- as.data.frame(colData(x)) - col_data <- cbind(group = rownames(col_data), col_data) + col_data <- data.frame(group = colnames(x), colData(x), row.names = NULL) write.table( col_data, paste0(dpath, "metadata.tsv"), sep = "\t", row.names = FALSE ) - sel_assay <- cbind( - `Representative Sequence` = rownames(sel_assay), - total = row_sums, sel_assay + sel_assay <- data.frame( + rownames(sel_assay), total = row_sums, sel_assay, row.names = NULL ) + colnames(sel_assay)[1L] <- "Representative Sequence" write.table( sel_assay, paste0(dpath, assay.type, ".tsv"), diff --git a/man/export-methods.Rd b/man/export-methods.Rd index 3d1edd26b..8954ded4b 100644 --- a/man/export-methods.Rd +++ b/man/export-methods.Rd @@ -33,12 +33,17 @@ exportMothur(x, ...) \S4method{exportMothur}{TreeSummarizedExperiment}(x, dpath, assay.type = "counts", tree.name = "phylo") } \arguments{ -\item{x}{a \code{\link[TreeSummarizedExperiment]{TreeSummarizedExperiment}} object.} +\item{x}{a \code{\link[TreeSummarizedExperiment]{TreeSummarizedExperiment}} +object.} \item{...}{Unused.} \item{dpath}{\code{Character scalar}.} +\item{rowdata.file}{\code{Character scalar}. (Default: \code{"rowdata"})} + +\item{coldata.file}{\code{Character scalar}. (Default: \code{"coldata"})} + \item{assay.dir}{\code{Character scalar}. (Default: \code{"assays"})} \item{rowtree.dir}{\code{Character scalar}. (Default: \code{"row_trees"})} @@ -51,9 +56,7 @@ exportMothur(x, ...) \item{assay.type}{\code{Character scalar}. (Default: \code{"counts"})} -\item{rowdata}{\code{Character scalar}. (Default: \code{"rowdata"})} - -\item{coldata}{\code{Character scalar}. (Default: \code{"coldata"})} +\item{tree.name}{\code{Character scalar}. (Default: \code{"phylo"})} } \value{ Directory at \code{dpath} with components of \code{x} each stored @@ -85,6 +88,7 @@ library(TreeSummarizedExperiment) tse <- makeTSE() assayNames(tse) <- "counts" +names(rowData(tse))[1] <- "Genus" # Export raw TreeSE components in custom directory exportRaw(tse, "out") From 902719c0b9eeafb2ceee25a1b241f0ed6fb9105f Mon Sep 17 00:00:00 2001 From: Giulio Benedetti Date: Thu, 23 Apr 2026 20:05:21 +0300 Subject: [PATCH 28/48] Fixes to exporters --- R/exporters.R | 18 +++++++++++------- pkgdown/_pkgdown.yml | 5 ++++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/R/exporters.R b/R/exporters.R index cff8f13f9..3169b7efe 100644 --- a/R/exporters.R +++ b/R/exporters.R @@ -165,7 +165,8 @@ setMethod("exportQIIME2", signature = c(x = "TreeSummarizedExperiment"), colnames(row_data) <- c("Feature ID", "Taxon", "Confidence") write.table( - row_data, paste0(dpath, "taxonomy.tsv"), sep = "\t", row.names = FALSE + row_data, paste0(dpath, "taxonomy.tsv"), + sep = "\t", quote = FALSE, row.names = FALSE ) col_data <- as.data.frame(colData(x)) @@ -183,7 +184,8 @@ setMethod("exportQIIME2", signature = c(x = "TreeSummarizedExperiment"), col_data <- cbind(`sample-id` = c("#q2:types", colnames(x)), col_data) write.table( - col_data, paste0(dpath, "metadata.tsv"), sep = "\t", row.names = FALSE + col_data, paste0(dpath, "metadata.tsv"), + sep = "\t", quote = FALSE, row.names = FALSE ) sel_assay <- data.frame(rownames(x), assay(x, assay.type), row.names = NULL) @@ -191,7 +193,7 @@ setMethod("exportQIIME2", signature = c(x = "TreeSummarizedExperiment"), write.table( sel_assay, paste0(dpath, assay.type, ".tsv"), - sep = "\t", row.names = FALSE + sep = "\t", quote = FALSE, row.names = FALSE ) row_tree <- rowTree(x, tree.name) @@ -227,23 +229,25 @@ setMethod("exportMothur", signature = c(x = "TreeSummarizedExperiment"), colnames(row_data) <- c("OTU", "Size", "Taxonomy") write.table( - row_data, paste0(dpath, "taxonomy.tsv"), sep = "\t", row.names = FALSE + row_data, paste0(dpath, "taxonomy.tsv"), + sep = "\t", quote = FALSE, row.names = FALSE ) col_data <- data.frame(group = colnames(x), colData(x), row.names = NULL) write.table( - col_data, paste0(dpath, "metadata.tsv"), sep = "\t", row.names = FALSE + col_data, paste0(dpath, "metadata.tsv"), + sep = "\t", quote = FALSE, row.names = FALSE ) sel_assay <- data.frame( rownames(sel_assay), total = row_sums, sel_assay, row.names = NULL ) - colnames(sel_assay)[1L] <- "Representative Sequence" + colnames(sel_assay)[1L] <- "Representative_Sequence" write.table( sel_assay, paste0(dpath, assay.type, ".tsv"), - sep = "\t", row.names = FALSE + sep = "\t", quote = FALSE, row.names = FALSE ) row_tree <- rowTree(x, tree.name) diff --git a/pkgdown/_pkgdown.yml b/pkgdown/_pkgdown.yml index d4f4fe208..d9d991444 100644 --- a/pkgdown/_pkgdown.yml +++ b/pkgdown/_pkgdown.yml @@ -51,7 +51,7 @@ reference: - summary - getDominant - getAbundant -- title: Data loading +- title: Import/export/convert data - contents: - importBIOM - importQIIME2 @@ -61,6 +61,9 @@ reference: - importTaxpasta - convertFromDADA2 - convertFromPhyloseq + - exportRaw + - exportMothur + - exportQIIME2 - title: Diversity - subtitle: Alpha Diversity From 089fee460e2b9dec1d1d67205e4bb1d8fd330312 Mon Sep 17 00:00:00 2001 From: Giulio Benedetti Date: Thu, 23 Apr 2026 21:09:04 +0300 Subject: [PATCH 29/48] Add option to create group file for Mothur export --- R/exporters.R | 28 +++++++++++++++++++++++----- man/export-methods.Rd | 10 +++++++++- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/R/exporters.R b/R/exporters.R index 3169b7efe..f959aee24 100644 --- a/R/exporters.R +++ b/R/exporters.R @@ -19,10 +19,6 @@ #' #' @param dpath \code{Character scalar}. #' -#' @param assay.type \code{Character scalar}. (Default: \code{"counts"}) -#' -#' @param tree.name \code{Character scalar}. (Default: \code{"phylo"}) -#' #' @param rowdata.file \code{Character scalar}. (Default: \code{"rowdata"}) #' #' @param coldata.file \code{Character scalar}. (Default: \code{"coldata"}) @@ -37,6 +33,12 @@ #' #' @param altexp.dir \code{Character scalar}. (Default: \code{"alt_exps"}) #' +#' @param assay.type \code{Character scalar}. (Default: \code{"counts"}) +#' +#' @param tree.name \code{Character scalar}. (Default: \code{"phylo"}) +#' +#' @param group.var \code{Character scalar}. (Default: \code{NULL}) +#' #' @param ... Unused. #' #' @returns Directory at \code{dpath} with components of \code{x} each stored @@ -212,13 +214,28 @@ setMethod("exportQIIME2", signature = c(x = "TreeSummarizedExperiment"), #' @rdname export-methods #' @importFrom ape write.tree write.FASTA setMethod("exportMothur", signature = c(x = "TreeSummarizedExperiment"), - function(x, dpath, assay.type = "counts", tree.name = "phylo"){ + function(x, dpath, assay.type = "counts", tree.name = "phylo", + group.var = NULL){ if( !endsWith(dpath, "/") ) dpath <- paste0(dpath, "/") if( !dir.exists(dpath) ) dir.create(dpath) + rownames(x) <- gsub("-", "_", rownames(x), fixed = TRUE) + + if( !is.null(group.var) ){ + + group <- rowData(x)[group.var] + group[[group.var]] <- gsub("-", "_", group[[group.var]]) + + write.table( + group, paste0(dpath, group.var, ".group"), + sep = "\t", quote = FALSE, col.names = FALSE + ) + } + row_data <- apply(rowData(x)[taxonomyRanks(x)], 1L, paste, collapse = ";") row_data <- gsub("(;|;NA)+$", "", row_data) + row_data <- gsub("-", "_", row_data, fixed = TRUE) sel_assay <- assay(x, assay.type) row_sums <- rowSums(sel_assay) @@ -253,6 +270,7 @@ setMethod("exportMothur", signature = c(x = "TreeSummarizedExperiment"), row_tree <- rowTree(x, tree.name) if( !is.null(row_tree) ){ + row_tree$tip.label <- gsub("-", "_", row_tree$tip.label) write.tree(row_tree, paste0(dpath, "tree.nwk")) } diff --git a/man/export-methods.Rd b/man/export-methods.Rd index 8954ded4b..9936c8e15 100644 --- a/man/export-methods.Rd +++ b/man/export-methods.Rd @@ -30,7 +30,13 @@ exportMothur(x, ...) \S4method{exportQIIME2}{TreeSummarizedExperiment}(x, dpath, assay.type = "counts", tree.name = "phylo") -\S4method{exportMothur}{TreeSummarizedExperiment}(x, dpath, assay.type = "counts", tree.name = "phylo") +\S4method{exportMothur}{TreeSummarizedExperiment}( + x, + dpath, + assay.type = "counts", + tree.name = "phylo", + group.var = NULL +) } \arguments{ \item{x}{a \code{\link[TreeSummarizedExperiment]{TreeSummarizedExperiment}} @@ -57,6 +63,8 @@ object.} \item{assay.type}{\code{Character scalar}. (Default: \code{"counts"})} \item{tree.name}{\code{Character scalar}. (Default: \code{"phylo"})} + +\item{group.var}{\code{Character scalar}. (Default: \code{NULL})} } \value{ Directory at \code{dpath} with components of \code{x} each stored From 698fac70877cf8df745b039a9de61d90847c81d5 Mon Sep 17 00:00:00 2001 From: Giulio Benedetti Date: Fri, 24 Apr 2026 12:07:09 +0300 Subject: [PATCH 30/48] Add group arg for qiime exporter --- R/exporters.R | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/R/exporters.R b/R/exporters.R index f959aee24..4ff078b6b 100644 --- a/R/exporters.R +++ b/R/exporters.R @@ -155,11 +155,25 @@ setMethod("exportRaw", signature = c(x = "TreeSummarizedExperiment"), #' @rdname export-methods #' @importFrom ape write.tree write.FASTA setMethod("exportQIIME2", signature = c(x = "TreeSummarizedExperiment"), - function(x, dpath, assay.type = "counts", tree.name = "phylo"){ + function(x, dpath, assay.type = "counts", tree.name = "phylo", + group.var = NULL){ if( !endsWith(dpath, "/") ) dpath <- paste0(dpath, "/") if( !dir.exists(dpath) ) dir.create(dpath) + if( !is.null(group.var) ){ + + group <- rowData(x)[[group.var]] + group <- gsub("-", "_", group) + group <- cbind(rownames(x), group) + colnames(group) <- c("Feature ID", group.var) + + write.table( + group, paste0(dpath, group.var, ".tsv"), + sep = "\t", quote = FALSE, row.names = FALSE + ) + } + row_data <- apply(rowData(x)[taxonomyRanks(x)], 1L, paste, collapse = ";_") row_data <- gsub("(;_|;_NA)+$", "", row_data) @@ -221,6 +235,7 @@ setMethod("exportMothur", signature = c(x = "TreeSummarizedExperiment"), if( !dir.exists(dpath) ) dir.create(dpath) rownames(x) <- gsub("-", "_", rownames(x), fixed = TRUE) + colnames(x) <- gsub("-", "_", colnames(x), fixed = TRUE) if( !is.null(group.var) ){ From 6574215c7279f1c67b35ae3e81e7d7ad31835abb Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 4 May 2026 19:41:01 +0300 Subject: [PATCH 31/48] Initial version that compiles --- src/assay.cpp | 14 + src/assay.h | 33 +- src/stripemap.h | 36 +- src/unifrac.cpp | 873 +++++++++-------------- src/{unifrac.hpp => unifrac.h} | 120 ++-- src/unifrac_R.cpp | 98 +-- src/unifrac_task.cpp | 103 ++- src/{unifrac_task.hpp => unifrac_task.h} | 213 +++--- 8 files changed, 657 insertions(+), 833 deletions(-) rename src/{unifrac.hpp => unifrac.h} (56%) rename src/{unifrac_task.hpp => unifrac_task.h} (79%) diff --git a/src/assay.cpp b/src/assay.cpp index c98dd0efe..4f4f3397a 100644 --- a/src/assay.cpp +++ b/src/assay.cpp @@ -59,6 +59,20 @@ std::vector Assay::get_obs_data(const std::string &id) const { return out; } +std::vector Assay::get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize) const { + std::vector out = std::vector(); + uint32_t idx = obs_id_index.at(id); + for(unsigned int i = start; i < end; i++) { + if (normalize) { + out.push_back(table(idx, i)/sample_counts[i]); + } + else { + out.push_back(table(idx, i)); + } + } + return out; +} + std::vector Assay::get_sample_counts(){ std::vector sample_counts = std::vector(); diff --git a/src/assay.h b/src/assay.h index 89b50c262..77c80f86b 100644 --- a/src/assay.h +++ b/src/assay.h @@ -41,14 +41,23 @@ class Assay { /* Get a dense vector of observation data * * @param id The observation ID to fetch - * @param out An allocated array of at least size n_samples. - * Values of an index position [0, n_samples) which do not - * have data will be zero'd. */ std::vector get_obs_data(const std::string &id) const; - private: - Rcpp::NumericMatrix table; // Access to raw sample counts in R's memory + /* get a dense vector of a range of observation data + * + * @param id The observation ID to fetch + * @param start Initial index + * @param end First index past the end + * @param normalize If set, divide by sample_counts + */ + std::vector get_obs_data_range(const std::string &id, + unsigned int start, + unsigned int end, + bool normalize) const; + +private: + Rcpp::NumericMatrix table; // Access to raw sample counts in R's memory std::vector get_sample_counts(); @@ -58,14 +67,14 @@ class Assay { std::unordered_map obs_id_index; /* Create an index mapping an ID to its corresponding index - * position. - * - * @param ids A vector of IDs to index - * @param map A hash table to populate - */ + * position. + * + * @param ids A vector of IDs to index + * @param map A hash table to populate + */ void create_id_index(std::vector &ids, - std::unordered_map &map); + std::unordered_map &map); }; } diff --git a/src/stripemap.h b/src/stripemap.h index 1ebc647e0..94467acaa 100644 --- a/src/stripemap.h +++ b/src/stripemap.h @@ -7,29 +7,29 @@ * See LICENSE file for more details */ -#ifndef __FAITH_PROPMAP -#define __FAITH_PROPMAP 1 +#ifndef __FAITH_STRIPEMAP +#define __FAITH_STRIPEMAP 1 #include #include #include +#include namespace su { -class StripeMap { -public: - StripeMap(uint32_t n_samples); - virtual ~StripeMap(); - void clear(uint32_t i); - void update(uint32_t i, std::vector vec); - std::vector get(uint32_t i); - bool is_empty(uint32_t i); - -private: - std::unordered_map> stripe_map; - uint32_t vecsize; // Size of stripe vectors is always the number of samples - uint32_t n_stripes; -}; - + class StripeMap { + public: + StripeMap(uint32_t n_samples); + virtual ~StripeMap(); + void clear(uint32_t i); + void update(uint32_t i, std::vector vec); + std::vector get(uint32_t i); + bool is_empty(uint32_t i); + + private: + std::unordered_map> stripe_map; + uint32_t vecsize; // Size of stripe vectors is always the number of samples + uint32_t n_stripes; + }; } -#endif /* __FAITH_PROPMAP */ +#endif /* __FAITH_STRIPEMAP */ diff --git a/src/unifrac.cpp b/src/unifrac.cpp index 9a5c8e70b..231f5ae3c 100644 --- a/src/unifrac.cpp +++ b/src/unifrac.cpp @@ -7,13 +7,12 @@ * See LICENSE file for more details */ -#include "unifrac.hpp" -#include "propmap.hpp" -#include "stripemap.hpp" -#include "tree.hpp" +#include "unifrac.h" +#include "propmap.h" +#include "stripemap.h" +#include "tree.h" -#include "biom_interface.hpp" -#include "affinity.hpp" +/* #include #include #include @@ -22,147 +21,209 @@ #include #include #include +*/ -#include "unifrac_internal.hpp" -void su::process_stripes(const su::Assay & table, - const su::BPTree & tree_sheared, - Method method, - bool variance_adjust, - su::StripeMap dm_stripes, - su::StripeMap dm_stripes_total, - su::task_parameters task) { - if(variance_adjust) - su::unifrac_vaw( - std::ref(table), - std::ref(tree_sheared), - method, - std::ref(dm_stripes), - std::ref(dm_stripes_total), - &tasks[tid]); - else - switch(method) { - case su::unweighted: - //A templated function... - unifracTT(table, tree, true, dm_stripes, dm_stripes_total, task); - break; - default: - fprintf(stderr, "Unknown unifrac task\n"); - exit(1); - break; - } + + +su::Method su::set_method(std::string requested_method) { + if(requested_method == "unweighted") + return unweighted; + else if(requested_method == "weighted_normalized") + return weighted_normalized; + else if(requested_method == "weighted_unnormalized") + return weighted_unnormalized; + else if(requested_method == "generalized") + return generalized; + /*else if(std::strcmp(requested_method, "unweighted_fp32") == 0) + method = unweighted_fp32; + else if(std::strcmp(requested_method, "weighted_normalized_fp32") == 0) + method = weighted_normalized_fp32; + else if(std::strcmp(requested_method, "weighted_unnormalized_fp32") == 0) + method = weighted_unnormalized_fp32; + else if(std::strcmp(requested_method, "generalized_fp32") == 0) + method = generalized_fp32; */ + else { + return unknown; + } +} + + + +su::mat_t su::one_off(const su::Assay & table, + const su::BPTree & tree, + std::string unifrac_method, + double alpha, + bool variance_adjust, + bool bypass_tips) { + + //Check that method is valid - pass it as something other than string? + su::Method method = set_method(unifrac_method); + + //Number of stripes to be used, basically half of samples + const unsigned int stripe_stop = (table.n_samples + 1) / 2; + + //Originally std::vector of double pointers - this is where the data travels? + su::StripeMap dm_stripes(table.n_samples); + su::StripeMap dm_stripes_total(table.n_samples); + + su::task_parameters task; + + //thread id - currently single-threaded + task.tid = 0; + //Stripes to start and stop on - single task, so the entire thing + task.start = 0; + task.stop = stripe_stop; + + task.bypass_tips = bypass_tips; + task.n_samples = table.n_samples; + task.g_unifrac_alpha = alpha; + + //Main action + //Calls either unifrac or _vaw depending on variance_adjust + //makes use of std::ref? + //Versions for accelerated and cpu - let's go with cpu for now + //method is "unweighted" by default, let's start with that and see what else may be needed + + + //This could potentially be threaded + //Wasn't in the code because doesn't work with openacc/openmp? + + su::unifrac(std::ref(table), + std::ref(tree), + method, + std::ref(dm_stripes), + std::ref(dm_stripes_total), + task, + variance_adjust); + + //Only use of std::thread in this version of code was for stripes to condensed form + //Basically each thread calls stripes_to_condensed_form + //Which is just a bunch of binomial calculations + + su::mat_t result; + result.n_samples = table.n_samples; + result.cf_size = su::comb_2(table.n_samples); + result.sample_ids = table.sample_ids; + result.is_upper_triangle = true; + result.condensed_form = su::stripes_to_condensed_form(dm_stripes, + table.n_samples, + task.start, + task.stop); + + return result; } + + + + + + template -inline void unifracTT(const su::Assay & table, +inline void su::unifracTT(const su::Assay & table, const su::BPTree & tree, const bool want_total, su::StripeMap dm_stripes, su::StripeMap dm_stripes_total, - const su::task_parameters & task_p) { + const su::task_parameters & task_p) +{ - const unsigned int n_samples = task_p->n_samples; - - //What is this used for? - //UNIFRAC_BLOCK = 16 - //const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up - const uint64_t n_samples_r = ((n_samples + 15)/16)*16; // round up + if(table.n_samples != task_p.n_samples) { + fprintf(stderr, "Task and table n_samples not equal\n"); + exit(EXIT_FAILURE); + } + const unsigned int n_samples = task_p.n_samples; + const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up - //How does this differ from normal propstack? - //Basically a vector of several fixed-size propstacks? - //Can probably be replaced with our current impl. - //su::PropStackMulti propstack_multi(table.n_samples); + //su::PropStackMulti propstack_multi(table.n_samples); su::PropMap propmap(table.n_samples); - su::StripeMap stripemap(table.n_samples); - su::StripeMap stripemap_total(table.n_samples); + const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; - //Not sure how relevant these are, but let's keep them for now - const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; - - //Initialize a task object with the given memory locations and parameters - //PResumably we want to pass the stripes by reference? - //Eh, just add a function to get them TaskT taskObj(dm_stripes, dm_stripes_total, max_emb, task_p); - std::vector lengths(max_emb) - - //Algorithm portion follows - - /* - * The values in the example vectors correspond to index positions of an - * element in the resulting distance matrix. So, in the example below, - * the following can be interpreted: - * - * [0 1 2] - * [1 2 3] - * - * As comparing the sample for row 0 against the sample for col 1, the - * sample for row 1 against the sample for col 2, the sample for row 2 - * against the sample for col 3. - * - * In other words, we're computing stripes of a distance matrix. In the - * following example, we're computing over 6 samples requiring 3 - * stripes. - * - * A; stripe == 0 - * [0 1 2 3 4 5] - * [1 2 3 4 5 0] - * - * B; stripe == 1 - * [0 1 2 3 4 5] - * [2 3 4 5 0 1] - * - * C; stripe == 2 - * [0 1 2 3 4 5] - * [3 4 5 0 1 2] - * - * The stripes end up computing the following positions in the distance - * matrix. - * - * x A B C x x - * x x A B C x - * x x x A B C - * C x x x A B - * B C x x x A - * A B C x x x - * - * However, we store those stripes as vectors, ie - * [ A A A A A A ] - * - * We end up performing N / 2 redundant calculations on the last stripe - * (see C) but that is small over large N. - */ - + std::vector lengths = std::vector(max_emb); + + /* + * The values in the example vectors correspond to index positions of an + * element in the resulting distance matrix. So, in the example below, + * the following can be interpreted: + * + * [0 1 2] + * [1 2 3] + * + * As comparing the sample for row 0 against the sample for col 1, the + * sample for row 1 against the sample for col 2, the sample for row 2 + * against the sample for col 3. + * + * In other words, we're computing stripes of a distance matrix. In the + * following example, we're computing over 6 samples requiring 3 + * stripes. + * + * A; stripe == 0 + * [0 1 2 3 4 5] + * [1 2 3 4 5 0] + * + * B; stripe == 1 + * [0 1 2 3 4 5] + * [2 3 4 5 0 1] + * + * C; stripe == 2 + * [0 1 2 3 4 5] + * [3 4 5 0 1 2] + * + * The stripes end up computing the following positions in the distance + * matrix. + * + * x A B C x x + * x x A B C x + * x x x A B C + * C x x x A B + * B C x x x A + * A B C x x x + * + * However, we store those stripes as vectors, ie + * [ A A A A A A ] + * + * We end up performing N / 2 redundant calculations on the last stripe + * (see C) but that is small over large N. + */ + unsigned int k = 0; // index in tree const unsigned int max_k = (tree.nparens / 2) - 1; - while (k node_proportions = propmap.get(node); + //TFloat *node_proportions = propstack.pop(node); + //su::set_proportions_range(node_proportions, tree, node, table, tstart, tend, propstack); + + //calculate proportions range for given node + su::set_proportions_range(tree, node, table, tstart, tend, propmap); - node_proportions = su::set_proportions(tree, node, table, propmap); + //propstack pop ERASES any existing vector for node and gives a blank one + //creates memory leaks if node isn't pushed before popping? + //get just returns the given vector + //push removes the vector from use + //Any time a propstack vector is modified, remember to do a propmap update if(task_p.bypass_tips && tree.isleaf(node)) continue; @@ -170,45 +231,183 @@ inline void unifracTT(const su::Assay & table, lengths[filled_emb] = tree.lengths[node]; filled_emb++; - taskObj.embed_proportions(node_proportions, my_filled_emb); + //store the proportions inside the taskobject's continuous buffer + //Shouldn't modify node_proportions + std::vector node_proportions = propmap.get(node); + taskObj.embed_proportions_range(node_proportions, tstart, tend, my_filled_emb); my_filled_emb++; } - + k=my_k; + //This is used to keep track of filled embeds over different threads? + //Does nothing without openacc + //taskObj.sync_embedded_proportions(filled_emb); + taskObj._run(filled_emb,lengths); + filled_emb=0; } + //I suppose want_total is used if you want the results as a percentage of the total? if(want_total) { - const uint64_t start_idx = task_p->start; - const uint64_t stop_idx = task_p->stop; + const uint64_t start_idx = task_p.start; + const uint64_t stop_idx = task_p.stop; - double * const dm_stripes_buf = taskObj.dm_stripes.buf; - const double * const dm_stripes_total_buf = taskObj.dm_stripes_total.buf; - - for(uint64_t i = start_idx; i < stop_idx; i++) + for(uint64_t i = start_idx; i < stop_idx; i++){ + /* + std::vector dm_stripes_buf = std::vector ; + std::vector dm_stripes_total_buf = taskObj.dm_stripes_total.get(idx); + std::copy(std::begin(taskObj.dm_stripes.buf), + std::end(taskObj.dm_stripes.buf), + std::begin(dm_stripes_buf) + (emb8<<8)); + */ + + std::vector dm_stripes_buf = taskObj.dm_stripes.buf; + std::vector dm_stripes_total_buf = taskObj.dm_stripes_total.buf; + for(uint64_t j = 0; j < n_samples; j++) { uint64_t idx = (i-start_idx)*n_samples_r+j; - dm_stripes_buf[idx]=dm_stripes_buf[idx]/dm_stripes_total_buf[idx]; + dm_stripes_buf[idx] = dm_stripes_buf[idx]/dm_stripes_total_buf[idx]; } + taskObj.dm_stripes.buf = dm_stripes_buf; + + /* + taskObj.dm_stripes.update(idx, dm_stripes_buf); + std::copy(std::begin(dm_stripes_buf), + std::end(dm_stripes_buf), + std::begin(taskObj.dm_stripes.buf) + ); + */ + } + } +} + + + +std::vector su::set_proportions_range(const su::BPTree & tree, + uint32_t node, + const su::Assay & table, + unsigned int start, + unsigned int end, + PropMap & pm, + bool normalize) { + const unsigned int els = end-start; + std::vector props = std::vector(els, 0.0); + if(tree.isleaf(node)) { + props = table.get_obs_data_range(tree.names[node], start, end, normalize); + } else { + const unsigned int right = tree.rightchild(node); + unsigned int current = tree.leftchild(node); + + while(current <= right && current != 0) { + std::vector vec = pm.get(current); // pull from prop map + pm.clear(current); // remove from prop map, place back on stack + + for(unsigned int i = 0; i < els; i++) + props[i] += vec[i]; + + current = tree.rightsibling(current); + } } + pm.update(node, props); + return props; } -su::mat_t su::stripes_to_condensed_form(su::StripeMap stripes, + + + + +void su::unifrac(const su::Assay &table, + const su::BPTree &tree, + su::Method unifrac_method, + su::StripeMap &dm_stripes, + su::StripeMap &dm_stripes_total, + const su::task_parameters task_p, + bool variance_adjust) +{ + if(variance_adjust) + { + /* + switch(unifrac_method) { + case su::unweighted: + unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_normalized: + unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_unnormalized: + unifrac_vawTT,double>( table, tree, false, dm_stripes,dm_stripes_total,task_p); + break; + case su::generalized: + unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::unweighted_fp32: + unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_normalized_fp32: + unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_unnormalized_fp32: + unifrac_vawTT,float >( table, tree, false, dm_stripes,dm_stripes_total,task_p); + break; + case su::generalized_fp32: + unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + default: + fprintf(stderr, "Unknown unifrac task\n"); + exit(1); + break; + } + */ + } + else + { + switch(unifrac_method) + { + case su::unweighted: + unifracTT( + table, tree, true, dm_stripes,dm_stripes_total, + task_p ); + break; + /*case su::weighted_normalized: + unifracTT,double>( + table, tree, true, dm_stripes,dm_stripes_total, + task_p ); + break; + case su::weighted_unnormalized: + unifracTT, + double>(table, tree, false, dm_stripes, + dm_stripes_total, task_p ); + break; + case su::generalized: + unifracTT,double>( + table, tree, true, dm_stripes,dm_stripes_total, + task_p ); + break; + */ + default: + fprintf(stderr, "Unknown unifrac task\n"); + exit(1); + break; + } + } +} + +std::vector su::stripes_to_condensed_form(su::StripeMap stripes, uint32_t n, - su::mat_t result, unsigned int start, unsigned int stop) { // n must be >= 2, but that should be enforced upstream as that would imply // computing unifrac on a single sample. - std::vector cf = result.condensed_form; - uint64_t comb_N = comb_2(n); + std::vector cf = std::vector(comb_N, 0.0); + for(unsigned int stripe = start; stripe < stop; stripe++) { + //Does stripemap contain all the stripes or just one thread's stripes? + std::vector dm_stripe = stripes.get(stripe); // compute the (i, j) position of each element in each stripe uint64_t i = 0; uint64_t j = stripe + 1; @@ -221,428 +420,10 @@ su::mat_t su::stripes_to_condensed_form(su::StripeMap stripes, // based off of // https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.squareform.html uint64_t comb_N_minus_i = comb_2(n - i); - cf[comb_N - comb_N_minus_i + (j - i - 1)] = stripes.get(stripe)[k]; - } - } - result.condensed_form = cf; - return result; -} - -double** su::deconvolute_stripes(std::vector &stripes, uint32_t n) { - // would be better to just do striped_to_condensed_form - double **dm; - dm = (double**)malloc(sizeof(double*) * n); - if(dm == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double*) * n, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - for(unsigned int i = 0; i < n; i++) { - dm[i] = (double*)malloc(sizeof(double) * n); - if(dm[i] == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double) * n, __FILE__, __LINE__); - exit(EXIT_FAILURE); + cf[comb_N - comb_N_minus_i + (j - i - 1)] = dm_stripe[k]; } - dm[i][i] = 0; } - - for(unsigned int i = 0; i < stripes.size(); i++) { - double *vec = stripes[i]; - unsigned int k = 0; - for(unsigned int row = 0, col = i + 1; row < n; row++, col++) { - if(col < n) { - dm[row][col] = vec[k]; - dm[col][row] = vec[k]; - } else { - dm[col % n][row] = vec[k]; - dm[row][col % n] = vec[k]; - } - k++; - } - } - return dm; -} - - - -// write in a 2D matrix -// also suitable for writing to disk -template -void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d) { - const uint64_t comb_N = su::comb_2(n); - for(uint64_t i = 0; i < n; i++) { - for(uint64_t j = 0; j < n; j++) { - TReal v; - if(i < j) { // upper triangle - const uint64_t comb_N_minus = su::comb_2(n - i); - v = cf[comb_N - comb_N_minus + (j - i - 1)]; - } else if (i > j) { // lower triangle - const uint64_t comb_N_minus = su::comb_2(n - j); - v = cf[comb_N - comb_N_minus + (i - j - 1)]; - } else { - v = 0.0; - } - buf2d[i*n+j] = v; - } - } -} - - -// make sure it is instantiated -template void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); -template void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); - -void su::condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d) { - su::condensed_form_to_matrix_T(cf,n,buf2d); -} - -void su::condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d) { - su::condensed_form_to_matrix_T(cf,n,buf2d); + return cf; } -/* - * The stripes end up computing the following positions in the distance - * matrix. - * - * x A B C x x - * x x A B C x - * x x x A B C - * C x x x A B - * B C x x x A - * A B C x x x - * - * However, we store those stripes as vectors, ie - * [ A A A A A A ] - */ - - -// Helper class -// Will cache pointers and automatically release stripes when all elements are used -class OnceManagedStripes { - private: - const uint32_t n_samples; - const uint32_t n_stripes; - const ManagedStripes &stripes; - std::vector stripe_ptr; - std::vector stripe_accessed; - - const double *get_stripe(const uint32_t stripe) { - if (stripe_ptr[stripe]==0) stripe_ptr[stripe]=stripes.get_stripe(stripe); - return stripe_ptr[stripe]; - } - - void release_stripe(const uint32_t stripe) { - stripes.release_stripe(stripe); - stripe_ptr[stripe]=0; - } - - public: - OnceManagedStripes(const ManagedStripes &_stripes, const uint32_t _n_samples, const uint32_t _n_stripes) - : n_samples(_n_samples), n_stripes(_n_stripes) - , stripes(_stripes) - , stripe_ptr(n_stripes) - , stripe_accessed(n_stripes) - {} - - ~OnceManagedStripes() - { - for(uint32_t i = 0; i < n_stripes; i++) { - if (stripe_ptr[i]!=0) { - release_stripe(i); - } - } - } - - double get_val(const uint32_t stripe, const uint32_t el) - { - if (stripe_ptr[stripe]==0) stripe_ptr[stripe]=stripes.get_stripe(stripe); - const double *mystripe = stripe_ptr[stripe]; - double val = mystripe[el]; - - stripe_accessed[stripe]++; - if (stripe_accessed[stripe]==n_samples) release_stripe(stripe); // we will not use this stripe anymore - - return val; - } - - -}; - -// write in a 2D matrix -// also suitable for writing to disk -template -void su::stripes_to_matrix_T(const ManagedStripes &_stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size) { - // n_samples must be >= 2, but that should be enforced upstream as that would imply - // computing unifrac on a single sample. - - // tile for for better memory access pattern - const uint32_t TILE = (tile_size>0) ? tile_size : (128/sizeof(TReal)); - const uint32_t n_samples_tup = (n_samples+(TILE-1))/TILE; // round up - - OnceManagedStripes stripes(_stripes, n_samples, n_stripes); - - - for(uint32_t oi = 0; oi < n_samples_tup; oi++) { // off diagonal - // alternate between inner and outer off-diagonal, due to wrap around in stripes - const uint32_t o = ((oi%2)==0) ? \ - (oi/2)*TILE : /* close to diagonal */ \ - (n_samples_tup-(oi/2)-1)*TILE; /* far from diagonal */ - - for(uint32_t d = 0; d < (n_samples-o); d+=TILE) { // diagonal - - uint32_t iOut = d; - uint32_t jOut = d+o; - - uint32_t iMax = std::min(iOut+TILE,n_samples); - uint32_t jMax = std::min(jOut+TILE,n_samples); - - - if (iOut==jOut) { - // on diagonal - for(uint64_t i = iOut; i < iMax; i++) { - buf2d[i*n_samples+i] = 0.0; - int64_t stripe=0; - - uint64_t j = i+1; - for(; (stripen_stripes) { - // ops, we overshoot... roll back - j-=(stripe-n_stripes); - stripe=n_stripes; - } - for(; (stripe(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size); -template void su::stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size); - -void su::stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size) { - return su::stripes_to_matrix_T(stripes, n_samples, n_stripes, buf2d, tile_size); -} - -void su::stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size) { - return su::stripes_to_matrix_T(stripes, n_samples, n_stripes, buf2d, tile_size); -} - - -void progressbar(float progress) { - // from http://stackoverflow.com/a/14539953 - // - // could encapsulate into a classs for displaying time elapsed etc - int barWidth = 70; - std::cout << "["; - int pos = barWidth * progress; - for (int i = 0; i < barWidth; ++i) { - if (i < pos) std::cout << "="; - else if (i == pos) std::cout << ">"; - else std::cout << " "; - } - std::cout << "] " << int(progress * 100.0) << " %\r"; - std::cout.flush(); -} - -// Computes Faith's PD for the samples in `table` over the phylogenetic -// tree given by `tree`. -// Assure that tree does not contain ids that are not in table -void su::faith_pd(biom_interface &table, - BPTree &tree, - double* result) { - PropStack propstack(table.n_samples); - - uint32_t node; - double *node_proportions; - double length; - - // for node in postorderselect - for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { - node = tree.postorderselect(k); - // get branch length - length = tree.lengths[node]; - - // get node proportions and set intermediate scores - node_proportions = propstack.pop(node); - set_proportions(node_proportions, tree, node, table, propstack); - - for (unsigned int sample = 0; sample < table.n_samples; sample++){ - // calculate contribution of node to score - result[sample] += (node_proportions[sample] > 0) * length; - } - } -} - - -#ifdef UNIFRAC_ENABLE_ACC - -// test only once, then use persistent value -static int proc_use_acc = -1; - -inline bool use_acc() { - if (proc_use_acc!=-1) return (proc_use_acc!=0); - int has_nvidia_gpu_rc = access("/proc/driver/nvidia/gpus", F_OK); - - bool print_info = false; - - if (const char* env_p = std::getenv("UNIFRAC_GPU_INFO")) { - print_info = true; - std::string env_s(env_p); - if ((env_s=="NO") || (env_s=="N") || (env_s=="no") || (env_s=="n") || - (env_s=="NEVER") || (env_s=="never")) { - print_info = false; - } - } - - - if (has_nvidia_gpu_rc != 0) { - if (print_info) printf("INFO (unifrac): GPU not found, using CPU\n"); - proc_use_acc=0; - return false; - } - - if (const char* env_p = std::getenv("UNIFRAC_USE_GPU")) { - std::string env_s(env_p); - if ((env_s=="NO") || (env_s=="N") || (env_s=="no") || (env_s=="n") || - (env_s=="NEVER") || (env_s=="never")) { - if (print_info) printf("INFO (unifrac): Use of GPU explicitly disabled, using CPU\n"); - proc_use_acc=0; - return false; - } - } - - if (print_info) printf("INFO (unifrac): Using GPU\n"); - proc_use_acc=1; - return true; -} -#endif - -void su::unifrac(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { -#ifdef UNIFRAC_ENABLE_ACC - if (use_acc()) { - su_acc::unifrac(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } else { -#else - if (true) { -#endif - su_cpu::unifrac(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } -} - - -void su::unifrac_vaw(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { -#ifdef UNIFRAC_ENABLE_ACC - if (use_acc()) { - su_acc::unifrac_vaw(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } else { -#else - if (true) { -#endif - su_cpu::unifrac_vaw(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } -} - - -void su::process_stripes(biom_interface &table, - BPTree &tree_sheared, - Method method, - bool variance_adjust, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - std::vector &threads, - std::vector &tasks) { - - // register a signal handler so we can ask the master thread for its - // progress - register_report_status(); - - // cannot use threading with openacc or openmp - for(unsigned int tid = 0; tid < threads.size(); tid++) { - if(variance_adjust) - su::unifrac_vaw( - std::ref(table), - std::ref(tree_sheared), - method, - std::ref(dm_stripes), - std::ref(dm_stripes_total), - &tasks[tid]); - else - su::unifrac( - std::ref(table), - std::ref(tree_sheared), - method, - std::ref(dm_stripes), - std::ref(dm_stripes_total), - &tasks[tid]); - } - - remove_report_status(); -} diff --git a/src/unifrac.hpp b/src/unifrac.h similarity index 56% rename from src/unifrac.hpp rename to src/unifrac.h index 83167d8d8..a69cf0d3e 100644 --- a/src/unifrac.hpp +++ b/src/unifrac.h @@ -7,16 +7,18 @@ * See LICENSE file for more details */ +#ifndef __UNIFRAC +#define __UNIFRAC 1 + #include +#include #include #include -#include -#include - -#ifndef __UNIFRAC -#include "task_parameters.hpp" -#include "biom_interface.hpp" +#include "assay.h" +#include "tree.h" +#include "propmap.h" +#include "unifrac_task.h" namespace su { @@ -27,22 +29,62 @@ std::vector condensed_form; std::vector sample_ids; } mat_t; + + enum Method {unweighted, + weighted_normalized, + weighted_unnormalized, + generalized, + unknown}; - // process the stripes described by tasks - void process_stripes(const su::Assay & table, - const su::BPTree & tree_sheared, - Method method, - bool variance_adjust, - su::StripeMap dm_stripes, - su::StripeMap dm_stripes_total, - su::task_parameters task); + Method set_method(std::string requested_method); - // Stripes to condensed form for the results - su::mat_t stripes_to_condensed_form(su::StripeMap stripes, - uint32_t n, - su::mat_t result, - unsigned int start, - unsigned int stop); + + /* Compute UniFrac - condensed form + * + * biom_filename the filename to the biom table. + * tree_filename the filename to the correspodning tree. + * unifrac_method the requested unifrac method. + * variance_adjust whether to apply variance adjustment. + * alpha GUniFrac alpha, only relevant if method == generalized. + * bypass_tips disregard tips, reduces compute by about 50% + * threads the number of threads to use. + * result the resulting distance matrix in condensed form, this is initialized within the method so using ** + * + * one_off returns the following error codes: + * + * okay : no problems encountered + * table_missing : the filename for the table does not exist + * tree_missing : the filename for the tree does not exist + * unknown_method : the requested method is unknown. + * table_empty : the table does not have any entries + */ + + su::mat_t one_off(const su::Assay & table, + const su::BPTree & tree, + std::string unifrac_method, + double alpha, + bool variance_adjust, + bool bypass_tips); + + // Chooses the right task for the job and constructs a unifracTT + void unifrac(const su::Assay &table, + const su::BPTree &tree, + su::Method unifrac_method, + su::StripeMap &dm_stripes, + su::StripeMap &dm_stripes_total, + const su::task_parameters task_p, + bool variance_adjust); + + // Sets proportion range + // Data is stored to props -> make return vector + // PropMap needs to be modified, thus passed by reference + std::vector set_proportions_range(const su::BPTree & tree, + uint32_t node, + const su::Assay & table, + unsigned int start, + unsigned int end, + PropMap & pm, + bool normalize = true); // Works the vectors template @@ -77,32 +119,16 @@ return val; } - template - inline void unifracTT(const su::biom_interface &table, - const su::BPTree &tree, - const bool want_total, - su::StripeMap dm_stripes, - su::StripeMap dm_stripes_total, - const su::task_parameters & task_p) - - enum Method {unweighted, - weighted_normalized, - weighted_unnormalized, - generalized}; - - void unifrac(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const task_parameters* task_p); - - void unifrac_vaw(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const task_parameters* task_p); + // Stripes to condensed form for the results + std::vector stripes_to_condensed_form(su::StripeMap stripes, + uint32_t n, + unsigned int start, + unsigned int stop); + + + + + double** deconvolute_stripes(std::vector &stripes, uint32_t n); @@ -138,5 +164,5 @@ void condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); } -#define __UNIFRAC 1 + #endif diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index 3e4ef89ce..595849985 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -16,14 +16,15 @@ #include "tree.h" #include "propmap.h" #include "stripemap.h" -#include "unifrac.hpp" -#include "unifrac_task.hpp" + +#include "unifrac.h" + // Calculate Unifrac // // @keywords internal // [[Rcpp::export(.unifrac_cpp)]] -Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, +Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree){ // // std::unordered_set to_keep(table.obs_ids.begin(), @@ -64,19 +65,14 @@ Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, su::Assay table = su::Assay(assay); std::string method = "unweighted"; - su::mat_t results = one_off(table, tree, method, false, 1.0, false); + su::mat_t results = su::one_off(table, tree, method, 1.0, false, false); //condensed_form is the main values, returned in result //Sample_ids can be handled with a map? //n_samples, cf_size, is_upper_triangle are single values that can be passed in some other way? - /* - Rcpp::NumericVector unifrac = Rcpp::NumericVector(results.size()); - for( unsigned int i = 0; i < results.cf_size; i++ ){ - unifrac[i] = results.condensed_form[i]; - } - */ + //Rcpp::NumericVector cf = Rcpp::NumericVector(results.cf_size); return Rcpp::List::create(Rcpp::Named("n_samples") = results.n_samples, Rcpp::Named("is_upper_triangle") = results.is_upper_triangle, @@ -84,85 +80,3 @@ Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, Rcpp::Named("c_form") = results.condensed_form); } - - - - - -/* Compute UniFrac - condensed form - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads to use. - * result the resulting distance matrix in condensed form, this is initialized within the method so using ** - * - * one_off returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - * table_empty : the table does not have any entries - */ - -su::mat_t one_off(const su::Assay & table, - const su::BPTree & tree, - std::string unifrac_method, - double alpha, - bool variance_adjust, - bool bypass_tips) { - - //Check that method is valid - pass it as something other than string? - //SET_METHOD(unifrac_method, unknown_method) - - //Stripes relate to matrix calculations - const unsigned int stripe_stop = (table.n_samples + 1) / 2; - - //Originally std::vector of double pointers - this is where the data travels? - su::StripeMap dm_stripes(table.n_samples); - su::StripeMap dm_stripes_total(table.n_samples); - - su::task_parameters task; - - task.tid = 0; - task.start = 0; - task.stop = stripe_stop; - task.bypass_tips = bypass_tips; - task.n_samples = n_samples; - task.g_unifrac_alpha = alpha; - - //Main action - //Calls either unifrac or _vaw depending on variance_adjust - //makes use of std::ref? - //Versions for accelerated and cpu - let's go with cpu for now - //method is "unweighted" by default, let's start with that and see what else may be needed - - //Threads are passed here after being created with a vector - //This does nothing except pass the number of threads, however - su::process_stripes(table, tree_sheared, method, variance_adjust, - dm_stripes, dm_stripes_total, task); - - - //Only use of threading in this version of code was for stripes to condensed form - //Basically each thread calls stripes_to_condensed_form - //Which is just a bunch of binomial calculations - - su::mat_t result; - result->n_samples = table.n_samples; - result->cf_size = su::comb_2(table.n_samples); - result->sample_ids = table.sample_ids; - result->condensed_form = std::vector(su::comb_2(table.n_samples), - 0.0); - result->is_upper_triangle = true; - - return su::stripes_to_condensed_form(dm_stripes, - table.n_samples, - result, - task.start, - task.stop); -} - diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp index 7cd8188e4..fa5ca167a 100644 --- a/src/unifrac_task.cpp +++ b/src/unifrac_task.cpp @@ -1,29 +1,44 @@ +/* + * BSD 3-Clause License + * + * Copyright (c) 2016-2021, UniFrac development team. + * All rights reserved. + * + * See LICENSE file for more details + */ -#include "unifrac_task.hpp" - -#include +#include #include +#include +#include +#include "tree.h" +#include "unifrac_task.h" -void su::UnifracUnweightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { +void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector lengths) { //Parameter finding //Task parameters determine stuff - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; + const uint64_t start_idx = this->task_p.start; + const uint64_t stop_idx = this->task_p.stop; + const uint64_t n_samples = this->task_p.n_samples; const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - + /* // openacc only works well with local variables const uint64_t * const __restrict__ embedded_proportions = this->embedded_proportions; TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - TFloat * const __restrict__ sums = this->sums; + */ - const uint64_t step_size = SUCMP_NM::UnifracUnweightedTask::step_size; + std::vector embedded_proportions = this->embedded_proportions; + std::vector dm_stripes_buf = this->dm_stripes.buf; + std::vector dm_stripes_total_buf = this->dm_stripes_total.buf; + std::vector sums = this->sums; + + const uint64_t step_size = su::UnifracUnweightedTask::step_size; const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up const uint64_t filled_embs_els = filled_embs/64; @@ -38,8 +53,20 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, const TFloat * __ for (uint64_t emb_el=0; emb_el psum = &(sums[emb8<<8]); - const TFloat * __restrict__ pl = &(lengths[emb8*8]); + + //TFloat * __restrict__ psum = &(sums[emb8<<8]); + //const TFloat * __restrict__ pl = &(lengths[emb8*8]); + + std::vector psum = std::vector(256); + std::vector pl = std::vector(8); + + std::copy( std::begin(sums) + (emb8<<8), + std::begin(sums) + (emb8<<8) + 256, + std::begin(psum) ); + + std::copy( std::begin(lengths) + (emb8*8), + std::begin(lengths) + (emb8*8) + 8, + std::begin(pl) ); // compute all the combinations for this block (8-bits total) // psum[0] = 0.0 // +0*pl[0]+0*pl[1]+0*pl[2]+... @@ -55,55 +82,79 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, const TFloat * __ (((b8_i >> 4) & 1) * pl[4]) + (((b8_i >> 5) & 1) * pl[5]) + (((b8_i >> 6) & 1) * pl[6]) + (((b8_i >> 7) & 1) * pl[7]); } + + std::copy(std::begin(psum), std::end(psum), + std::begin(sums) + (emb8<<8)); } } - if (filled_embs_rem>0) { // add also the overflow elements const uint64_t emb_el=filled_embs_els; for (uint64_t sub8=0; sub8<8; sub8++) { // we are summing we have enough buffer in sums const uint64_t emb8 = emb_el*8+sub8; - TFloat * __restrict__ psum = &(sums[emb8<<8]); + + //TFloat * __restrict__ psum = &(sums[emb8<<8]); + + std::vector psum = std::vector(256); + std::copy( std::begin(sums) + (emb8<<8), + std::begin(sums) + (emb8<<8) + 256, + std::begin(psum) ); + // compute all the combinations for this block, set to 0 any past the limit // as above for (uint64_t b8_i=0; b8_i<0x100; b8_i++) { - TFloat val= 0; + double val= 0; for (uint64_t li=(emb8*8); li> (li-(emb8*8))) & 1) * lengths[li]; } psum[b8_i] = val; } + + std::copy(std::begin(psum), std::end(psum), + std::begin(sums) + (emb8<<8)); } } // point of thread for(uint64_t sk = 0; sk < sample_steps ; sk++) { for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + + const uint64_t idx = stripe-start_idx; + + std::vector dm_stripe = this->dm_stripes.dm_stripes.get(idx); + std::vector dm_stripe_total = this->dm_stripes_total.dm_stripes.get(idx); + for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; + const uint64_t k = sk*step_size + ik; // within-stripe index (0:n_samples-1) + //const uint64_t idx = (stripe-start_idx) * n_samples_r; //n_samples_r seems to relate to continuous buffer shenanigans + + //TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; + //TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; + ////TFloat *dm_stripe = dm_stripes[stripe]; + ////TFloat *dm_stripe_total = dm_stripes_total[stripe]; if (k>=n_samples) continue; // past the limit const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound bool did_update = false; - TFloat my_stripe = 0.0; - TFloat my_stripe_total = 0.0; + double my_stripe = 0.0; + double my_stripe_total = 0.0; //This is the main calculation phase for (uint64_t emb_el=0; emb_el psum = std::vector(2048); + std::copy( std::begin(sums) + (emb_el * 2048), + std::begin(sums) + (emb_el * 2048) + 2048, + std::begin(psum) ); uint64_t u1 = embedded_proportions[offset + k]; uint64_t v1 = embedded_proportions[offset + l1]; @@ -141,7 +192,11 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, const TFloat * __ dm_stripe[k] += my_stripe; dm_stripe_total[k] += my_stripe_total; } + } + + this->dm_stripes.dm_stripes.update(idx, dm_stripe); + this->dm_stripes_total.dm_stripes.update(idx, dm_stripe_total); } } } diff --git a/src/unifrac_task.hpp b/src/unifrac_task.h similarity index 79% rename from src/unifrac_task.hpp rename to src/unifrac_task.h index df094171e..05868880e 100644 --- a/src/unifrac_task.hpp +++ b/src/unifrac_task.h @@ -7,24 +7,13 @@ * See LICENSE file for more details */ -#include "task_parameters.hpp" -#include "stripemap.h" - -/** REMOVE **/ -#include -#include -#include -#include -#include -#include - - #ifndef __UNIFRAC_TASKS #define __UNIFRAC_TASKS 1 + +#include "stripemap.h" + // CPUs don't need such a big alignment #define UNIFRAC_BLOCK 16 -#endif - namespace su { @@ -50,74 +39,58 @@ namespace su { - /* - Task parameters - struct of parameters - UnifracTaskVector - vector with special things - dm_stripes: Vector of vectors: Replace with stripemap - task_p - - - */ + // Note: This adds a copy, which is suboptimal // But was the easiest way to get a contiguous buffer // And it does allow for fp32 compute, when desired - //Seems to have a block of unused stuff at the front? - //Accessed via the class UnifracTaskVector { private: - su::StripeMap dm_stripes; const su::task_parameters task_p; public: + su::StripeMap & dm_stripes; const unsigned int start_idx; const unsigned int n_samples; const uint64_t n_samples_r; std::vector buf; - UnifracTaskVector(su::StripeMap _dm_stripes, const su::task_parameters _task_p) - : dm_stripes(_dm_stripes), task_p(_task_p) - , start_idx(task_p->start), n_samples(task_p->n_samples) + UnifracTaskVector(su::StripeMap _dm_stripes, + const su::task_parameters _task_p) + : task_p(_task_p), dm_stripes(_dm_stripes) + , start_idx(task_p.start), n_samples(task_p.n_samples) , n_samples_r(((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK) // round up //buf is just a new array with as many stripes as called for in task_p //n_samples_r tells us how many unifrac_blocks are required for n_samples. //Originally this was a null comparison, we might need to check what it does specifically , buf((dm_stripes.is_empty(start_idx)) ? std::vector() : - std::vector(n_samples_r*(task_p->stop-start_idx), 0.0)) // dm_stripes could be null, in which case keep it null + std::vector(n_samples_r*(task_p.stop-start_idx), 0.0)) // dm_stripes could be null, in which case keep it null { if (!buf.empty()) { - for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { - std::vector dm_stripe = dm_stripes.get(stripe); - //This just returns the specified section of buf, and copies the values from dm_stripe there - std::vector buf_stripe = this->operator[](stripe); - //double * buf_stripe = this->operator[](stripe); - for(unsigned int j=0; j dm_stripe = dm_stripes.get(stripe); + //copy stripe to appropriate segment of buffer + //The stripes themselves have n_samples elements, + //but in the buffer each stripe gets n_samples_r elements? + std::copy(std::begin(dm_stripe), std::end(dm_stripe), + std::begin(buf) + ((stripe-start_idx)*n_samples_r) ); } } } - //idx seems to go in steps of n_samples - //[] returns the first element of a n_samples_r sized chunk, i.e. a stripe? - //Start_idx is determined by task - //It's probably safe to just give them stripe_based access - std::vector& operator[](unsigned int idx) { return buf+((idx-start_idx)*n_samples_r);} - const double * operator[](unsigned int idx) const { return buf+((idx-start_idx)*n_samples_r);} - //Destructor copies the buffer values back into dm_stripe ~UnifracTaskVector() { - double * const ibuf = buf; - if (ibuf != NULL) { - for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { - std::vector dm_stripe = dm_stripes[stripe]; - double * buf_stripe = this->operator[](stripe); - dm_stripes.update(stripe, vec) + if (!buf.empty()) { + for(unsigned int stripe=start_idx; stripe < task_p.stop; stripe++) { + std::vector vec = dm_stripes.get(stripe); + std::copy( std::begin(buf) + ((stripe-start_idx)*n_samples_r), + std::begin(buf) + ((stripe-start_idx+1)*n_samples_r), + std::begin(vec) ); + dm_stripes.update(stripe, vec); } } } @@ -128,6 +101,8 @@ namespace su { }; + + /***********************************************/ @@ -142,37 +117,58 @@ namespace su { su::task_parameters task_p; const unsigned int max_embs; - std::vector embedded_proportions; + std::vector embedded_proportions; //Continuous vector - each stripe has n_samples_r elements, for complex reasons? + //Has at most max_embs stripes - when filled, results stored in task _run() and embeds cleared to continue - UnifracTaskBase(su::StripeMap _dm_stripes, su::StripeMap _dm_stripes_total, - unsigned int _max_embs, su::task_parameters _task_p) - : dm_stripes(_dm_stripes,_task_p), dm_stripes_total(_dm_stripes_total,_task_p), task_p(_task_p) - , max_embs(_max_embs)) - { - uint64_t bsize = dm_stripes.n_samples_r * get_emb_els(max_embs); - embedded_proportions = std::vector(bsize, 0.0) - } + UnifracTaskBase(su::StripeMap _dm_stripes, + su::StripeMap _dm_stripes_total, + unsigned int _max_embs, + su::task_parameters _task_p) + : dm_stripes(_dm_stripes,_task_p), + dm_stripes_total(_dm_stripes_total,_task_p), + task_p(_task_p), + max_embs(_max_embs), + embedded_proportions(initialize_embedded(dm_stripes.n_samples_r, + _max_embs)) + {} virtual ~UnifracTaskBase() {} static unsigned int get_emb_els(unsigned int max_embs); + static std::vector initialize_embedded( + const uint64_t n_samples_r, + unsigned int max_embs ) + { + uint64_t bsize = n_samples_r * get_emb_els(max_embs); + return std::vector(bsize); + } + //Need to return a vector? - void embed_proportions_range(std::vector in, unsigned int start, unsigned int end, unsigned int emb); - void embed_proportions(std::vector in, unsigned int emb) {embed_proportions_range(in,0,dm_stripes.n_samples,emb);} + void embed_proportions_range( + std::vector in, + unsigned int start, + unsigned int end, + unsigned int emb); + + void embed_proportions( + std::vector in, + unsigned int emb) + { + embed_proportions_range(in,0,dm_stripes.n_samples,emb); + } + + // // ===== Internal, do not use directly ======= // // Just copy from one buffer to another - // May convert between fp formats in the process (if TOut!=double) - //out has all the stripes? - //in has just a specific section? std::vector embed_proportions_range_straight( - std::vector out, - std::vector in, + std::vector out, + std::vector in, unsigned int start, unsigned int end, unsigned int emb) const @@ -198,37 +194,41 @@ namespace su { } + // packed bool // Compute (in[:]>0) on each element, and store only the boolean bit. // The output values are stored in a multi-byte format, one bit per emb index, // so it will likely take multiple passes to store all the values // // Note: assumes we are processing emb in increasing order, starting from 0 - template void embed_proportions_range_bool( - std::vector out, + + //Only used with uint64_t + std::vector embed_proportions_range_bool( + std::vector out, std::vector in, unsigned int start, unsigned int end, unsigned int emb) const { - const unsigned int n_packed = sizeof(TOut)*8;// e.g. 32 for unit32_t + const unsigned int n_packed = sizeof(uint64_t)*8; const unsigned int n_samples = dm_stripes.n_samples; const uint64_t n_samples_r = dm_stripes.n_samples_r; + // The output values are stored in a multi-byte format, one bit per emb index // Compute the element to store the bit into, as well as whichbit in that element unsigned int emb_block = emb/n_packed; // beginning of the element block unsigned int emb_bit = emb%n_packed; // bit inside the elements const uint64_t offset = emb_block * n_samples_r; - if (emb_bit==0) { + if (emb_bit == 0) { // assign for emb_bit==0, so it clears the other bits // assumes we processing emb in increasing order, starting from 0 for(unsigned int i = start; i < end; i++) { - out[offset + i] = (in[i-start] > 0); + out[offset + i] = (in[i - start] > 0); } - if (end==n_samples) { + if (end == n_samples) { // avoid NaNs for(unsigned int i = n_samples; i < n_samples_r; i++) { out[offset + i] = 0; @@ -237,24 +237,49 @@ namespace su { } else { // just update my bit for(unsigned int i = start; i < end; i++) { - out[offset + i] |= (TOut(in[i-start] > 0) << emb_bit); + out[offset + i] |= (uint64_t(in[i-start] > 0) << emb_bit); } // the rest of the els are already OK } + return out; } }; - // straight embeded_proportions - template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_straight(embedded_proportions,in,start,end,emb);} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return max_embs;} - //packed bool embeded_proportions - template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+31)/32;} - template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+63)/64;} + template<> inline void UnifracTaskBase::embed_proportions_range( + std::vector in, + unsigned int start, + unsigned int end, + unsigned int emb ) + { + embedded_proportions = embed_proportions_range_straight(embedded_proportions,in,start,end,emb); + } + + template<> inline unsigned int UnifracTaskBase::get_emb_els( + unsigned int max_embs ) + { + return max_embs; + } + + + + + template<> inline void UnifracTaskBase::embed_proportions_range( + std::vector in, + unsigned int start, + unsigned int end, + unsigned int emb ) + { + embedded_proportions = embed_proportions_range_bool(embedded_proportions,in,start,end,emb); + } + + template<> inline unsigned int UnifracTaskBase::get_emb_els( + unsigned int max_embs ) + { + return (max_embs+63)/64; + } @@ -324,7 +349,7 @@ namespace su { virtual void run(unsigned int filled_embs, std::vector length) {_run(filled_embs, length);} - void _run(unsigned int filled_embs, std::vector length); + void _run(unsigned int filled_embs, std::vector lengths); private: std::vector sums; // temp buffer }; @@ -556,16 +581,16 @@ namespace su { // // template // class UnifracVawGeneralizedTask : public UnifracVawTask { - public: - UnifracVawGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - const TFloat * _sample_total_counts, - unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - }; + // public: + // UnifracVawGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, + // const TFloat * _sample_total_counts, + // unsigned int _max_embs, const su::task_parameters* _task_p) + // : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} + // + // virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} + // + // void _run(unsigned int filled_embs, const TFloat * __restrict__ length); + // }; } From 8b65f6ecebe5c554ea973e2aafd42795c8975e44 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Tue, 5 May 2026 01:28:21 +0300 Subject: [PATCH 32/48] Pass StripeMap by reference to fix crashing --- src/stripemap.cpp | 10 +++ src/unifrac.cpp | 187 +++++++++++++++++++++++++------------------ src/unifrac.h | 8 +- src/unifrac_R.cpp | 4 + src/unifrac_task.cpp | 44 ++++++++++ src/unifrac_task.h | 10 +-- 6 files changed, 175 insertions(+), 88 deletions(-) diff --git a/src/stripemap.cpp b/src/stripemap.cpp index 940fd3682..b4a907245 100644 --- a/src/stripemap.cpp +++ b/src/stripemap.cpp @@ -11,6 +11,10 @@ #include "assay.h" #include "stripemap.h" +//for sleep +#include +#include + #include using namespace su; @@ -41,6 +45,12 @@ void StripeMap::clear(uint32_t i){ } void StripeMap::update(uint32_t node, std::vector vec){ + + Rcpp::Rcout << "node: "<< node << "\n"; + Rcpp::Rcout << "n_stripes: "<< n_stripes << "\n"; + Rcpp::Rcout << "vecsize: "<< vecsize << "\n"; + Rcpp::Rcout << "vec size: "<< vec.size() << "\n"; + stripe_map[node] = vec; } diff --git a/src/unifrac.cpp b/src/unifrac.cpp index 231f5ae3c..ba8b88d7b 100644 --- a/src/unifrac.cpp +++ b/src/unifrac.cpp @@ -57,6 +57,8 @@ su::mat_t su::one_off(const su::Assay & table, bool variance_adjust, bool bypass_tips) { + Rcpp::Rcout << "Start one_off\n"; + //Check that method is valid - pass it as something other than string? su::Method method = set_method(unifrac_method); @@ -97,6 +99,8 @@ su::mat_t su::one_off(const su::Assay & table, task, variance_adjust); + Rcpp::Rcout << "unifrac done\n"; + //Only use of std::thread in this version of code was for stripes to condensed form //Basically each thread calls stripes_to_condensed_form //Which is just a bunch of binomial calculations @@ -111,12 +115,93 @@ su::mat_t su::one_off(const su::Assay & table, task.start, task.stop); + Rcpp::Rcout << "one_off done\n"; + return result; } +void su::unifrac(const su::Assay &table, + const su::BPTree &tree, + su::Method unifrac_method, + su::StripeMap & dm_stripes, + su::StripeMap & dm_stripes_total, + const su::task_parameters task_p, + bool variance_adjust) +{ + + Rcpp::Rcout << "Start unifrac\n"; + + if(variance_adjust) + { + /* + switch(unifrac_method) { + case su::unweighted: + unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_normalized: + unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_unnormalized: + unifrac_vawTT,double>( table, tree, false, dm_stripes,dm_stripes_total,task_p); + break; + case su::generalized: + unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::unweighted_fp32: + unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_normalized_fp32: + unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + case su::weighted_unnormalized_fp32: + unifrac_vawTT,float >( table, tree, false, dm_stripes,dm_stripes_total,task_p); + break; + case su::generalized_fp32: + unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); + break; + default: + fprintf(stderr, "Unknown unifrac task\n"); + exit(1); + break; + } + */ + } + else + { + switch(unifrac_method) + { + case su::unweighted: + unifracTT( + table, tree, true, dm_stripes,dm_stripes_total, + task_p ); + break; + /*case su::weighted_normalized: + unifracTT,double>( + table, tree, true, dm_stripes,dm_stripes_total, + task_p ); + break; + case su::weighted_unnormalized: + unifracTT, + double>(table, tree, false, dm_stripes, + dm_stripes_total, task_p ); + break; + case su::generalized: + unifracTT,double>( + table, tree, true, dm_stripes,dm_stripes_total, + task_p ); + break; + */ + default: + fprintf(stderr, "Unknown unifrac task\n"); + exit(1); + break; + } + } +} + @@ -125,11 +210,13 @@ template inline void su::unifracTT(const su::Assay & table, const su::BPTree & tree, const bool want_total, - su::StripeMap dm_stripes, - su::StripeMap dm_stripes_total, + su::StripeMap & dm_stripes, + su::StripeMap & dm_stripes_total, const su::task_parameters & task_p) { + Rcpp::Rcout << "Start unifracTT\n"; + if(table.n_samples != task_p.n_samples) { fprintf(stderr, "Task and table n_samples not equal\n"); exit(EXIT_FAILURE); @@ -144,8 +231,14 @@ inline void su::unifracTT(const su::Assay & table, const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; + + + Rcpp::Rcout << "Start taskObj\n"; + TaskT taskObj(dm_stripes, dm_stripes_total, max_emb, task_p); + Rcpp::Rcout << "taskObj done\n"; + std::vector lengths = std::vector(max_emb); /* @@ -193,6 +286,8 @@ inline void su::unifracTT(const su::Assay & table, * (see C) but that is small over large N. */ + Rcpp::Rcout << "Start calcs\n"; + unsigned int k = 0; // index in tree const unsigned int max_k = (tree.nparens / 2) - 1; @@ -234,7 +329,10 @@ inline void su::unifracTT(const su::Assay & table, //store the proportions inside the taskobject's continuous buffer //Shouldn't modify node_proportions std::vector node_proportions = propmap.get(node); + + //Rcpp::Rcout << "start embed_proportions_range\n"; taskObj.embed_proportions_range(node_proportions, tstart, tend, my_filled_emb); + //Rcpp::Rcout << "embed_proportions_range done\n"; my_filled_emb++; } @@ -243,13 +341,18 @@ inline void su::unifracTT(const su::Assay & table, //This is used to keep track of filled embeds over different threads? //Does nothing without openacc //taskObj.sync_embedded_proportions(filled_emb); - + + Rcpp::Rcout << "start taskObj._run\n"; taskObj._run(filled_emb,lengths); + Rcpp::Rcout << "taskObj._run done\n"; filled_emb=0; } + Rcpp::Rcout << "calcs done\n"; + + //I suppose want_total is used if you want the results as a percentage of the total? if(want_total) { const uint64_t start_idx = task_p.start; @@ -282,6 +385,8 @@ inline void su::unifracTT(const su::Assay & table, */ } } + + Rcpp::Rcout << "unifracTT done\n"; } @@ -319,82 +424,6 @@ std::vector su::set_proportions_range(const su::BPTree & tree, -void su::unifrac(const su::Assay &table, - const su::BPTree &tree, - su::Method unifrac_method, - su::StripeMap &dm_stripes, - su::StripeMap &dm_stripes_total, - const su::task_parameters task_p, - bool variance_adjust) -{ - if(variance_adjust) - { - /* - switch(unifrac_method) { - case su::unweighted: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized: - unifrac_vawTT,double>( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::unweighted_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized_fp32: - unifrac_vawTT,float >( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - default: - fprintf(stderr, "Unknown unifrac task\n"); - exit(1); - break; - } - */ - } - else - { - switch(unifrac_method) - { - case su::unweighted: - unifracTT( - table, tree, true, dm_stripes,dm_stripes_total, - task_p ); - break; - /*case su::weighted_normalized: - unifracTT,double>( - table, tree, true, dm_stripes,dm_stripes_total, - task_p ); - break; - case su::weighted_unnormalized: - unifracTT, - double>(table, tree, false, dm_stripes, - dm_stripes_total, task_p ); - break; - case su::generalized: - unifracTT,double>( - table, tree, true, dm_stripes,dm_stripes_total, - task_p ); - break; - */ - default: - fprintf(stderr, "Unknown unifrac task\n"); - exit(1); - break; - } - } -} - std::vector su::stripes_to_condensed_form(su::StripeMap stripes, uint32_t n, unsigned int start, diff --git a/src/unifrac.h b/src/unifrac.h index a69cf0d3e..bcbb69b9a 100644 --- a/src/unifrac.h +++ b/src/unifrac.h @@ -70,8 +70,8 @@ void unifrac(const su::Assay &table, const su::BPTree &tree, su::Method unifrac_method, - su::StripeMap &dm_stripes, - su::StripeMap &dm_stripes_total, + su::StripeMap & dm_stripes, + su::StripeMap & dm_stripes_total, const su::task_parameters task_p, bool variance_adjust); @@ -91,8 +91,8 @@ inline void unifracTT(const su::Assay & table, const su::BPTree & tree, const bool want_total, - su::StripeMap dm_stripes, - su::StripeMap dm_stripes_total, + su::StripeMap & dm_stripes, + su::StripeMap & dm_stripes_total, const su::task_parameters & task_p); inline uint64_t comb_2(uint64_t N) { diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index 595849985..95e5c4ab5 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -65,8 +65,12 @@ Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, su::Assay table = su::Assay(assay); std::string method = "unweighted"; + Rcpp::Rcout << "Start\n"; + su::mat_t results = su::one_off(table, tree, method, 1.0, false, false); + Rcpp::Rcout << "All done\n"; + //condensed_form is the main values, returned in result //Sample_ids can be handled with a map? //n_samples, cf_size, is_upper_triangle are single values that can be passed in some other way? diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp index fa5ca167a..54fc17834 100644 --- a/src/unifrac_task.cpp +++ b/src/unifrac_task.cpp @@ -12,11 +12,18 @@ #include #include +//for sleep +#include +#include + #include "tree.h" #include "unifrac_task.h" void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector lengths) { + Rcpp::Rcout << "start UnifracUnweightedTask::_run\n"; + sleep(2); + //Parameter finding //Task parameters determine stuff @@ -38,6 +45,9 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector dm_stripes_total_buf = this->dm_stripes_total.buf; std::vector sums = this->sums; + Rcpp::Rcout << "buffers done\n"; + sleep(2); + const uint64_t step_size = su::UnifracUnweightedTask::step_size; const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up @@ -47,6 +57,11 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector0) { // add also the overflow elements const uint64_t emb_el=filled_embs_els; @@ -117,17 +136,32 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector dm_stripe = this->dm_stripes.dm_stripes.get(idx); std::vector dm_stripe_total = this->dm_stripes_total.dm_stripes.get(idx); for(uint64_t ik = 0; ik < step_size ; ik++) { + + Rcpp::Rcout << " ik: " << ik << "\n"; + //sleep(2); + const uint64_t k = sk*step_size + ik; // within-stripe index (0:n_samples-1) //const uint64_t idx = (stripe-start_idx) * n_samples_r; //n_samples_r seems to relate to continuous buffer shenanigans @@ -148,6 +182,8 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vectordm_stripes.dm_stripes.update(idx, dm_stripe); this->dm_stripes_total.dm_stripes.update(idx, dm_stripe_total); + } } + Rcpp::Rcout << "stripes done\n"; + sleep(2); + + Rcpp::Rcout << "UnifracUnweightedTask::_run done\n"; + sleep(2); } // diff --git a/src/unifrac_task.h b/src/unifrac_task.h index 05868880e..0b24fb59f 100644 --- a/src/unifrac_task.h +++ b/src/unifrac_task.h @@ -56,7 +56,7 @@ namespace su { const uint64_t n_samples_r; std::vector buf; - UnifracTaskVector(su::StripeMap _dm_stripes, + UnifracTaskVector(su::StripeMap & _dm_stripes, const su::task_parameters _task_p) : task_p(_task_p), dm_stripes(_dm_stripes) , start_idx(task_p.start), n_samples(task_p.n_samples) @@ -120,8 +120,8 @@ namespace su { std::vector embedded_proportions; //Continuous vector - each stripe has n_samples_r elements, for complex reasons? //Has at most max_embs stripes - when filled, results stored in task _run() and embeds cleared to continue - UnifracTaskBase(su::StripeMap _dm_stripes, - su::StripeMap _dm_stripes_total, + UnifracTaskBase(su::StripeMap & _dm_stripes, + su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) : dm_stripes(_dm_stripes,_task_p), @@ -313,7 +313,7 @@ namespace su { public: - UnifracTask(su::StripeMap _dm_stripes, su::StripeMap _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) + UnifracTask(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) : UnifracTaskBase(_dm_stripes, _dm_stripes_total, _max_embs, _task_p) {} virtual ~UnifracTask() {} @@ -338,7 +338,7 @@ namespace su { static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_BOOL; // Note: _max_emb MUST be multiple of 64 - UnifracUnweightedTask(su::StripeMap _dm_stripes, su::StripeMap _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) + UnifracUnweightedTask(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) { const unsigned int bsize = _max_embs*32; From 025dcf516c03e418fdba306bb77db0b93d1b8746 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Wed, 6 May 2026 11:01:39 +0300 Subject: [PATCH 33/48] Unweighted now produces accurate results However, speed is slower than current implementation --- src/assay.h | 2 +- src/propmap.cpp | 37 +++++++++++++++++++ src/propmap.h | 15 ++++++++ src/stripemap.cpp | 13 +------ src/stripemap.h | 1 + src/unifrac.cpp | 88 ++++---------------------------------------- src/unifrac.h | 14 +------ src/unifrac_R.cpp | 11 ++++-- src/unifrac_task.cpp | 81 ++++++---------------------------------- src/unifrac_task.h | 20 ++++------ 10 files changed, 90 insertions(+), 192 deletions(-) diff --git a/src/assay.h b/src/assay.h index 77c80f86b..307cf41df 100644 --- a/src/assay.h +++ b/src/assay.h @@ -58,7 +58,7 @@ class Assay { private: Rcpp::NumericMatrix table; // Access to raw sample counts in R's memory - + std::vector get_sample_counts(); /* At construction, lookups mapping IDs -> index position within an diff --git a/src/propmap.cpp b/src/propmap.cpp index d08e75f03..c2a822b2b 100644 --- a/src/propmap.cpp +++ b/src/propmap.cpp @@ -85,4 +85,41 @@ std::vector su::set_proportions(const BPTree &tree, ps.update(node, props); return(props); +} + + +std::vector su::set_proportions_range(const su::BPTree & tree, + uint32_t node, + const su::Assay & table, + unsigned int start, + unsigned int end, + PropMap & pm, + bool normalize) { + const unsigned int els = end-start; + std::vector props = std::vector(); + if(tree.isleaf(node)) { + std::string leaf = tree.names[node]; + props = table.get_obs_data_range(leaf, start, end, normalize); + } else { + unsigned int current = tree.leftchild(node); + unsigned int right = tree.rightchild(node); + + for( unsigned int i = 0; i < els; i++ ){ + props.push_back(0); + } + + while(current <= right && current != 0) { + std::vector vec = pm.get(current); // pull from prop map + pm.clear(current); // remove from prop map, place back on stack + + for(unsigned int i = 0; i < els; i++){ + props[i] = props[i] + vec[i]; + } + + current = tree.rightsibling(current); + } + } + + pm.update(node, props); + return props; } \ No newline at end of file diff --git a/src/propmap.h b/src/propmap.h index ec936d194..a82275a9a 100644 --- a/src/propmap.h +++ b/src/propmap.h @@ -7,6 +7,8 @@ * See LICENSE file for more details */ + + #ifndef __FAITH_PROPMAP #define __FAITH_PROPMAP 1 @@ -18,6 +20,7 @@ #include "assay.h" namespace su { + class PropMap { public: PropMap(uint32_t vecsize); @@ -35,6 +38,18 @@ std::vector set_proportions(const BPTree &tree, uint32_t node, const Assay &table, PropMap &ps, bool normalize = true); + +// Sets proportion range +// Data is stored to props -> make return vector +// PropMap needs to be modified, thus passed by reference +std::vector set_proportions_range(const su::BPTree & tree, + uint32_t node, + const su::Assay & table, + unsigned int start, + unsigned int end, + PropMap & pm, + bool normalize = true); } + #endif /* __FAITH_PROPMAP */ diff --git a/src/stripemap.cpp b/src/stripemap.cpp index b4a907245..e861a8963 100644 --- a/src/stripemap.cpp +++ b/src/stripemap.cpp @@ -11,10 +11,6 @@ #include "assay.h" #include "stripemap.h" -//for sleep -#include -#include - #include using namespace su; @@ -29,8 +25,7 @@ StripeMap::StripeMap(uint32_t n_samples) } } -StripeMap::~StripeMap() { -} +StripeMap::~StripeMap() {} std::vector StripeMap::get(uint32_t i){ if( stripe_map.count(i) > 0 ){ @@ -45,12 +40,6 @@ void StripeMap::clear(uint32_t i){ } void StripeMap::update(uint32_t node, std::vector vec){ - - Rcpp::Rcout << "node: "<< node << "\n"; - Rcpp::Rcout << "n_stripes: "<< n_stripes << "\n"; - Rcpp::Rcout << "vecsize: "<< vecsize << "\n"; - Rcpp::Rcout << "vec size: "<< vec.size() << "\n"; - stripe_map[node] = vec; } diff --git a/src/stripemap.h b/src/stripemap.h index 94467acaa..a435df970 100644 --- a/src/stripemap.h +++ b/src/stripemap.h @@ -7,6 +7,7 @@ * See LICENSE file for more details */ + #ifndef __FAITH_STRIPEMAP #define __FAITH_STRIPEMAP 1 diff --git a/src/unifrac.cpp b/src/unifrac.cpp index ba8b88d7b..3375fc068 100644 --- a/src/unifrac.cpp +++ b/src/unifrac.cpp @@ -26,6 +26,7 @@ + su::Method su::set_method(std::string requested_method) { if(requested_method == "unweighted") return unweighted; @@ -57,8 +58,6 @@ su::mat_t su::one_off(const su::Assay & table, bool variance_adjust, bool bypass_tips) { - Rcpp::Rcout << "Start one_off\n"; - //Check that method is valid - pass it as something other than string? su::Method method = set_method(unifrac_method); @@ -99,8 +98,6 @@ su::mat_t su::one_off(const su::Assay & table, task, variance_adjust); - Rcpp::Rcout << "unifrac done\n"; - //Only use of std::thread in this version of code was for stripes to condensed form //Basically each thread calls stripes_to_condensed_form //Which is just a bunch of binomial calculations @@ -115,8 +112,6 @@ su::mat_t su::one_off(const su::Assay & table, task.start, task.stop); - Rcpp::Rcout << "one_off done\n"; - return result; } @@ -132,8 +127,6 @@ void su::unifrac(const su::Assay &table, bool variance_adjust) { - Rcpp::Rcout << "Start unifrac\n"; - if(variance_adjust) { /* @@ -177,6 +170,7 @@ void su::unifrac(const su::Assay &table, unifracTT( table, tree, true, dm_stripes,dm_stripes_total, task_p ); + break; /*case su::weighted_normalized: unifracTT,double>( @@ -215,8 +209,6 @@ inline void su::unifracTT(const su::Assay & table, const su::task_parameters & task_p) { - Rcpp::Rcout << "Start unifracTT\n"; - if(table.n_samples != task_p.n_samples) { fprintf(stderr, "Task and table n_samples not equal\n"); exit(EXIT_FAILURE); @@ -225,20 +217,13 @@ inline void su::unifracTT(const su::Assay & table, const unsigned int n_samples = task_p.n_samples; const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up - //su::PropStackMulti propstack_multi(table.n_samples); su::PropMap propmap(table.n_samples); const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; - - - Rcpp::Rcout << "Start taskObj\n"; - TaskT taskObj(dm_stripes, dm_stripes_total, max_emb, task_p); - Rcpp::Rcout << "taskObj done\n"; - std::vector lengths = std::vector(max_emb); /* @@ -286,8 +271,6 @@ inline void su::unifracTT(const su::Assay & table, * (see C) but that is small over large N. */ - Rcpp::Rcout << "Start calcs\n"; - unsigned int k = 0; // index in tree const unsigned int max_k = (tree.nparens / 2) - 1; @@ -300,7 +283,7 @@ inline void su::unifracTT(const su::Assay & table, // ck = 0 // chunk the progress to maximize cache reuse const unsigned int tstart = 0; - const unsigned int tend = 0; // end of propstack? + const unsigned int tend = n_samples; // end of propstack? unsigned int my_filled_emb = 0; unsigned int my_k=k_start; @@ -312,7 +295,8 @@ inline void su::unifracTT(const su::Assay & table, //su::set_proportions_range(node_proportions, tree, node, table, tstart, tend, propstack); //calculate proportions range for given node - su::set_proportions_range(tree, node, table, tstart, tend, propmap); + std::vector node_proportions = su::set_proportions_range(tree, node, table, tstart, tend, propmap); + //propstack pop ERASES any existing vector for node and gives a blank one //creates memory leaks if node isn't pushed before popping? @@ -326,13 +310,7 @@ inline void su::unifracTT(const su::Assay & table, lengths[filled_emb] = tree.lengths[node]; filled_emb++; - //store the proportions inside the taskobject's continuous buffer - //Shouldn't modify node_proportions - std::vector node_proportions = propmap.get(node); - - //Rcpp::Rcout << "start embed_proportions_range\n"; taskObj.embed_proportions_range(node_proportions, tstart, tend, my_filled_emb); - //Rcpp::Rcout << "embed_proportions_range done\n"; my_filled_emb++; } @@ -342,89 +320,37 @@ inline void su::unifracTT(const su::Assay & table, //Does nothing without openacc //taskObj.sync_embedded_proportions(filled_emb); - Rcpp::Rcout << "start taskObj._run\n"; taskObj._run(filled_emb,lengths); - Rcpp::Rcout << "taskObj._run done\n"; filled_emb=0; } - - Rcpp::Rcout << "calcs done\n"; - - //I suppose want_total is used if you want the results as a percentage of the total? if(want_total) { const uint64_t start_idx = task_p.start; const uint64_t stop_idx = task_p.stop; for(uint64_t i = start_idx; i < stop_idx; i++){ - /* - std::vector dm_stripes_buf = std::vector ; - std::vector dm_stripes_total_buf = taskObj.dm_stripes_total.get(idx); - std::copy(std::begin(taskObj.dm_stripes.buf), - std::end(taskObj.dm_stripes.buf), - std::begin(dm_stripes_buf) + (emb8<<8)); - */ - std::vector dm_stripes_buf = taskObj.dm_stripes.buf; std::vector dm_stripes_total_buf = taskObj.dm_stripes_total.buf; for(uint64_t j = 0; j < n_samples; j++) { - uint64_t idx = (i-start_idx)*n_samples_r+j; + uint64_t idx = ((i-start_idx)*n_samples_r)+j; dm_stripes_buf[idx] = dm_stripes_buf[idx]/dm_stripes_total_buf[idx]; } taskObj.dm_stripes.buf = dm_stripes_buf; - - /* - taskObj.dm_stripes.update(idx, dm_stripes_buf); - std::copy(std::begin(dm_stripes_buf), - std::end(dm_stripes_buf), - std::begin(taskObj.dm_stripes.buf) + ); - */ } } - - Rcpp::Rcout << "unifracTT done\n"; } -std::vector su::set_proportions_range(const su::BPTree & tree, - uint32_t node, - const su::Assay & table, - unsigned int start, - unsigned int end, - PropMap & pm, - bool normalize) { - const unsigned int els = end-start; - std::vector props = std::vector(els, 0.0); - if(tree.isleaf(node)) { - props = table.get_obs_data_range(tree.names[node], start, end, normalize); - } else { - const unsigned int right = tree.rightchild(node); - unsigned int current = tree.leftchild(node); - - while(current <= right && current != 0) { - std::vector vec = pm.get(current); // pull from prop map - pm.clear(current); // remove from prop map, place back on stack - - for(unsigned int i = 0; i < els; i++) - props[i] += vec[i]; - - current = tree.rightsibling(current); - } - } - pm.update(node, props); - return props; -} - -std::vector su::stripes_to_condensed_form(su::StripeMap stripes, +std::vector su::stripes_to_condensed_form(su::StripeMap & stripes, uint32_t n, unsigned int start, unsigned int stop) { diff --git a/src/unifrac.h b/src/unifrac.h index bcbb69b9a..09e9d96c7 100644 --- a/src/unifrac.h +++ b/src/unifrac.h @@ -30,6 +30,7 @@ std::vector sample_ids; } mat_t; + enum Method {unweighted, weighted_normalized, weighted_unnormalized, @@ -75,17 +76,6 @@ const su::task_parameters task_p, bool variance_adjust); - // Sets proportion range - // Data is stored to props -> make return vector - // PropMap needs to be modified, thus passed by reference - std::vector set_proportions_range(const su::BPTree & tree, - uint32_t node, - const su::Assay & table, - unsigned int start, - unsigned int end, - PropMap & pm, - bool normalize = true); - // Works the vectors template inline void unifracTT(const su::Assay & table, @@ -120,7 +110,7 @@ } // Stripes to condensed form for the results - std::vector stripes_to_condensed_form(su::StripeMap stripes, + std::vector stripes_to_condensed_form(su::StripeMap & stripes, uint32_t n, unsigned int start, unsigned int stop); diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index 95e5c4ab5..2cd0dc0cf 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -20,12 +20,15 @@ #include "unifrac.h" + + // Calculate Unifrac // // @keywords internal // [[Rcpp::export(.unifrac_cpp)]] Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree){ + // // std::unordered_set to_keep(table.obs_ids.begin(), // table.obs_ids.end()); @@ -65,17 +68,17 @@ Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, su::Assay table = su::Assay(assay); std::string method = "unweighted"; - Rcpp::Rcout << "Start\n"; + std::unordered_set to_keep(table.obs_ids.begin(), + table.obs_ids.end()); - su::mat_t results = su::one_off(table, tree, method, 1.0, false, false); + su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - Rcpp::Rcout << "All done\n"; + su::mat_t results = su::one_off(table, tree_sheared, method, 1.0, false, false); //condensed_form is the main values, returned in result //Sample_ids can be handled with a map? //n_samples, cf_size, is_upper_triangle are single values that can be passed in some other way? - //Rcpp::NumericVector cf = Rcpp::NumericVector(results.cf_size); return Rcpp::List::create(Rcpp::Named("n_samples") = results.n_samples, diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp index 54fc17834..869cf759a 100644 --- a/src/unifrac_task.cpp +++ b/src/unifrac_task.cpp @@ -12,18 +12,11 @@ #include #include -//for sleep -#include -#include - #include "tree.h" #include "unifrac_task.h" void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector lengths) { - Rcpp::Rcout << "start UnifracUnweightedTask::_run\n"; - sleep(2); - //Parameter finding //Task parameters determine stuff @@ -41,12 +34,7 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector embedded_proportions = this->embedded_proportions; - std::vector dm_stripes_buf = this->dm_stripes.buf; - std::vector dm_stripes_total_buf = this->dm_stripes_total.buf; - std::vector sums = this->sums; - Rcpp::Rcout << "buffers done\n"; - sleep(2); const uint64_t step_size = su::UnifracUnweightedTask::step_size; const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up @@ -56,13 +44,6 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector psum = std::vector(256); std::vector pl = std::vector(8); - std::copy( std::begin(sums) + (emb8<<8), - std::begin(sums) + (emb8<<8) + 256, + std::copy( std::begin(this->sums) + (emb8<<8), + std::begin(this->sums) + (emb8<<8) + 256, std::begin(psum) ); std::copy( std::begin(lengths) + (emb8*8), @@ -99,15 +80,10 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vectorsums) + (emb8<<8)); } } - Rcpp::Rcout << "pre-compute done\n"; - sleep(2); - Rcpp::Rcout << "start overflow elements\n"; - sleep(2); - if (filled_embs_rem>0) { // add also the overflow elements const uint64_t emb_el=filled_embs_els; for (uint64_t sub8=0; sub8<8; sub8++) { @@ -117,8 +93,8 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector psum = std::vector(256); - std::copy( std::begin(sums) + (emb8<<8), - std::begin(sums) + (emb8<<8) + 256, + std::copy( std::begin(this->sums) + (emb8<<8), + std::begin(this->sums) + (emb8<<8) + 256, std::begin(psum) ); @@ -133,37 +109,15 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vectorsums) + (emb8<<8)); } } - Rcpp::Rcout << "overflow elements done\n"; - sleep(2); - - //problem occurs here - Rcpp::Rcout << "start stripes\n"; - sleep(2); // point of thread for(uint64_t sk = 0; sk < sample_steps ; sk++) { - Rcpp::Rcout << "sk: " << sk << "\n"; - //sleep(2); - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - - Rcpp::Rcout << " stripe: " << stripe << "\n"; - //sleep(2); - - const uint64_t idx = stripe-start_idx; - - std::vector dm_stripe = this->dm_stripes.dm_stripes.get(idx); - std::vector dm_stripe_total = this->dm_stripes_total.dm_stripes.get(idx); - for(uint64_t ik = 0; ik < step_size ; ik++) { - - Rcpp::Rcout << " ik: " << ik << "\n"; - //sleep(2); - const uint64_t k = sk*step_size + ik; // within-stripe index (0:n_samples-1) - //const uint64_t idx = (stripe-start_idx) * n_samples_r; //n_samples_r seems to relate to continuous buffer shenanigans + const uint64_t idx = (stripe-start_idx) * n_samples_r; //n_samples_r seems to relate to continuous buffer shenanigans //TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; //TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; @@ -182,14 +136,12 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector psum = std::vector(2048); - std::copy( std::begin(sums) + (emb_el * 2048), - std::begin(sums) + (emb_el * 2048) + 2048, + std::copy( std::begin(this->sums) + (emb_el * 2048), + std::begin(this->sums) + (emb_el * 2048) + 2048, std::begin(psum) ); uint64_t u1 = embedded_proportions[offset + k]; @@ -197,8 +149,6 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vectordm_stripes.buf[idx + k] += my_stripe; + this->dm_stripes_total.buf[idx + k] += my_stripe_total; } } - - this->dm_stripes.dm_stripes.update(idx, dm_stripe); - this->dm_stripes_total.dm_stripes.update(idx, dm_stripe_total); - } } - Rcpp::Rcout << "stripes done\n"; - sleep(2); - - Rcpp::Rcout << "UnifracUnweightedTask::_run done\n"; - sleep(2); } // diff --git a/src/unifrac_task.h b/src/unifrac_task.h index 0b24fb59f..189e7ec92 100644 --- a/src/unifrac_task.h +++ b/src/unifrac_task.h @@ -37,10 +37,6 @@ namespace su { double g_unifrac_alpha; // generalized unifrac alpha }; - - - - // Note: This adds a copy, which is suboptimal // But was the easiest way to get a contiguous buffer // And it does allow for fp32 compute, when desired @@ -80,7 +76,7 @@ namespace su { } } } - + //Destructor copies the buffer values back into dm_stripe ~UnifracTaskVector() { @@ -88,7 +84,7 @@ namespace su { for(unsigned int stripe=start_idx; stripe < task_p.stop; stripe++) { std::vector vec = dm_stripes.get(stripe); std::copy( std::begin(buf) + ((stripe-start_idx)*n_samples_r), - std::begin(buf) + ((stripe-start_idx+1)*n_samples_r), + std::begin(buf) + ((stripe-start_idx)*n_samples_r) + n_samples, std::begin(vec) ); dm_stripes.update(stripe, vec); } @@ -146,13 +142,13 @@ namespace su { //Need to return a vector? void embed_proportions_range( - std::vector in, + const std::vector & in, unsigned int start, unsigned int end, unsigned int emb); void embed_proportions( - std::vector in, + const std::vector & in, unsigned int emb) { embed_proportions_range(in,0,dm_stripes.n_samples,emb); @@ -168,7 +164,7 @@ namespace su { std::vector embed_proportions_range_straight( std::vector out, - std::vector in, + const std::vector & in, unsigned int start, unsigned int end, unsigned int emb) const @@ -205,7 +201,7 @@ namespace su { //Only used with uint64_t std::vector embed_proportions_range_bool( std::vector out, - std::vector in, + const std::vector & in, unsigned int start, unsigned int end, unsigned int emb) const @@ -249,7 +245,7 @@ namespace su { template<> inline void UnifracTaskBase::embed_proportions_range( - std::vector in, + const std::vector & in, unsigned int start, unsigned int end, unsigned int emb ) @@ -267,7 +263,7 @@ namespace su { template<> inline void UnifracTaskBase::embed_proportions_range( - std::vector in, + const std::vector & in, unsigned int start, unsigned int end, unsigned int emb ) From 46e35199a43a033611ba81f3a58762f2bf076bab Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Wed, 6 May 2026 13:52:19 +0300 Subject: [PATCH 34/48] Revome unnecessary copying for speed Unweighted is now significantly faster and less memory intensive than the existing implementation --- src/propmap.cpp | 12 +----- src/unifrac.cpp | 13 +++--- src/unifrac_R.cpp | 56 ++++++++++---------------- src/unifrac_task.cpp | 94 ++++++++++++++++++++++++-------------------- 4 files changed, 81 insertions(+), 94 deletions(-) diff --git a/src/propmap.cpp b/src/propmap.cpp index c2a822b2b..1964e2664 100644 --- a/src/propmap.cpp +++ b/src/propmap.cpp @@ -54,7 +54,7 @@ std::vector su::set_proportions(const BPTree &tree, const Assay &table, PropMap &ps, bool normalize){ - std::vector props = std::vector(); + std::vector props = std::vector(table.n_samples, 0.0); if( tree.isleaf(node) ){ std::string leaf = tree.names[node]; props = table.get_obs_data(leaf); // get row for the specified node @@ -67,10 +67,6 @@ std::vector su::set_proportions(const BPTree &tree, unsigned int current = tree.leftchild(node); unsigned int right = tree.rightchild(node); - for( unsigned int i = 0; i < table.n_samples; i++ ){ - props.push_back(0); - } - while( current <= right && current != 0 ){ std::vector vec = ps.get(current); // Pull from prop map ps.clear(current); // Remove from prop map @@ -96,7 +92,7 @@ std::vector su::set_proportions_range(const su::BPTree & tree, PropMap & pm, bool normalize) { const unsigned int els = end-start; - std::vector props = std::vector(); + std::vector props = std::vector(els, 0.0); if(tree.isleaf(node)) { std::string leaf = tree.names[node]; props = table.get_obs_data_range(leaf, start, end, normalize); @@ -104,10 +100,6 @@ std::vector su::set_proportions_range(const su::BPTree & tree, unsigned int current = tree.leftchild(node); unsigned int right = tree.rightchild(node); - for( unsigned int i = 0; i < els; i++ ){ - props.push_back(0); - } - while(current <= right && current != 0) { std::vector vec = pm.get(current); // pull from prop map pm.clear(current); // remove from prop map, place back on stack diff --git a/src/unifrac.cpp b/src/unifrac.cpp index 3375fc068..487d49873 100644 --- a/src/unifrac.cpp +++ b/src/unifrac.cpp @@ -7,6 +7,8 @@ * See LICENSE file for more details */ +#include + #include "unifrac.h" #include "propmap.h" #include "stripemap.h" @@ -274,6 +276,8 @@ inline void su::unifracTT(const su::Assay & table, unsigned int k = 0; // index in tree const unsigned int max_k = (tree.nparens / 2) - 1; + + // num_prop_chunks = 1 while (k dm_stripes_buf = taskObj.dm_stripes.buf; - std::vector dm_stripes_total_buf = taskObj.dm_stripes_total.buf; - for(uint64_t j = 0; j < n_samples; j++) { uint64_t idx = ((i-start_idx)*n_samples_r)+j; - dm_stripes_buf[idx] = dm_stripes_buf[idx]/dm_stripes_total_buf[idx]; + taskObj.dm_stripes.buf[idx] = taskObj.dm_stripes.buf[idx]/taskObj.dm_stripes_total.buf[idx]; } - - taskObj.dm_stripes.buf = dm_stripes_buf; } } } diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index 2cd0dc0cf..5b39f007d 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -10,6 +10,20 @@ #include #include +#include +/* + auto start = std::chrono::high_resolution_clock::now(); + auto stop = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(stop - start); + Rcpp::Rcout << "Main thread: " << duration.count() << "\n"; + + + start = std::chrono::high_resolution_clock::now(); + stop = std::chrono::high_resolution_clock::now(); + duration = std::chrono::duration_cast(stop - start); + Rcpp::Rcout << "Condensed form: " << duration.count() << "\n"; + */ + #include #include "assay.h" @@ -29,40 +43,9 @@ Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree){ - // - // std::unordered_set to_keep(table.obs_ids.begin(), - // table.obs_ids.end()); - // - // su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - // - // su::PropMap propmap(table.n_samples); - // - // uint32_t node; - // std::vector node_proportions; - // double length; - // - // std::vector results = std::vector(table.n_samples, 0.0); - // - // - // // For node in postorderselect - // const unsigned int max_k = (tree_sheared.nparens>1) ? - // ((tree_sheared.nparens / 2) - 1) : 0; - // - // for( unsigned int k = 0; k < max_k; k++ ){ - // node = tree_sheared.postorderselect(k); - // - // // Get branch length - // length = tree_sheared.lengths[node]; - // - // // Get node proportions and set intermediate scores - // node_proportions = set_proportions(tree_sheared, node, table, propmap, - // false); - // - // for( unsigned int sample = 0; sample < table.n_samples; sample++ ){ - // // Calculate contribution of node to score - // results[sample] += (node_proportions[sample] > 0) * length; - // } - // } + + + auto start = std::chrono::high_resolution_clock::now(); su::BPTree tree = su::BPTree(rowTree); su::Assay table = su::Assay(assay); @@ -79,7 +62,10 @@ Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, //Sample_ids can be handled with a map? //n_samples, cf_size, is_upper_triangle are single values that can be passed in some other way? - //Rcpp::NumericVector cf = Rcpp::NumericVector(results.cf_size); + auto stop = std::chrono::high_resolution_clock::now(); + auto duration = std::chrono::duration_cast(stop - start); + + Rcpp::Rcout << "Main thread: " << duration.count() << "\n"; return Rcpp::List::create(Rcpp::Named("n_samples") = results.n_samples, Rcpp::Named("is_upper_triangle") = results.is_upper_triangle, diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp index 869cf759a..782584560 100644 --- a/src/unifrac_task.cpp +++ b/src/unifrac_task.cpp @@ -12,6 +12,8 @@ #include #include +#include + #include "tree.h" #include "unifrac_task.h" @@ -35,7 +37,6 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector embedded_proportions = this->embedded_proportions; - const uint64_t step_size = su::UnifracUnweightedTask::step_size; const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up @@ -53,16 +54,17 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector psum = std::vector(256); + //std::vector psum = std::vector(256); std::vector pl = std::vector(8); - std::copy( std::begin(this->sums) + (emb8<<8), - std::begin(this->sums) + (emb8<<8) + 256, - std::begin(psum) ); + //std::copy( std::begin(this->sums) + (emb8<<8), + // std::begin(this->sums) + (emb8<<8) + 256, + // std::begin(psum) ); - std::copy( std::begin(lengths) + (emb8*8), - std::begin(lengths) + (emb8*8) + 8, - std::begin(pl) ); + uint64_t len_off = emb8*8; + //std::copy( std::begin(lengths) + (emb8*8), + // std::begin(lengths) + (emb8*8) + 8, + // std::begin(pl) ); // compute all the combinations for this block (8-bits total) // psum[0] = 0.0 // +0*pl[0]+0*pl[1]+0*pl[2]+... @@ -73,14 +75,14 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector> 0) & 1) * pl[0]) + (((b8_i >> 1) & 1) * pl[1]) + - (((b8_i >> 2) & 1) * pl[2]) + (((b8_i >> 3) & 1) * pl[3]) + - (((b8_i >> 4) & 1) * pl[4]) + (((b8_i >> 5) & 1) * pl[5]) + - (((b8_i >> 6) & 1) * pl[6]) + (((b8_i >> 7) & 1) * pl[7]); + sums[(emb8<<8) + b8_i] = (((b8_i >> 0) & 1) * lengths[len_off + 0]) + (((b8_i >> 1) & 1) * lengths[len_off + 1]) + + (((b8_i >> 2) & 1) * lengths[len_off + 2]) + (((b8_i >> 3) & 1) * lengths[len_off + 3]) + + (((b8_i >> 4) & 1) * lengths[len_off + 4]) + (((b8_i >> 5) & 1) * lengths[len_off + 5]) + + (((b8_i >> 6) & 1) * lengths[len_off + 6]) + (((b8_i >> 7) & 1) * lengths[len_off + 7]); } - std::copy(std::begin(psum), std::end(psum), - std::begin(this->sums) + (emb8<<8)); + //std::copy(std::begin(psum), std::end(psum), + // std::begin(this->sums) + (emb8<<8)); } } @@ -92,10 +94,10 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector psum = std::vector(256); - std::copy( std::begin(this->sums) + (emb8<<8), - std::begin(this->sums) + (emb8<<8) + 256, - std::begin(psum) ); + //std::vector psum = std::vector(256); + //std::copy( std::begin(this->sums) + (emb8<<8), + // std::begin(this->sums) + (emb8<<8) + 256, + // std::begin(psum) ); // compute all the combinations for this block, set to 0 any past the limit @@ -105,17 +107,22 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector> (li-(emb8*8))) & 1) * lengths[li]; } - psum[b8_i] = val; + sums[(emb8<<8) + b8_i] = val; } - std::copy(std::begin(psum), std::end(psum), - std::begin(this->sums) + (emb8<<8)); + //std::copy(std::begin(psum), std::end(psum), + // std::begin(this->sums) + (emb8<<8)); } } + // point of thread for(uint64_t sk = 0; sk < sample_steps ; sk++) { + for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t ik = 0; ik < step_size ; ik++) { + + const uint64_t k = sk*step_size + ik; // within-stripe index (0:n_samples-1) const uint64_t idx = (stripe-start_idx) * n_samples_r; //n_samples_r seems to relate to continuous buffer shenanigans @@ -132,17 +139,20 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector psum = std::vector(2048); - std::copy( std::begin(this->sums) + (emb_el * 2048), - std::begin(this->sums) + (emb_el * 2048) + 2048, - std::begin(psum) ); + //Suurin syöppö? + //std::vector psum = std::vector(2048); + //std::copy( std::begin(this->sums) + (emb_el * 2048), + // std::begin(this->sums) + (emb_el * 2048) + 2048, + // std::begin(psum) ); + + uint64_t sums_off = emb_el * 2048; uint64_t u1 = embedded_proportions[offset + k]; uint64_t v1 = embedded_proportions[offset + l1]; @@ -157,22 +167,22 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector> 8) & 0xff)] + - psum[0x200+((x1 >> 16) & 0xff)] + - psum[0x300+((x1 >> 24) & 0xff)] + - psum[0x400+((x1 >> 32) & 0xff)] + - psum[0x500+((x1 >> 40) & 0xff)] + - psum[0x600+((x1 >> 48) & 0xff)] + - psum[0x700+((x1 >> 56) )]; - my_stripe_total += psum[ (o1 & 0xff)] + - psum[0x100+((o1 >> 8) & 0xff)] + - psum[0x200+((o1 >> 16) & 0xff)] + - psum[0x300+((o1 >> 24) & 0xff)] + - psum[0x400+((o1 >> 32) & 0xff)] + - psum[0x500+((o1 >> 40) & 0xff)] + - psum[0x600+((o1 >> 48) & 0xff)] + - psum[0x700+((o1 >> 56) )]; + my_stripe += sums[sums_off + (x1 & 0xff)] + + sums[sums_off + 0x100+((x1 >> 8) & 0xff)] + + sums[sums_off + 0x200+((x1 >> 16) & 0xff)] + + sums[sums_off + 0x300+((x1 >> 24) & 0xff)] + + sums[sums_off + 0x400+((x1 >> 32) & 0xff)] + + sums[sums_off + 0x500+((x1 >> 40) & 0xff)] + + sums[sums_off + 0x600+((x1 >> 48) & 0xff)] + + sums[sums_off + 0x700+((x1 >> 56) )]; + my_stripe_total += sums[sums_off + (o1 & 0xff)] + + sums[sums_off + 0x100+((o1 >> 8) & 0xff)] + + sums[sums_off + 0x200+((o1 >> 16) & 0xff)] + + sums[sums_off + 0x300+((o1 >> 24) & 0xff)] + + sums[sums_off + 0x400+((o1 >> 32) & 0xff)] + + sums[sums_off + 0x500+((o1 >> 40) & 0xff)] + + sums[sums_off + 0x600+((o1 >> 48) & 0xff)] + + sums[sums_off + 0x700+((o1 >> 56) )]; } } From 8ed15b526520b612a190a393106d030db1cd10cc Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Thu, 21 May 2026 00:49:23 +0300 Subject: [PATCH 35/48] Added method for Weighted Unifrac --- src/unifrac.cpp | 129 +++---------- src/unifrac.h | 87 ++++----- src/unifrac_R.cpp | 10 +- src/unifrac_task.cpp | 439 ++++++++++++++++++++----------------------- src/unifrac_task.h | 122 ++++++------ 5 files changed, 329 insertions(+), 458 deletions(-) diff --git a/src/unifrac.cpp b/src/unifrac.cpp index 487d49873..b448218a5 100644 --- a/src/unifrac.cpp +++ b/src/unifrac.cpp @@ -27,42 +27,16 @@ - - -su::Method su::set_method(std::string requested_method) { - if(requested_method == "unweighted") - return unweighted; - else if(requested_method == "weighted_normalized") - return weighted_normalized; - else if(requested_method == "weighted_unnormalized") - return weighted_unnormalized; - else if(requested_method == "generalized") - return generalized; - /*else if(std::strcmp(requested_method, "unweighted_fp32") == 0) - method = unweighted_fp32; - else if(std::strcmp(requested_method, "weighted_normalized_fp32") == 0) - method = weighted_normalized_fp32; - else if(std::strcmp(requested_method, "weighted_unnormalized_fp32") == 0) - method = weighted_unnormalized_fp32; - else if(std::strcmp(requested_method, "generalized_fp32") == 0) - method = generalized_fp32; */ - else { - return unknown; - } -} + su::mat_t su::one_off(const su::Assay & table, const su::BPTree & tree, - std::string unifrac_method, - double alpha, - bool variance_adjust, + bool weighted, + bool normalized, bool bypass_tips) { - //Check that method is valid - pass it as something other than string? - su::Method method = set_method(unifrac_method); - //Number of stripes to be used, basically half of samples const unsigned int stripe_stop = (table.n_samples + 1) / 2; @@ -77,10 +51,9 @@ su::mat_t su::one_off(const su::Assay & table, //Stripes to start and stop on - single task, so the entire thing task.start = 0; task.stop = stripe_stop; - task.bypass_tips = bypass_tips; + task.n_samples = table.n_samples; - task.g_unifrac_alpha = alpha; //Main action //Calls either unifrac or _vaw depending on variance_adjust @@ -94,11 +67,11 @@ su::mat_t su::one_off(const su::Assay & table, su::unifrac(std::ref(table), std::ref(tree), - method, std::ref(dm_stripes), std::ref(dm_stripes_total), - task, - variance_adjust); + weighted, + normalized, + task); //Only use of std::thread in this version of code was for stripes to condensed form //Basically each thread calls stripes_to_condensed_form @@ -122,79 +95,29 @@ su::mat_t su::one_off(const su::Assay & table, void su::unifrac(const su::Assay &table, const su::BPTree &tree, - su::Method unifrac_method, su::StripeMap & dm_stripes, su::StripeMap & dm_stripes_total, - const su::task_parameters task_p, - bool variance_adjust) + bool weighted, + bool normalized, + const su::task_parameters task_p) { - - if(variance_adjust) - { - /* - switch(unifrac_method) { - case su::unweighted: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized: - unifrac_vawTT,double>( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::unweighted_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized_fp32: - unifrac_vawTT,float >( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - default: - fprintf(stderr, "Unknown unifrac task\n"); - exit(1); - break; - } - */ + //unweighted + if (weighted == false) { + unifracTT( + table, tree, true, dm_stripes, dm_stripes_total, + task_p ); } - else - { - switch(unifrac_method) - { - case su::unweighted: - unifracTT( - table, tree, true, dm_stripes,dm_stripes_total, - task_p ); - - break; - /*case su::weighted_normalized: - unifracTT,double>( - table, tree, true, dm_stripes,dm_stripes_total, - task_p ); - break; - case su::weighted_unnormalized: - unifracTT, - double>(table, tree, false, dm_stripes, - dm_stripes_total, task_p ); - break; - case su::generalized: - unifracTT,double>( - table, tree, true, dm_stripes,dm_stripes_total, - task_p ); - break; - */ - default: - fprintf(stderr, "Unknown unifrac task\n"); - exit(1); - break; - } + //weighted normalized + else if (normalized) { + unifracTT( + table, tree, true, dm_stripes, dm_stripes_total, + task_p ); + } + //weighted normalized + else { + unifracTT( + table, tree, true, dm_stripes, dm_stripes_total, + task_p ); } } diff --git a/src/unifrac.h b/src/unifrac.h index 09e9d96c7..9f85be795 100644 --- a/src/unifrac.h +++ b/src/unifrac.h @@ -29,16 +29,6 @@ std::vector condensed_form; std::vector sample_ids; } mat_t; - - - enum Method {unweighted, - weighted_normalized, - weighted_unnormalized, - generalized, - unknown}; - - Method set_method(std::string requested_method); - /* Compute UniFrac - condensed form * @@ -62,19 +52,18 @@ su::mat_t one_off(const su::Assay & table, const su::BPTree & tree, - std::string unifrac_method, - double alpha, - bool variance_adjust, + bool weighted, + bool normalized, bool bypass_tips); // Chooses the right task for the job and constructs a unifracTT void unifrac(const su::Assay &table, const su::BPTree &tree, - su::Method unifrac_method, su::StripeMap & dm_stripes, su::StripeMap & dm_stripes_total, - const su::task_parameters task_p, - bool variance_adjust); + bool weighted, + bool normalized, + const su::task_parameters task_p); // Works the vectors template @@ -119,39 +108,39 @@ - - double** deconvolute_stripes(std::vector &stripes, uint32_t n); - - class ManagedStripes { - public: - virtual ~ManagedStripes() {} - virtual const double *get_stripe(uint32_t stripe) const = 0; - virtual void release_stripe(uint32_t stripe) const = 0; - }; - - class MemoryStripes : public ManagedStripes { - private: - const double * const * stripes; // just a pointer, not owned - public: - MemoryStripes(const double * const * _stripes) : stripes(_stripes) {} - MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} - MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} - MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} - MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} - - virtual const double *get_stripe(uint32_t stripe) const {return stripes[stripe];} - virtual void release_stripe(uint32_t stripe) const {}; - }; - - // tile_size==0 means memory optimized - template void stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size=0); - void stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size=0); - void stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size=0); - - - template void condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d); - void condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); - void condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); + // + // double** deconvolute_stripes(std::vector &stripes, uint32_t n); + // + // class ManagedStripes { + // public: + // virtual ~ManagedStripes() {} + // virtual const double *get_stripe(uint32_t stripe) const = 0; + // virtual void release_stripe(uint32_t stripe) const = 0; + // }; + // + // class MemoryStripes : public ManagedStripes { + // private: + // const double * const * stripes; // just a pointer, not owned + // public: + // MemoryStripes(const double * const * _stripes) : stripes(_stripes) {} + // MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} + // MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} + // MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} + // MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} + // + // virtual const double *get_stripe(uint32_t stripe) const {return stripes[stripe];} + // virtual void release_stripe(uint32_t stripe) const {}; + // }; + // + // // tile_size==0 means memory optimized + // template void stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size=0); + // void stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size=0); + // void stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size=0); + // + // + // template void condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d); + // void condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); + // void condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); } diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index 5b39f007d..d96788b62 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -12,6 +12,7 @@ #include /* +#include auto start = std::chrono::high_resolution_clock::now(); auto stop = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(stop - start); @@ -41,9 +42,12 @@ // @keywords internal // [[Rcpp::export(.unifrac_cpp)]] Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, - const Rcpp::List & rowTree){ - + const Rcpp::List & rowTree, + bool weighted, + bool normalized, + bool bypass_tips){ + // Normalized matches the results given by weighted auto start = std::chrono::high_resolution_clock::now(); @@ -56,7 +60,7 @@ Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - su::mat_t results = su::one_off(table, tree_sheared, method, 1.0, false, false); + su::mat_t results = su::one_off(table, tree_sheared, weighted, normalized, bypass_tips); //condensed_form is the main values, returned in result //Sample_ids can be handled with a map? diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp index 782584560..69e6dc8bf 100644 --- a/src/unifrac_task.cpp +++ b/src/unifrac_task.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -35,7 +36,7 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vectorsums; */ - std::vector embedded_proportions = this->embedded_proportions; + //std::vector embedded_proportions = this->embedded_proportions; const uint64_t step_size = su::UnifracUnweightedTask::step_size; const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up @@ -196,121 +197,207 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector -// void SUCMP_NM::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { -// const uint64_t start_idx = this->task_p->start; -// const uint64_t stop_idx = this->task_p->stop; -// const uint64_t n_samples = this->task_p->n_samples; -// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; -// -// // openacc only works well with local variables -// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; -// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; -// -// bool * const __restrict__ zcheck = this->zcheck; -// TFloat * const __restrict__ sums = this->sums; -// -// const uint64_t step_size = SUCMP_NM::UnifracUnnormalizedWeightedTask::step_size; -// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up -// -// // check for zero values and pre-compute single column sums -// #ifdef _OPENACC -// #pragma acc parallel loop present(embedded_proportions,lengths,zcheck,sums) -// #else -// #pragma omp parallel for default(shared) -// #endif -// for(uint64_t k=0; k::acc_vector_size; -// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,lengths,zcheck,sums) async -// #else -// // use dynamic scheduling due to non-homogeneity in the loop -// #pragma omp parallel for default(shared) schedule(dynamic,1) -// #endif -// for(uint64_t sk = 0; sk < sample_steps ; sk++) { -// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { -// for(uint64_t ik = 0; ik < step_size ; ik++) { -// const uint64_t k = sk*step_size + ik; -// -// if (k>=n_samples) continue; // past the limit -// -// const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here -// -// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound -// -// const bool allzero_k = zcheck[k]; -// const bool allzero_l1 = zcheck[l1]; -// -// if (allzero_k && allzero_l1) { -// // nothing to do, would have to add 0 -// } else { -// TFloat my_stripe; -// -// if (allzero_k || allzero_l1) { -// // one side has all zeros -// // we can use the distributed property, and use the pre-computed values -// -// const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 -// k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 -// -// // keep reads in the same place to maximize GPU warp performance -// my_stripe = sums[ridx]; -// -// } else { -// // both sides non zero, use the explicit but slow approach -// my_stripe = 0.0; -// -// #pragma acc loop seq -// for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); -// #endif -// } + +void su::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, std::vector lengths) { + + + //Parameter finding + + //Task parameters determine stuff + const uint64_t start_idx = this->task_p.start; + const uint64_t stop_idx = this->task_p.stop; + const uint64_t n_samples = this->task_p.n_samples; + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + + /* + // openacc only works well with local variables + const uint64_t * const __restrict__ embedded_proportions = this->embedded_proportions; + TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; + TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; + TFloat * const __restrict__ sums = this->sums; + */ + + const uint64_t step_size = su::UnifracNormalizedWeightedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + + //std::vector zcheck = this->zcheck; + //std::vector sums = this->sums; + + for(uint64_t k=0; k=n_samples) continue; // past the limit + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + const bool allzero_k = zcheck[k]; + const bool allzero_l1 = zcheck[l1]; + + if (allzero_k && allzero_l1) { + // nothing to do, would have to add 0 + } else { + const uint64_t idx = (stripe-start_idx) * n_samples_r; + + // the totals can always use the distributed property + this->dm_stripes_total.buf[idx + k] += sums[k] + sums[l1]; + + double my_stripe; + + if (allzero_k || allzero_l1) { + // one side has all zeros + // we can use the distributed property, and use the pre-computed values + + const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 + k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 + + // keep reads in the same place to maximize GPU warp performance + my_stripe = sums[ridx]; + + } else { + // both sides non zero, use the explicit but slow approach + + my_stripe = 0.0; + + for (uint64_t emb=0; embdm_stripes.buf[idx + k] += my_stripe; + } + + } // for ik + } // for stripe + } // for sk +} + + + + + +void su::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, std::vector lengths) { + //Task parameters determine stuff + const uint64_t start_idx = this->task_p.start; + const uint64_t stop_idx = this->task_p.stop; + const uint64_t n_samples = this->task_p.n_samples; + const uint64_t n_samples_r = this->dm_stripes.n_samples_r; + + // bool * const __restrict__ zcheck = this->zcheck; + // TFloat * const __restrict__ sums = this->sums; + + const uint64_t step_size = su::UnifracUnnormalizedWeightedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + + // check for zero values and pre-compute single column sums + + for(uint64_t k=0; k=n_samples) continue; // past the limit + + const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + const bool allzero_k = zcheck[k]; + const bool allzero_l1 = zcheck[l1]; + + if (allzero_k && allzero_l1) { + // nothing to do, would have to add 0 + } else { + double my_stripe; + + if (allzero_k || allzero_l1) { + // one side has all zeros + // we can use the distributed property, and use the pre-computed values + + const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 + k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 + + // keep reads in the same place to maximize GPU warp performance + my_stripe = sums[ridx]; + + } else { + // both sides non zero, use the explicit but slow approach + my_stripe = 0.0; + + for (uint64_t emb=0; embdm_stripes.buf[idx + k] += my_stripe; + } + + } // for ik + } // for stripe + } // for sk +} + + + + + // // template // void SUCMP_NM::UnifracVawUnnormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { @@ -380,125 +467,7 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector -// void SUCMP_NM::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { -// const uint64_t start_idx = this->task_p->start; -// const uint64_t stop_idx = this->task_p->stop; -// const uint64_t n_samples = this->task_p->n_samples; -// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; -// -// // openacc only works well with local variables -// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; -// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; -// TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; -// -// bool * const __restrict__ zcheck = this->zcheck; -// TFloat * const __restrict__ sums = this->sums; -// -// const uint64_t step_size = SUCMP_NM::UnifracNormalizedWeightedTask::step_size; -// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up -// -// // check for zero values and pre-compute single column sums -// #ifdef _OPENACC -// #pragma acc parallel loop present(embedded_proportions,lengths,zcheck,sums) -// #else -// #pragma omp parallel for default(shared) -// #endif -// for(uint64_t k=0; k::acc_vector_size; -// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,lengths,zcheck,sums) async -// #else -// // use dynamic scheduling due to non-homogeneity in the loop -// #pragma omp parallel for schedule(dynamic,1) default(shared) -// #endif -// for(uint64_t sk = 0; sk < sample_steps ; sk++) { -// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { -// for(uint64_t ik = 0; ik < step_size ; ik++) { -// const uint64_t k = sk*step_size + ik; -// -// if (k>=n_samples) continue; // past the limit -// -// const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here -// -// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound -// -// const bool allzero_k = zcheck[k]; -// const bool allzero_l1 = zcheck[l1]; -// -// if (allzero_k && allzero_l1) { -// // nothing to do, would have to add 0 -// } else { -// const uint64_t idx = (stripe-start_idx) * n_samples_r; -// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; -// TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; -// //TFloat *dm_stripe = dm_stripes[stripe]; -// //TFloat *dm_stripe_total = dm_stripes_total[stripe]; -// -// // the totals can always use the distributed property -// dm_stripe_total[k] += sums[k] + sums[l1]; -// -// TFloat my_stripe; -// -// if (allzero_k || allzero_l1) { -// // one side has all zeros -// // we can use the distributed property, and use the pre-computed values -// -// const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 -// k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 -// -// // keep reads in the same place to maximize GPU warp performance -// my_stripe = sums[ridx]; -// -// } else { -// // both sides non zero, use the explicit but slow approach -// -// my_stripe = 0.0; -// -// #pragma acc loop seq -// for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); -// #endif -// } + // // template // void SUCMP_NM::UnifracVawNormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { diff --git a/src/unifrac_task.h b/src/unifrac_task.h index 189e7ec92..3eb10fc87 100644 --- a/src/unifrac_task.h +++ b/src/unifrac_task.h @@ -315,7 +315,7 @@ namespace su { virtual ~UnifracTask() {} //Probably should return a vector? - virtual void run(unsigned int filled_embs, std::vector length) = 0; + virtual void run(unsigned int filled_embs, std::vector lengths) = 0; protected: static const unsigned int RECOMMENDED_MAX_EMBS_STRAIGHT = 128-16; // a little less to leave a bit of space of maxed-out L1 @@ -343,7 +343,7 @@ namespace su { virtual ~UnifracUnweightedTask() {} - virtual void run(unsigned int filled_embs, std::vector length) {_run(filled_embs, length);} + virtual void run(unsigned int filled_embs, std::vector lengths) {_run(filled_embs, lengths);} void _run(unsigned int filled_embs, std::vector lengths); private: @@ -351,6 +351,33 @@ namespace su { }; + /***********************************************/ + + class UnifracNormalizedWeightedTask : public UnifracTask { + public: + static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; + + UnifracNormalizedWeightedTask(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) + : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) + { + const unsigned int n_samples = this->task_p.n_samples; + + zcheck = std::vector(n_samples, 0); + sums = std::vector(n_samples, 0.0); + } + + virtual ~UnifracNormalizedWeightedTask() + { + } + + virtual void run(unsigned int filled_embs, std::vector lengths) {_run(filled_embs, lengths);} + + void _run(unsigned int filled_embs, std::vector lengths); + protected: + // temp buffers + std::vector zcheck; + std::vector sums; + }; @@ -360,72 +387,31 @@ namespace su { /***********************************************/ -// template -// class UnifracUnnormalizedWeightedTask : public UnifracTask { -// public: -// static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; -// -// UnifracUnnormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) -// : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) -// { -// const unsigned int n_samples = this->task_p->n_samples; -// -// zcheck = NULL; -// sums = NULL; -// posix_memalign((void **)&zcheck, 4096, sizeof(bool) * n_samples); -// posix_memalign((void **)&sums , 4096, sizeof(TFloat) * n_samples); -// #pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) -// } -// -// virtual ~UnifracUnnormalizedWeightedTask() -// { -// free(sums); -// free(zcheck); -// } -// -// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} -// -// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); -// protected: -// // temp buffers -// bool *zcheck; -// TFloat *sums; -// }; -// -// /***********************************************/ -// -// template -// class UnifracNormalizedWeightedTask : public UnifracTask { -// public: -// static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; -// -// UnifracNormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) -// : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) -// { -// const unsigned int n_samples = this->task_p->n_samples; -// -// zcheck = NULL; -// sums = NULL; -// posix_memalign((void **)&zcheck, 4096, sizeof(bool) * n_samples); -// posix_memalign((void **)&sums , 4096, sizeof(TFloat) * n_samples); -// #pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) -// } -// -// virtual ~UnifracNormalizedWeightedTask() -// { -// free(sums); -// free(zcheck); -// } -// -// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} -// -// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); -// protected: -// // temp buffers -// bool *zcheck; -// TFloat *sums; -// }; -// + class UnifracUnnormalizedWeightedTask : public UnifracTask { + public: + static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; + + UnifracUnnormalizedWeightedTask(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) + : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) + { + const unsigned int n_samples = this->task_p.n_samples; + + zcheck = std::vector(n_samples, 0); + sums = std::vector(n_samples, 0.0); + } + + virtual ~UnifracUnnormalizedWeightedTask() {} + + virtual void run(unsigned int filled_embs, std::vector lengths) {_run(filled_embs, lengths);} + + void _run(unsigned int filled_embs, std::vector lengths); + protected: + // temp buffers + std::vector zcheck; + std::vector sums; + }; + + // // /***********************************************/ // From baf83ccb8de07f7fa6162f0903a8427fda54e178 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Thu, 21 May 2026 00:55:31 +0300 Subject: [PATCH 36/48] Remove unnecessary code blocks --- src/unifrac.cpp | 53 +---- src/unifrac.h | 41 ---- src/unifrac_R.cpp | 3 +- src/unifrac_task.cpp | 541 ------------------------------------------- src/unifrac_task.h | 196 ---------------- 5 files changed, 4 insertions(+), 830 deletions(-) diff --git a/src/unifrac.cpp b/src/unifrac.cpp index b448218a5..39de10188 100644 --- a/src/unifrac.cpp +++ b/src/unifrac.cpp @@ -34,13 +34,11 @@ su::mat_t su::one_off(const su::Assay & table, const su::BPTree & tree, bool weighted, - bool normalized, bool bypass_tips) { //Number of stripes to be used, basically half of samples const unsigned int stripe_stop = (table.n_samples + 1) / 2; - //Originally std::vector of double pointers - this is where the data travels? su::StripeMap dm_stripes(table.n_samples); su::StripeMap dm_stripes_total(table.n_samples); @@ -55,13 +53,6 @@ su::mat_t su::one_off(const su::Assay & table, task.n_samples = table.n_samples; - //Main action - //Calls either unifrac or _vaw depending on variance_adjust - //makes use of std::ref? - //Versions for accelerated and cpu - let's go with cpu for now - //method is "unweighted" by default, let's start with that and see what else may be needed - - //This could potentially be threaded //Wasn't in the code because doesn't work with openacc/openmp? @@ -70,13 +61,8 @@ su::mat_t su::one_off(const su::Assay & table, std::ref(dm_stripes), std::ref(dm_stripes_total), weighted, - normalized, task); - //Only use of std::thread in this version of code was for stripes to condensed form - //Basically each thread calls stripes_to_condensed_form - //Which is just a bunch of binomial calculations - su::mat_t result; result.n_samples = table.n_samples; result.cf_size = su::comb_2(table.n_samples); @@ -98,7 +84,6 @@ void su::unifrac(const su::Assay &table, su::StripeMap & dm_stripes, su::StripeMap & dm_stripes_total, bool weighted, - bool normalized, const su::task_parameters task_p) { //unweighted @@ -108,23 +93,14 @@ void su::unifrac(const su::Assay &table, task_p ); } //weighted normalized - else if (normalized) { - unifracTT( - table, tree, true, dm_stripes, dm_stripes_total, - task_p ); - } - //weighted normalized else { - unifracTT( + unifracTT( table, tree, true, dm_stripes, dm_stripes_total, task_p ); } } - - - template inline void su::unifracTT(const su::Assay & table, const su::BPTree & tree, @@ -142,7 +118,6 @@ inline void su::unifracTT(const su::Assay & table, const unsigned int n_samples = task_p.n_samples; const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up - //su::PropStackMulti propstack_multi(table.n_samples); su::PropMap propmap(table.n_samples); const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; @@ -210,7 +185,7 @@ inline void su::unifracTT(const su::Assay & table, // ck = 0 // chunk the progress to maximize cache reuse const unsigned int tstart = 0; - const unsigned int tend = n_samples; // end of propstack? + const unsigned int tend = n_samples; unsigned int my_filled_emb = 0; unsigned int my_k=k_start; @@ -218,19 +193,9 @@ inline void su::unifracTT(const su::Assay & table, const uint32_t node = tree.postorderselect(my_k); my_k++; - //TFloat *node_proportions = propstack.pop(node); - //su::set_proportions_range(node_proportions, tree, node, table, tstart, tend, propstack); - //calculate proportions range for given node std::vector node_proportions = su::set_proportions_range(tree, node, table, tstart, tend, propmap); - - //propstack pop ERASES any existing vector for node and gives a blank one - //creates memory leaks if node isn't pushed before popping? - //get just returns the given vector - //push removes the vector from use - //Any time a propstack vector is modified, remember to do a propmap update - if(task_p.bypass_tips && tree.isleaf(node)) continue; @@ -243,10 +208,6 @@ inline void su::unifracTT(const su::Assay & table, k=my_k; - //This is used to keep track of filled embeds over different threads? - //Does nothing without openacc - //taskObj.sync_embedded_proportions(filled_emb); - taskObj._run(filled_emb,lengths); filled_emb=0; @@ -266,12 +227,6 @@ inline void su::unifracTT(const su::Assay & table, } } - - - - - - std::vector su::stripes_to_condensed_form(su::StripeMap & stripes, uint32_t n, unsigned int start, @@ -301,6 +256,4 @@ std::vector su::stripes_to_condensed_form(su::StripeMap & stripes, } } return cf; -} - - +} \ No newline at end of file diff --git a/src/unifrac.h b/src/unifrac.h index 9f85be795..7404f525c 100644 --- a/src/unifrac.h +++ b/src/unifrac.h @@ -53,7 +53,6 @@ su::mat_t one_off(const su::Assay & table, const su::BPTree & tree, bool weighted, - bool normalized, bool bypass_tips); // Chooses the right task for the job and constructs a unifracTT @@ -62,7 +61,6 @@ su::StripeMap & dm_stripes, su::StripeMap & dm_stripes_total, bool weighted, - bool normalized, const su::task_parameters task_p); // Works the vectors @@ -103,45 +101,6 @@ uint32_t n, unsigned int start, unsigned int stop); - - - - - - // - // double** deconvolute_stripes(std::vector &stripes, uint32_t n); - // - // class ManagedStripes { - // public: - // virtual ~ManagedStripes() {} - // virtual const double *get_stripe(uint32_t stripe) const = 0; - // virtual void release_stripe(uint32_t stripe) const = 0; - // }; - // - // class MemoryStripes : public ManagedStripes { - // private: - // const double * const * stripes; // just a pointer, not owned - // public: - // MemoryStripes(const double * const * _stripes) : stripes(_stripes) {} - // MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} - // MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} - // MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} - // MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} - // - // virtual const double *get_stripe(uint32_t stripe) const {return stripes[stripe];} - // virtual void release_stripe(uint32_t stripe) const {}; - // }; - // - // // tile_size==0 means memory optimized - // template void stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size=0); - // void stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size=0); - // void stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size=0); - // - // - // template void condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d); - // void condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); - // void condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); - } #endif diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index d96788b62..de964ec85 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -44,7 +44,6 @@ Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree, bool weighted, - bool normalized, bool bypass_tips){ // Normalized matches the results given by weighted @@ -60,7 +59,7 @@ Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - su::mat_t results = su::one_off(table, tree_sheared, weighted, normalized, bypass_tips); + su::mat_t results = su::one_off(table, tree_sheared, weighted, bypass_tips); //condensed_form is the main values, returned in result //Sample_ids can be handled with a map? diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp index 69e6dc8bf..bb9e59722 100644 --- a/src/unifrac_task.cpp +++ b/src/unifrac_task.cpp @@ -28,16 +28,6 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vectortask_p.n_samples; const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - /* - // openacc only works well with local variables - const uint64_t * const __restrict__ embedded_proportions = this->embedded_proportions; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - TFloat * const __restrict__ sums = this->sums; - */ - - //std::vector embedded_proportions = this->embedded_proportions; - const uint64_t step_size = su::UnifracUnweightedTask::step_size; const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up @@ -52,20 +42,10 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector psum = std::vector(256); std::vector pl = std::vector(8); - //std::copy( std::begin(this->sums) + (emb8<<8), - // std::begin(this->sums) + (emb8<<8) + 256, - // std::begin(psum) ); uint64_t len_off = emb8*8; - //std::copy( std::begin(lengths) + (emb8*8), - // std::begin(lengths) + (emb8*8) + 8, - // std::begin(pl) ); // compute all the combinations for this block (8-bits total) // psum[0] = 0.0 // +0*pl[0]+0*pl[1]+0*pl[2]+... @@ -81,9 +61,6 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector> 4) & 1) * lengths[len_off + 4]) + (((b8_i >> 5) & 1) * lengths[len_off + 5]) + (((b8_i >> 6) & 1) * lengths[len_off + 6]) + (((b8_i >> 7) & 1) * lengths[len_off + 7]); } - - //std::copy(std::begin(psum), std::end(psum), - // std::begin(this->sums) + (emb8<<8)); } } @@ -93,14 +70,6 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector psum = std::vector(256); - //std::copy( std::begin(this->sums) + (emb8<<8), - // std::begin(this->sums) + (emb8<<8) + 256, - // std::begin(psum) ); - - // compute all the combinations for this block, set to 0 any past the limit // as above for (uint64_t b8_i=0; b8_i<0x100; b8_i++) { @@ -111,8 +80,6 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vectorsums) + (emb8<<8)); } } @@ -127,11 +94,6 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector=n_samples) continue; // past the limit const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound @@ -145,13 +107,6 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector psum = std::vector(2048); - //std::copy( std::begin(this->sums) + (emb_el * 2048), - // std::begin(this->sums) + (emb_el * 2048) + 2048, - // std::begin(psum) ); uint64_t sums_off = emb_el * 2048; @@ -209,14 +164,6 @@ void su::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, std::vect const uint64_t n_samples = this->task_p.n_samples; const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - /* - // openacc only works well with local variables - const uint64_t * const __restrict__ embedded_proportions = this->embedded_proportions; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - TFloat * const __restrict__ sums = this->sums; - */ - const uint64_t step_size = su::UnifracNormalizedWeightedTask::step_size; const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up @@ -299,491 +246,3 @@ void su::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, std::vect } // for stripe } // for sk } - - - - - -void su::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, std::vector lengths) { - //Task parameters determine stuff - const uint64_t start_idx = this->task_p.start; - const uint64_t stop_idx = this->task_p.stop; - const uint64_t n_samples = this->task_p.n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // bool * const __restrict__ zcheck = this->zcheck; - // TFloat * const __restrict__ sums = this->sums; - - const uint64_t step_size = su::UnifracUnnormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - // check for zero values and pre-compute single column sums - - for(uint64_t k=0; k=n_samples) continue; // past the limit - - const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const bool allzero_k = zcheck[k]; - const bool allzero_l1 = zcheck[l1]; - - if (allzero_k && allzero_l1) { - // nothing to do, would have to add 0 - } else { - double my_stripe; - - if (allzero_k || allzero_l1) { - // one side has all zeros - // we can use the distributed property, and use the pre-computed values - - const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 - k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 - - // keep reads in the same place to maximize GPU warp performance - my_stripe = sums[ridx]; - - } else { - // both sides non zero, use the explicit but slow approach - my_stripe = 0.0; - - for (uint64_t emb=0; embdm_stripes.buf[idx + k] += my_stripe; - } - - } // for ik - } // for stripe - } // for sk -} - - - - - -// -// template -// void SUCMP_NM::UnifracVawUnnormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { -// const uint64_t start_idx = this->task_p->start; -// const uint64_t stop_idx = this->task_p->stop; -// const uint64_t n_samples = this->task_p->n_samples; -// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; -// -// // openacc only works well with local variables -// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; -// const TFloat * const __restrict__ embedded_counts = this->embedded_counts; -// const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; -// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; -// -// const uint64_t step_size = SUCMP_NM::UnifracVawUnnormalizedWeightedTask::step_size; -// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up -// -// // point of thread -// #ifdef _OPENACC -// const unsigned int acc_vector_size = SUCMP_NM::UnifracVawUnnormalizedWeightedTask::acc_vector_size; -// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,lengths) async -// #else -// #pragma omp parallel for default(shared) schedule(dynamic,1) -// #endif -// for(uint64_t sk = 0; sk < sample_steps ; sk++) { -// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { -// for(uint64_t ik = 0; ik < step_size ; ik++) { -// const uint64_t k = sk*step_size + ik; -// const uint64_t idx = (stripe-start_idx) * n_samples_r; -// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; -// //TFloat *dm_stripe = dm_stripes[stripe]; -// -// if (k>=n_samples) continue; // past the limit -// -// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound -// -// const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; -// -// TFloat my_stripe = dm_stripe[k]; -// -// #pragma acc loop seq -// for (uint64_t emb=0; emb 0) { -// TFloat u1 = embedded_proportions[offset + k]; -// TFloat v1 = embedded_proportions[offset + l1]; -// TFloat diff1 = fabs(u1 - v1); -// TFloat length = lengths[emb]; -// -// my_stripe += (diff1 * length) / vaw; -// } -// } -// -// dm_stripe[k] = my_stripe; -// } -// -// } -// } -// -// #ifdef _OPENACC -// // next iteration will use the alternative space -// std::swap(this->embedded_proportions,this->embedded_proportions_alt); -// #endif -// } -// - -// -// template -// void SUCMP_NM::UnifracVawNormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { -// const uint64_t start_idx = this->task_p->start; -// const uint64_t stop_idx = this->task_p->stop; -// const uint64_t n_samples = this->task_p->n_samples; -// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; -// -// // openacc only works well with local variables -// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; -// const TFloat * const __restrict__ embedded_counts = this->embedded_counts; -// const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; -// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; -// TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; -// -// const uint64_t step_size = SUCMP_NM::UnifracVawNormalizedWeightedTask::step_size; -// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up -// -// // point of thread -// #ifdef _OPENACC -// const unsigned int acc_vector_size = SUCMP_NM::UnifracVawNormalizedWeightedTask::acc_vector_size; -// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async -// #else -// #pragma omp parallel for schedule(dynamic,1) default(shared) -// #endif -// for(uint64_t sk = 0; sk < sample_steps ; sk++) { -// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { -// for(uint64_t ik = 0; ik < step_size ; ik++) { -// const uint64_t k = sk*step_size + ik; -// const uint64_t idx = (stripe-start_idx) * n_samples_r; -// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; -// TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; -// //TFloat *dm_stripe = dm_stripes[stripe]; -// //TFloat *dm_stripe_total = dm_stripes_total[stripe]; -// -// if (k>=n_samples) continue; // past the limit -// -// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound -// -// const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; -// -// TFloat my_stripe = dm_stripe[k]; -// TFloat my_stripe_total = dm_stripe_total[k]; -// -// #pragma acc loop seq -// for (uint64_t emb=0; emb 0) { -// TFloat u1 = embedded_proportions[offset + k]; -// TFloat v1 = embedded_proportions[offset + l1]; -// TFloat diff1 = fabs(u1 - v1); -// TFloat length = lengths[emb]; -// -// my_stripe += (diff1 * length) / vaw; -// my_stripe_total += ((u1 + v1) * length) / vaw; -// } -// } -// -// dm_stripe[k] = my_stripe; -// dm_stripe_total[k] = my_stripe_total; -// -// } -// -// } -// } -// -// #ifdef _OPENACC -// // next iteration will use the alternative space -// std::swap(this->embedded_proportions,this->embedded_proportions_alt); -// #endif -// } -// -// template -// void SUCMP_NM::UnifracGeneralizedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { -// const uint64_t start_idx = this->task_p->start; -// const uint64_t stop_idx = this->task_p->stop; -// const uint64_t n_samples = this->task_p->n_samples; -// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; -// -// // openacc only works well with local variables -// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; -// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; -// TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; -// -// const TFloat g_unifrac_alpha = this->task_p->g_unifrac_alpha; -// -// const uint64_t step_size = SUCMP_NM::UnifracGeneralizedTask::step_size; -// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up -// -// // point of thread -// #ifdef _OPENACC -// const unsigned int acc_vector_size = SUCMP_NM::UnifracGeneralizedTask::acc_vector_size; -// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,lengths) async -// #else -// #pragma omp parallel for schedule(dynamic,1) default(shared) -// #endif -// for(uint64_t sk = 0; sk < sample_steps ; sk++) { -// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { -// for(uint64_t ik = 0; ik < step_size ; ik++) { -// const uint64_t k = sk*step_size + ik; -// const uint64_t idx = (stripe-start_idx) * n_samples_r; -// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; -// TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; -// //TFloat *dm_stripe = dm_stripes[stripe]; -// //TFloat *dm_stripe_total = dm_stripes_total[stripe]; -// -// if (k>=n_samples) continue; // past the limit -// -// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound -// -// TFloat my_stripe = dm_stripe[k]; -// TFloat my_stripe_total = dm_stripe_total[k]; -// -// #pragma acc loop seq -// for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); -// #endif -// } -// -// template -// void SUCMP_NM::UnifracVawGeneralizedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { -// const uint64_t start_idx = this->task_p->start; -// const uint64_t stop_idx = this->task_p->stop; -// const uint64_t n_samples = this->task_p->n_samples; -// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; -// -// const TFloat g_unifrac_alpha = this->task_p->g_unifrac_alpha; -// -// // openacc only works well with local variables -// const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; -// const TFloat * const __restrict__ embedded_counts = this->embedded_counts; -// const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; -// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; -// TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; -// -// const uint64_t step_size = SUCMP_NM::UnifracVawGeneralizedTask::step_size; -// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up -// // quick hack, to be finished -// -// // point of thread -// #ifdef _OPENACC -// const unsigned int acc_vector_size = SUCMP_NM::UnifracVawGeneralizedTask::acc_vector_size; -// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async -// #else -// #pragma omp parallel for schedule(dynamic,1) default(shared) -// #endif -// for(uint64_t sk = 0; sk < sample_steps ; sk++) { -// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { -// for(uint64_t ik = 0; ik < step_size ; ik++) { -// const uint64_t k = sk*step_size + ik; -// const uint64_t idx = (stripe-start_idx) * n_samples_r; -// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; -// TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; -// //TFloat *dm_stripe = dm_stripes[stripe]; -// //TFloat *dm_stripe_total = dm_stripes_total[stripe]; -// -// if (k>=n_samples) continue; // past the limit -// -// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound -// -// const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; -// -// TFloat my_stripe = dm_stripe[k]; -// TFloat my_stripe_total = dm_stripe_total[k]; -// -// #pragma acc loop seq -// for (uint64_t emb=0; emb 0) { -// TFloat u1 = embedded_proportions[offset + k]; -// TFloat v1 = embedded_proportions[offset + l1]; -// TFloat length = lengths[emb]; -// -// TFloat sum1 = (u1 + v1) / vaw; -// TFloat sub1 = fabs(u1 - v1) / vaw; -// TFloat sum_pow1 = pow(sum1, g_unifrac_alpha) * length; -// -// my_stripe += sum_pow1 * (sub1 / sum1); -// my_stripe_total += sum_pow1; -// } -// } -// -// dm_stripe[k] = my_stripe; -// dm_stripe_total[k] = my_stripe_total; -// -// } -// } -// } -// -// #ifdef _OPENACC -// // next iteration will use the alternative space -// std::swap(this->embedded_proportions,this->embedded_proportions_alt); -// #endif -// } -// -// template -// void SUCMP_NM::UnifracVawUnweightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { -// const uint64_t start_idx = this->task_p->start; -// const uint64_t stop_idx = this->task_p->stop; -// const uint64_t n_samples = this->task_p->n_samples; -// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; -// -// // openacc only works well with local variables -// const uint32_t * const __restrict__ embedded_proportions = this->embedded_proportions; -// const TFloat * const __restrict__ embedded_counts = this->embedded_counts; -// const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; -// TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; -// TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; -// -// const uint64_t step_size = SUCMP_NM::UnifracVawUnweightedTask::step_size; -// const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up -// -// const uint64_t filled_embs_els = (filled_embs+31)/32; // round up -// -// // point of thread -// #ifdef _OPENACC -// const unsigned int acc_vector_size = SUCMP_NM::UnifracVawUnweightedTask::acc_vector_size; -// #pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async -// #else -// #pragma omp parallel for schedule(dynamic,1) default(shared) -// #endif -// for(uint64_t sk = 0; sk < sample_steps ; sk++) { -// for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { -// for(uint64_t ik = 0; ik < step_size ; ik++) { -// const uint64_t k = sk*step_size + ik; -// const uint64_t idx = (stripe-start_idx) * n_samples_r; -// TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; -// TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; -// //TFloat *dm_stripe = dm_stripes[stripe]; -// //TFloat *dm_stripe_total = dm_stripes_total[stripe]; -// -// if (k>=n_samples) continue; // past the limit -// -// const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound -// -// TFloat my_stripe = dm_stripe[k]; -// TFloat my_stripe_total = dm_stripe_total[k]; -// -// const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; -// -// #pragma acc loop seq -// for (uint64_t emb_el=0; emb_el 0) { -// TFloat length = lengths[emb]; -// TFloat lv1 = length / vaw; -// -// my_stripe += ((x1 >> ei) & 1)*lv1; -// my_stripe_total += ((o1 >> ei) & 1)*lv1; -// } -// } -// } -// } -// -// dm_stripe[k] = my_stripe; -// dm_stripe_total[k] = my_stripe_total; -// -// } -// -// } -// } -// -// #ifdef _OPENACC -// // next iteration will use the alternative space -// std::swap(this->embedded_proportions,this->embedded_proportions_alt); -// #endif -// } -// diff --git a/src/unifrac_task.h b/src/unifrac_task.h index 3eb10fc87..855850835 100644 --- a/src/unifrac_task.h +++ b/src/unifrac_task.h @@ -379,202 +379,6 @@ namespace su { std::vector sums; }; - - - - - - /***********************************************/ - - - class UnifracUnnormalizedWeightedTask : public UnifracTask { - public: - static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; - - UnifracUnnormalizedWeightedTask(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) - { - const unsigned int n_samples = this->task_p.n_samples; - - zcheck = std::vector(n_samples, 0); - sums = std::vector(n_samples, 0.0); - } - - virtual ~UnifracUnnormalizedWeightedTask() {} - - virtual void run(unsigned int filled_embs, std::vector lengths) {_run(filled_embs, lengths);} - - void _run(unsigned int filled_embs, std::vector lengths); - protected: - // temp buffers - std::vector zcheck; - std::vector sums; - }; - - -// -// /***********************************************/ -// -// -// template -// class UnifracGeneralizedTask : public UnifracTask { -// public: -// static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; -// -// UnifracGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) -// : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) {} -// -// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} -// -// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); -// }; -// -// /* void unifrac_vaw tasks -// * -// * all methods utilize the same function signature. that signature is as follows: -// * -// * dm_stripes vector the stripes of the distance matrix being accumulated -// * into for unique branch length -// * dm_stripes vector the stripes of the distance matrix being accumulated -// * into for total branch length (e.g., to normalize unweighted unifrac) -// * embedded_proportions the proportions vector for a sample, or rather -// * the counts vector normalized to 1. this vector is embedded as it is -// * duplicated: if A, B and C are proportions for features A, B, and C, the -// * vector will look like [A B C A B C]. -// * embedded_counts the counts vector embedded in the same way and order as -// * embedded_proportions. the values of this array are unnormalized feature -// * counts for the subtree. -// * sample_total_counts the total unnormalized feature counts for all samples -// * embedded in the same way and order as embedded_proportions. -// * length the branch length of the current node to its parent. -// * task_p task specific parameters. -// */ -// template -// class UnifracVawTask : public UnifracTaskBase { -// protected: -// #ifdef _OPENACC -// // The parallel nature of GPUs needs a largish step -// #ifndef SMALLGPU -// // default to larger step, which makes a big difference for bigger GPUs like V100 -// static const unsigned int step_size = 32; -// // keep the vector size just big enough to keep the used emb array inside the 32k buffer -// static const unsigned int acc_vector_size = 32*32*8/sizeof(TFloat); -// #else -// // smaller GPUs prefer a slightly smaller step -// static const unsigned int step_size = 16; -// // keep the vector size just big enough to keep the used emb array inside the 32k buffer -// static const unsigned int acc_vector_size = 32*32*8/sizeof(TFloat); -// #endif -// #else -// // The serial nature of CPU cores prefers a small step -// static const unsigned int step_size = 4; -// #endif -// -// public: -// TFloat * const embedded_counts; -// const TFloat * const sample_total_counts; -// -// static const unsigned int RECOMMENDED_MAX_EMBS = 128; -// -// UnifracVawTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, -// const TFloat * _sample_total_counts, -// unsigned int _max_embs, const su::task_parameters* _task_p) -// : UnifracTaskBase(_dm_stripes, _dm_stripes_total, _max_embs, _task_p) -// , embedded_counts(UnifracTaskBase::initialize_embedded(this->dm_stripes.n_samples_r,_max_embs)), sample_total_counts(_sample_total_counts) {} -// -// -// /* delete -// UnifracVawTask(UnifracTaskBase &baseObj, -// const TEmb * _embedded_proportions, const TFloat * _sample_total_counts, unsigned int _max_embs) -// : UnifracTaskBase(baseObj) -// , embedded_proportions(_embedded_proportions), embedded_counts(initialize_embedded()), sample_total_counts(_sample_total_counts), max_embs(_max_embs) {} -// */ -// -// -// virtual ~UnifracVawTask() {} -// -// void sync_embedded_counts(unsigned int filled_embs) -// { -// #ifdef _OPENACC -// const uint64_t n_samples_r = this->dm_stripes.n_samples_r; -// const uint64_t bsize = n_samples_r * filled_embs; -// #pragma acc update device(embedded_counts[:bsize]) -// #endif -// } -// -// void sync_embedded(unsigned int filled_embs) { this->sync_embedded_proportions(filled_embs); this->sync_embedded_counts(filled_embs);} -// -// void embed_range(const TFloat* __restrict__ in_proportions, const TFloat* __restrict__ in_counts, unsigned int start, unsigned int end, unsigned int emb) { -// this->embed_proportions_range(in_proportions,start,end,emb); -// this->embed_proportions_range_straight(this->embedded_counts,in_counts,start,end,emb); -// } -// void embed(const TFloat* __restrict__ in_proportions, const double* __restrict__ in_counts, unsigned int emb) { embed_range(in_proportions,in_counts,0,this->dm_stripes.n_samples,emb);} -// -// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) = 0; -// }; -// -// /***********************************************/ -// -// -// template -// class UnifracVawUnnormalizedWeightedTask : public UnifracVawTask { -// public: -// UnifracVawUnnormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, -// const TFloat * _sample_total_counts, -// unsigned int _max_embs, const su::task_parameters* _task_p) -// : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} -// -// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} -// -// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); -// }; -// -// /***********************************************/ -// -// template -// class UnifracVawNormalizedWeightedTask : public UnifracVawTask { -// public: -// UnifracVawNormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, -// const TFloat * _sample_total_counts, -// unsigned int _max_embs, const su::task_parameters* _task_p) -// : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} -// -// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} -// -// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); -// }; -// -// /***********************************************/ -// -// template -// class UnifracVawUnweightedTask : public UnifracVawTask { -// public: -// UnifracVawUnweightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, -// const TFloat * _sample_total_counts, -// unsigned int _max_embs, const su::task_parameters* _task_p) -// : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} -// -// virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} -// -// void _run(unsigned int filled_embs, const TFloat * __restrict__ length); -// }; -// -// /***********************************************/ -// -// template -// class UnifracVawGeneralizedTask : public UnifracVawTask { - // public: - // UnifracVawGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - // const TFloat * _sample_total_counts, - // unsigned int _max_embs, const su::task_parameters* _task_p) - // : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} - // - // virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - // - // void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - // }; - - } #endif From 8ba8c120935459561bd08af69fae8c9136c434b4 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Fri, 22 May 2026 03:34:16 +0300 Subject: [PATCH 37/48] Clean up formatting --- src/propmap.cpp | 20 ++---- src/propmap.h | 19 ++---- src/stripemap.cpp | 27 ++++---- src/stripemap.h | 30 ++++----- src/tree.h | 4 +- src/unifrac.cpp | 39 +++++------ src/unifrac.h | 149 +++++++++++++++++++------------------------ src/unifrac_R.cpp | 67 ++++++++++--------- src/unifrac_task.cpp | 66 +++++++++---------- src/unifrac_task.h | 10 +-- 10 files changed, 199 insertions(+), 232 deletions(-) diff --git a/src/propmap.cpp b/src/propmap.cpp index 1964e2664..1f2e798a6 100644 --- a/src/propmap.cpp +++ b/src/propmap.cpp @@ -11,14 +11,6 @@ #include "assay.h" #include "propmap.h" -#include -#include -#include -#include -#include -#include -#include - #include using namespace su; @@ -49,10 +41,10 @@ void PropMap::update(uint32_t node, std::vector vec){ prop_map[node] = vec; } -std::vector su::set_proportions(const BPTree &tree, +std::vector su::set_proportions(const BPTree & tree, uint32_t node, - const Assay &table, - PropMap &ps, + const Assay & table, + PropMap & pm, bool normalize){ std::vector props = std::vector(table.n_samples, 0.0); if( tree.isleaf(node) ){ @@ -68,8 +60,8 @@ std::vector su::set_proportions(const BPTree &tree, unsigned int right = tree.rightchild(node); while( current <= right && current != 0 ){ - std::vector vec = ps.get(current); // Pull from prop map - ps.clear(current); // Remove from prop map + std::vector vec = pm.get(current); // Pull from prop map + pm.clear(current); // Remove from prop map for( unsigned int i = 0; i < table.n_samples; i++ ){ props[i] = props[i] + vec[i]; @@ -79,7 +71,7 @@ std::vector su::set_proportions(const BPTree &tree, } } - ps.update(node, props); + pm.update(node, props); return(props); } diff --git a/src/propmap.h b/src/propmap.h index a82275a9a..3bf3df900 100644 --- a/src/propmap.h +++ b/src/propmap.h @@ -7,10 +7,8 @@ * See LICENSE file for more details */ - - -#ifndef __FAITH_PROPMAP -#define __FAITH_PROPMAP 1 +#ifndef __FAITH_PROPMAP_H +#define __FAITH_PROPMAP_H 1 #include #include @@ -34,14 +32,11 @@ class PropMap { uint32_t defaultsize; }; -std::vector set_proportions(const BPTree &tree, uint32_t node, - const Assay &table, - PropMap &ps, +std::vector set_proportions(const BPTree & tree, uint32_t node, + const Assay & table, + PropMap & pm, bool normalize = true); -// Sets proportion range -// Data is stored to props -> make return vector -// PropMap needs to be modified, thus passed by reference std::vector set_proportions_range(const su::BPTree & tree, uint32_t node, const su::Assay & table, @@ -49,7 +44,7 @@ std::vector set_proportions_range(const su::BPTree & tree, unsigned int end, PropMap & pm, bool normalize = true); -} +} -#endif /* __FAITH_PROPMAP */ +#endif /* __FAITH_PROPMAP_H */ diff --git a/src/stripemap.cpp b/src/stripemap.cpp index e861a8963..2a73909ff 100644 --- a/src/stripemap.cpp +++ b/src/stripemap.cpp @@ -1,23 +1,21 @@ /* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ +* BSD 3-Clause License +* +* Copyright (c) 2016-2021, UniFrac development team. +* All rights reserved. +* +* See LICENSE file for more details +*/ #include "tree.h" #include "assay.h" #include "stripemap.h" -#include - using namespace su; -StripeMap::StripeMap(uint32_t n_samples) - : stripe_map(), - vecsize(n_samples) +StripeMap::StripeMap(uint32_t n_samples) + : stripe_map() + , vecsize(n_samples) { n_stripes = (n_samples + 1) / 2; for( unsigned int i = 0; i < n_stripes; i++ ){ @@ -25,14 +23,15 @@ StripeMap::StripeMap(uint32_t n_samples) } } -StripeMap::~StripeMap() {} +StripeMap::~StripeMap(){ +} std::vector StripeMap::get(uint32_t i){ if( stripe_map.count(i) > 0 ){ return stripe_map.at(i); } else { return(std::vector()); - } + } } void StripeMap::clear(uint32_t i){ diff --git a/src/stripemap.h b/src/stripemap.h index a435df970..b30721364 100644 --- a/src/stripemap.h +++ b/src/stripemap.h @@ -1,23 +1,22 @@ /* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ +* BSD 3-Clause License +* +* Copyright (c) 2016-2021, UniFrac development team. +* All rights reserved. +* +* See LICENSE file for more details +*/ - -#ifndef __FAITH_STRIPEMAP -#define __FAITH_STRIPEMAP 1 +#ifndef __UNIFRAC_STRIPEMAP_H +#define __UNIFRAC_STRIPEMAP_H 1 #include #include #include -#include namespace su { - class StripeMap { + +class StripeMap { public: StripeMap(uint32_t n_samples); virtual ~StripeMap(); @@ -28,9 +27,10 @@ namespace su { private: std::unordered_map> stripe_map; - uint32_t vecsize; // Size of stripe vectors is always the number of samples + uint32_t vecsize; // equal to number of samples uint32_t n_stripes; - }; +}; + } -#endif /* __FAITH_STRIPEMAP */ +#endif /* __UNIFRAC_STRIPEMAP_H */ diff --git a/src/tree.h b/src/tree.h index 05b116e61..0ba293bb2 100644 --- a/src/tree.h +++ b/src/tree.h @@ -19,6 +19,7 @@ #include namespace su { + class BPTree { public: /* Tracked attributes */ @@ -134,7 +135,8 @@ class BPTree { int32_t bwd(uint32_t i, int32_t d) const; int32_t enclose(uint32_t i) const; - }; +}; + } #endif /* __FAITH_TREE_H */ diff --git a/src/unifrac.cpp b/src/unifrac.cpp index 39de10188..6963dbd59 100644 --- a/src/unifrac.cpp +++ b/src/unifrac.cpp @@ -14,23 +14,6 @@ #include "stripemap.h" #include "tree.h" -/* -#include -#include -#include -#include -#include -#include -#include -#include -*/ - - - - - - - su::mat_t su::one_off(const su::Assay & table, const su::BPTree & tree, bool weighted, @@ -116,7 +99,8 @@ inline void su::unifracTT(const su::Assay & table, } const unsigned int n_samples = task_p.n_samples; - const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up + const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1) / + UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up su::PropMap propmap(table.n_samples); @@ -194,7 +178,13 @@ inline void su::unifracTT(const su::Assay & table, my_k++; //calculate proportions range for given node - std::vector node_proportions = su::set_proportions_range(tree, node, table, tstart, tend, propmap); + std::vector node_proportions = su::set_proportions_range( + tree, + node, + table, + tstart, + tend, + propmap); if(task_p.bypass_tips && tree.isleaf(node)) continue; @@ -202,7 +192,10 @@ inline void su::unifracTT(const su::Assay & table, lengths[filled_emb] = tree.lengths[node]; filled_emb++; - taskObj.embed_proportions_range(node_proportions, tstart, tend, my_filled_emb); + taskObj.embed_proportions_range(node_proportions, + tstart, + tend, + my_filled_emb); my_filled_emb++; } @@ -221,7 +214,8 @@ inline void su::unifracTT(const su::Assay & table, for(uint64_t i = start_idx; i < stop_idx; i++){ for(uint64_t j = 0; j < n_samples; j++) { uint64_t idx = ((i-start_idx)*n_samples_r)+j; - taskObj.dm_stripes.buf[idx] = taskObj.dm_stripes.buf[idx]/taskObj.dm_stripes_total.buf[idx]; + taskObj.dm_stripes.buf[idx] = taskObj.dm_stripes.buf[idx] / + taskObj.dm_stripes_total.buf[idx]; } } } @@ -248,7 +242,8 @@ std::vector su::stripes_to_condensed_form(su::StripeMap & stripes, i = 0; j = n - (stripe + 1); } - // determine the position in the condensed form vector for a given (i, j) + // determine the position in the condensed form vector for a given + // (i, j) // based off of // https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.squareform.html uint64_t comb_N_minus_i = comb_2(n - i); diff --git a/src/unifrac.h b/src/unifrac.h index 7404f525c..c9deb04da 100644 --- a/src/unifrac.h +++ b/src/unifrac.h @@ -7,8 +7,8 @@ * See LICENSE file for more details */ -#ifndef __UNIFRAC -#define __UNIFRAC 1 +#ifndef __UNIFRAC_H +#define __UNIFRAC_H 1 #include #include @@ -20,87 +20,68 @@ #include "propmap.h" #include "unifrac_task.h" - namespace su { - - typedef struct mat { - unsigned int n_samples; - unsigned int cf_size; - bool is_upper_triangle; - std::vector condensed_form; - std::vector sample_ids; - } mat_t; - - /* Compute UniFrac - condensed form - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads to use. - * result the resulting distance matrix in condensed form, this is initialized within the method so using ** - * - * one_off returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - * table_empty : the table does not have any entries - */ - - su::mat_t one_off(const su::Assay & table, - const su::BPTree & tree, - bool weighted, - bool bypass_tips); - - // Chooses the right task for the job and constructs a unifracTT - void unifrac(const su::Assay &table, - const su::BPTree &tree, - su::StripeMap & dm_stripes, - su::StripeMap & dm_stripes_total, - bool weighted, - const su::task_parameters task_p); - - // Works the vectors - template - inline void unifracTT(const su::Assay & table, - const su::BPTree & tree, - const bool want_total, - su::StripeMap & dm_stripes, - su::StripeMap & dm_stripes_total, - const su::task_parameters & task_p); - - inline uint64_t comb_2(uint64_t N) { - // based off of _comb_int_long - // https://github.com/scipy/scipy/blob/v0.19.1/scipy/special/_comb.pyx - - // Compute binom(N, k) for integers. - // - // we're disregarding overflow as that practically should not - // happen unless the number of samples processed is in excess - // of 4 billion - uint64_t val, j, M, nterms; - uint64_t k = 2; - - M = N + 1; - nterms = k < (N - k) ? k : N - k; - - val = 1; - - for(j = 1; j < nterms + 1; j++) { - val *= M - j; - val /= j; - } - return val; - } - - // Stripes to condensed form for the results - std::vector stripes_to_condensed_form(su::StripeMap & stripes, - uint32_t n, - unsigned int start, - unsigned int stop); +namespace su { + +typedef struct mat { + unsigned int n_samples; + unsigned int cf_size; + bool is_upper_triangle; + std::vector condensed_form; + std::vector sample_ids; +} mat_t; + +su::mat_t one_off(const su::Assay & table, + const su::BPTree & tree, + bool weighted, + bool bypass_tips); + +// Chooses the right task for the job and constructs a unifracTT +void unifrac(const su::Assay &table, + const su::BPTree &tree, + su::StripeMap & dm_stripes, + su::StripeMap & dm_stripes_total, + bool weighted, + const su::task_parameters task_p); + +// Works the vectors +template +inline void unifracTT(const su::Assay & table, + const su::BPTree & tree, + const bool want_total, + su::StripeMap & dm_stripes, + su::StripeMap & dm_stripes_total, + const su::task_parameters & task_p); + +inline uint64_t comb_2(uint64_t N) { + // based off of _comb_int_long + // https://github.com/scipy/scipy/blob/v0.19.1/scipy/special/_comb.pyx + + // Compute binom(N, k) for integers. + // + // we're disregarding overflow as that practically should not + // happen unless the number of samples processed is in excess + // of 4 billion + uint64_t val, j, M, nterms; + uint64_t k = 2; + + M = N + 1; + nterms = k < (N - k) ? k : N - k; + + val = 1; + + for(j = 1; j < nterms + 1; j++) { + val *= M - j; + val /= j; } + return val; +} + +// Stripes to condensed form for the results +std::vector stripes_to_condensed_form(su::StripeMap & stripes, + uint32_t n, + unsigned int start, + unsigned int stop); + +} -#endif +#endif /* __UNIFRAC_H */ diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index de964ec85..30909f176 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -10,21 +10,6 @@ #include #include -#include -/* -#include - auto start = std::chrono::high_resolution_clock::now(); - auto stop = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(stop - start); - Rcpp::Rcout << "Main thread: " << duration.count() << "\n"; - - - start = std::chrono::high_resolution_clock::now(); - stop = std::chrono::high_resolution_clock::now(); - duration = std::chrono::duration_cast(stop - start); - Rcpp::Rcout << "Condensed form: " << duration.count() << "\n"; - */ - #include #include "assay.h" @@ -35,24 +20,43 @@ #include "unifrac.h" - - // Calculate Unifrac // +// This function calculates Unifrac distances for a given assay and rowTree, +// using a C++ implementation of the Striped Unifrac algorithm. +// +// @details +// This function makes several assumptions about the contents of +// \code{assay} and \code{rowTree}, namely that: +// \itemize{ +// \item \code{assay} and \code{rowTree} are both non-empty. +// \item \code{assay} has row and column names. +// \item \code{rowTree}'s nodes are arranged in cladewise order. +// } +// These checks should all be handled in the surrounding R code. +// +// The C++ code was adapted from an implementation by the Unifrac team +// (Armstrong et al. 2021), which is licensed under the BSD 3-Clause license. +// +// @param assay An R numeric matrix containing the assay of a \code{TreeSE} +// object. +// @param rowTree An \code{ape::phylo} object containing the rowTree of a +// \code{TreeSE} object. +// @param weighted Boolean: Whether to calculate unweighted or weighted Unifrac. +// @param bypass_tips Boolean: Whether to bypass tips during calculations. This +// speeds up calculations considerably, and does not seem to have a noticeable +// effect on the results. +// @return A vector containing Unifrac distances. +// // @keywords internal // [[Rcpp::export(.unifrac_cpp)]] -Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, +Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree, bool weighted, bool bypass_tips){ - // Normalized matches the results given by weighted - - auto start = std::chrono::high_resolution_clock::now(); - su::BPTree tree = su::BPTree(rowTree); su::Assay table = su::Assay(assay); - std::string method = "unweighted"; std::unordered_set to_keep(table.obs_ids.begin(), table.obs_ids.end()); @@ -61,18 +65,13 @@ Rcpp::List unifrac_cpp(const Rcpp::NumericMatrix & assay, su::mat_t results = su::one_off(table, tree_sheared, weighted, bypass_tips); - //condensed_form is the main values, returned in result - //Sample_ids can be handled with a map? - //n_samples, cf_size, is_upper_triangle are single values that can be passed in some other way? - - auto stop = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(stop - start); + unsigned int n = results.condensed_form.size(); + Rcpp::NumericVector unifrac = Rcpp::NumericVector(n); - Rcpp::Rcout << "Main thread: " << duration.count() << "\n"; + for( unsigned int i = 0; i < n; i++ ){ + unifrac[i] = results.condensed_form[i]; + } - return Rcpp::List::create(Rcpp::Named("n_samples") = results.n_samples, - Rcpp::Named("is_upper_triangle") = results.is_upper_triangle, - Rcpp::Named("cf_size") = results.cf_size, - Rcpp::Named("c_form") = results.condensed_form); + return unifrac; } diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp index bb9e59722..db21a3ed5 100644 --- a/src/unifrac_task.cpp +++ b/src/unifrac_task.cpp @@ -18,25 +18,23 @@ #include "tree.h" #include "unifrac_task.h" -void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vector lengths) { - - //Parameter finding - - //Task parameters determine stuff +void su::UnifracUnweightedTask::_run(unsigned int filled_embs, + std::vector lengths){ const uint64_t start_idx = this->task_p.start; const uint64_t stop_idx = this->task_p.stop; const uint64_t n_samples = this->task_p.n_samples; const uint64_t n_samples_r = this->dm_stripes.n_samples_r; const uint64_t step_size = su::UnifracUnweightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; const uint64_t filled_embs_els = filled_embs/64; const uint64_t filled_embs_rem = filled_embs%64; const uint64_t filled_embs_els_round = (filled_embs+63)/64; - // pre-compute sums of length elements, since they are likely to be accessed many times + // pre-compute sums of length elements, since they are likely to be accessed + // many times // We will use a 8-bit map, to keep it small enough to keep in L1 cache for (uint64_t emb_el=0; emb_el> 0) & 1) * lengths[len_off + 0]) + (((b8_i >> 1) & 1) * lengths[len_off + 1]) + - (((b8_i >> 2) & 1) * lengths[len_off + 2]) + (((b8_i >> 3) & 1) * lengths[len_off + 3]) + - (((b8_i >> 4) & 1) * lengths[len_off + 4]) + (((b8_i >> 5) & 1) * lengths[len_off + 5]) + - (((b8_i >> 6) & 1) * lengths[len_off + 6]) + (((b8_i >> 7) & 1) * lengths[len_off + 7]); + for (uint64_t b8_i=0; b8_i<0x100; b8_i++){ + sums[(emb8<<8) + b8_i] = + (((b8_i >> 0) & 1) * lengths[len_off + 0]) + + (((b8_i >> 1) & 1) * lengths[len_off + 1]) + + (((b8_i >> 2) & 1) * lengths[len_off + 2]) + + (((b8_i >> 3) & 1) * lengths[len_off + 3]) + + (((b8_i >> 4) & 1) * lengths[len_off + 4]) + + (((b8_i >> 5) & 1) * lengths[len_off + 5]) + + (((b8_i >> 6) & 1) * lengths[len_off + 6]) + + (((b8_i >> 7) & 1) * lengths[len_off + 7]); } } } - if (filled_embs_rem>0) { // add also the overflow elements + if (filled_embs_rem>0){ // add also the overflow elements const uint64_t emb_el=filled_embs_els; for (uint64_t sub8=0; sub8<8; sub8++) { // we are summing we have enough buffer in sums @@ -84,11 +87,11 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, std::vectordm_stripes.buf[idx + k] += my_stripe; - this->dm_stripes_total.buf[idx + k] += my_stripe_total; + dm_stripes.buf[idx + k] += my_stripe; + dm_stripes_total.buf[idx + k] += my_stripe_total; } - } } } } -void su::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, std::vector lengths) { - +void su::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, + std::vector lengths){ //Parameter finding @@ -165,7 +167,7 @@ void su::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, std::vect const uint64_t n_samples_r = this->dm_stripes.n_samples_r; const uint64_t step_size = su::UnifracNormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; //std::vector zcheck = this->zcheck; //std::vector sums = this->sums; @@ -192,7 +194,8 @@ void su::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, std::vect for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; // within-stripe index (0:n_samples-1) + // within-stripe index (0:n_samples-1) + const uint64_t k = sk*step_size + ik; if (k>=n_samples) continue; // past the limit @@ -214,19 +217,20 @@ void su::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, std::vect if (allzero_k || allzero_l1) { // one side has all zeros // we can use the distributed property, and use the pre-computed values - - const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 - k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 - + + // if (nonzero_l1), ridx = fabs(k-l1) = l1 with k==0 + // if (nonzero_k), ridx = fabs(k-l1) = k with l1==0 + const uint64_t ridx = (allzero_k) ? l1 : k; + // keep reads in the same place to maximize GPU warp performance my_stripe = sums[ridx]; } else { // both sides non zero, use the explicit but slow approach - + my_stripe = 0.0; - for (uint64_t emb=0; embdm_stripes.buf[idx + k] += my_stripe; + dm_stripes.buf[idx + k] += my_stripe; } } // for ik diff --git a/src/unifrac_task.h b/src/unifrac_task.h index 855850835..4aedd769d 100644 --- a/src/unifrac_task.h +++ b/src/unifrac_task.h @@ -7,8 +7,8 @@ * See LICENSE file for more details */ -#ifndef __UNIFRAC_TASKS -#define __UNIFRAC_TASKS 1 +#ifndef __UNIFRAC_TASK_H +#define __UNIFRAC_TASK_H 1 #include "stripemap.h" @@ -64,6 +64,7 @@ namespace su { std::vector() : std::vector(n_samples_r*(task_p.stop-start_idx), 0.0)) // dm_stripes could be null, in which case keep it null { + if (!buf.empty()) { //Initialize buffer to dm_stripe values for(unsigned int stripe=start_idx; stripe < task_p.stop; stripe++) { @@ -79,7 +80,7 @@ namespace su { //Destructor copies the buffer values back into dm_stripe ~UnifracTaskVector() - { + { if (!buf.empty()) { for(unsigned int stripe=start_idx; stripe < task_p.stop; stripe++) { std::vector vec = dm_stripes.get(stripe); @@ -89,6 +90,7 @@ namespace su { dm_stripes.update(stripe, vec); } } + } private: @@ -381,4 +383,4 @@ namespace su { } -#endif +#endif /* __UNIFRAC_TASK_H */ From 482e65a0658e807ad7576ea5a0b28e4dabb3cc76 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Fri, 22 May 2026 13:25:49 +0300 Subject: [PATCH 38/48] Add C++ function to R code --- NAMESPACE | 6 ------ R/RcppExports.R | 8 ++++---- R/calculateUnifrac.R | 10 +++++++--- src/RcppExports.cpp | 21 +++++++++++++++------ 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 448801eaa..b0f050187 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -368,10 +368,7 @@ importFrom(MultiAssayExperiment,MultiAssayExperiment) importFrom(MultiAssayExperiment,experiments) importFrom(MultiAssayExperiment,intersectColumns) importFrom(MultiAssayExperiment,sampleMap) -<<<<<<< HEAD -======= importFrom(Rcpp,evalCpp) ->>>>>>> upstream/devel importFrom(Rcpp,sourceCpp) importFrom(S4Vectors,"metadata<-") importFrom(S4Vectors,DataFrame) @@ -495,7 +492,4 @@ importFrom(vegan,rrarefy) importFrom(vegan,scores) importFrom(vegan,vegdist) useDynLib(mia) -<<<<<<< HEAD -======= useDynLib(mia, .registration = TRUE) ->>>>>>> upstream/devel diff --git a/R/RcppExports.R b/R/RcppExports.R index d3dbe1891..5044dbcdb 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -2,14 +2,14 @@ # Generator token: 10BE3573-1514-4C36-9D1C-5A225CD40393 .faith_cpp <- function(assay, rowTree) { -<<<<<<< HEAD - .Call('_mia_faith_cpp', PACKAGE = 'mia', assay, rowTree) -======= .Call(`_mia_faith_cpp`, assay, rowTree) } .apply_transformation_difference_or_division <- function(mat, method = "difference") { .Call(`_mia_apply_transformation_difference_or_division`, mat, method) ->>>>>>> upstream/devel +} + +.unifrac_cpp <- function(assay, rowTree, weighted, bypass_tips) { + .Call(`_mia_unifrac_cpp`, assay, rowTree, weighted, bypass_tips) } diff --git a/R/calculateUnifrac.R b/R/calculateUnifrac.R index 903daae44..dc6640137 100644 --- a/R/calculateUnifrac.R +++ b/R/calculateUnifrac.R @@ -2,7 +2,7 @@ #' @importFrom ecodive weighted_unifrac #' @importFrom ecodive unweighted_unifrac .get_unifrac <- function( - x, tree, weighted = FALSE, node.label = nodeLab, nodeLab = NULL, ...){ + x, tree, new = FALSE, weighted = FALSE, node.label = nodeLab, nodeLab = NULL, ...){ # Transpose the matrix so that the orientation is the same as in other # dissimilatity methods x <- t(x) @@ -73,8 +73,12 @@ x <- .merge_assay_by_rows(x, node.label, ...) # Calculate unifrac. Use implementation from ecodive package - FUN <- if( weighted ) weighted_unifrac else unweighted_unifrac - res <- FUN(t(x), tree = tree) + if( new ){ + res <- .unifrac_cpp(x, tree, weighted, F) + } else { + FUN <- if( weighted ) weighted_unifrac else unweighted_unifrac + res <- FUN(t(x), tree = tree) + } return(res) } diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index 9899df0fb..da9b9ef92 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -22,11 +22,6 @@ BEGIN_RCPP return rcpp_result_gen; END_RCPP } -<<<<<<< HEAD - -static const R_CallMethodDef CallEntries[] = { - {"_mia_faith_cpp", (DL_FUNC) &_mia_faith_cpp, 2}, -======= // apply_transformation_difference_or_division S4 apply_transformation_difference_or_division(NumericMatrix mat, std::string method); RcppExport SEXP _mia_apply_transformation_difference_or_division(SEXP matSEXP, SEXP methodSEXP) { @@ -39,11 +34,25 @@ BEGIN_RCPP return rcpp_result_gen; END_RCPP } +// unifrac_cpp +Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix& assay, const Rcpp::List& rowTree, bool weighted, bool bypass_tips); +RcppExport SEXP _mia_unifrac_cpp(SEXP assaySEXP, SEXP rowTreeSEXP, SEXP weightedSEXP, SEXP bypass_tipsSEXP) { +BEGIN_RCPP + Rcpp::RObject rcpp_result_gen; + Rcpp::RNGScope rcpp_rngScope_gen; + Rcpp::traits::input_parameter< const Rcpp::NumericMatrix& >::type assay(assaySEXP); + Rcpp::traits::input_parameter< const Rcpp::List& >::type rowTree(rowTreeSEXP); + Rcpp::traits::input_parameter< bool >::type weighted(weightedSEXP); + Rcpp::traits::input_parameter< bool >::type bypass_tips(bypass_tipsSEXP); + rcpp_result_gen = Rcpp::wrap(unifrac_cpp(assay, rowTree, weighted, bypass_tips)); + return rcpp_result_gen; +END_RCPP +} static const R_CallMethodDef CallEntries[] = { {"_mia_faith_cpp", (DL_FUNC) &_mia_faith_cpp, 2}, {"_mia_apply_transformation_difference_or_division", (DL_FUNC) &_mia_apply_transformation_difference_or_division, 2}, ->>>>>>> upstream/devel + {"_mia_unifrac_cpp", (DL_FUNC) &_mia_unifrac_cpp, 4}, {NULL, NULL, 0} }; From 1a9354e7aa38e3ed35526483364b7235f8aab3a3 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Fri, 22 May 2026 14:00:59 +0300 Subject: [PATCH 39/48] Change weighted from normalized to unnormalized --- src/unifrac.cpp | 6 +- src/unifrac_task.cpp | 140 +++++++++++++++++++++---------------------- src/unifrac_task.h | 6 +- 3 files changed, 75 insertions(+), 77 deletions(-) diff --git a/src/unifrac.cpp b/src/unifrac.cpp index 6963dbd59..727b1e630 100644 --- a/src/unifrac.cpp +++ b/src/unifrac.cpp @@ -75,10 +75,10 @@ void su::unifrac(const su::Assay &table, table, tree, true, dm_stripes, dm_stripes_total, task_p ); } - //weighted normalized + //weighted unnormalized else { - unifracTT( - table, tree, true, dm_stripes, dm_stripes_total, + unifracTT( + table, tree, false, dm_stripes, dm_stripes_total, task_p ); } } diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp index db21a3ed5..23dd0e12b 100644 --- a/src/unifrac_task.cpp +++ b/src/unifrac_task.cpp @@ -155,96 +155,94 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, } -void su::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, - std::vector lengths){ - - //Parameter finding - +void su::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, std::vector lengths) { //Task parameters determine stuff const uint64_t start_idx = this->task_p.start; const uint64_t stop_idx = this->task_p.stop; const uint64_t n_samples = this->task_p.n_samples; const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - const uint64_t step_size = su::UnifracNormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; + // bool * const __restrict__ zcheck = this->zcheck; + // TFloat * const __restrict__ sums = this->sums; + + const uint64_t step_size = su::UnifracUnnormalizedWeightedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + + // check for zero values and pre-compute single column sums - //std::vector zcheck = this->zcheck; - //std::vector sums = this->sums; - for(uint64_t k=0; k=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const bool allzero_k = zcheck[k]; - const bool allzero_l1 = zcheck[l1]; - - if (allzero_k && allzero_l1) { - // nothing to do, would have to add 0 - } else { - const uint64_t idx = (stripe-start_idx) * n_samples_r; - - // the totals can always use the distributed property - this->dm_stripes_total.buf[idx + k] += sums[k] + sums[l1]; - - double my_stripe; - - if (allzero_k || allzero_l1) { - // one side has all zeros - // we can use the distributed property, and use the pre-computed values - - // if (nonzero_l1), ridx = fabs(k-l1) = l1 with k==0 - // if (nonzero_k), ridx = fabs(k-l1) = k with l1==0 - const uint64_t ridx = (allzero_k) ? l1 : k; - - // keep reads in the same place to maximize GPU warp performance - my_stripe = sums[ridx]; - - } else { - // both sides non zero, use the explicit but slow approach - - my_stripe = 0.0; - - for (uint64_t emb=0; emb=n_samples) continue; // past the limit + + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound + + const bool allzero_k = zcheck[k]; + const bool allzero_l1 = zcheck[l1]; + + if (allzero_k && allzero_l1) { + // nothing to do, would have to add 0 + } else { + double my_stripe; + + if (allzero_k || allzero_l1) { + // one side has all zeros + // we can use the distributed property, and use the pre-computed values + + const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 + k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 + + // keep reads in the same place to maximize GPU warp performance + my_stripe = sums[ridx]; + + } else { + // both sides non zero, use the explicit but slow approach + my_stripe = 0.0; + + for (uint64_t emb=0; emb { + class UnifracUnnormalizedWeightedTask : public UnifracTask { public: static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; - UnifracNormalizedWeightedTask(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) + UnifracUnnormalizedWeightedTask(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) { const unsigned int n_samples = this->task_p.n_samples; @@ -368,7 +368,7 @@ namespace su { sums = std::vector(n_samples, 0.0); } - virtual ~UnifracNormalizedWeightedTask() + virtual ~UnifracUnnormalizedWeightedTask() { } From c444f8fbe7639b803234401286083dfccfc8b731 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Sun, 24 May 2026 23:36:29 +0300 Subject: [PATCH 40/48] Add chunked processing for performance --- src/assay.cpp | 4 +-- src/assay.h | 2 +- src/propmap.h | 26 ++++++++++++++ src/stripemap.h | 6 ++-- src/unifrac.cpp | 84 ++++++++++++++++++++++++-------------------- src/unifrac_R.cpp | 1 - src/unifrac_task.cpp | 33 +++++++---------- src/unifrac_task.h | 30 ++++++++-------- 8 files changed, 105 insertions(+), 81 deletions(-) diff --git a/src/assay.cpp b/src/assay.cpp index 4f4f3397a..68c71c1ed 100644 --- a/src/assay.cpp +++ b/src/assay.cpp @@ -18,8 +18,8 @@ using namespace su; -Assay::Assay(const Rcpp::NumericMatrix & assay){ - table = assay; +Assay::Assay(const Rcpp::NumericMatrix & assay): + table(assay) { sample_ids = std::vector(); obs_ids = std::vector(); diff --git a/src/assay.h b/src/assay.h index 36a3133ca..0aff086c3 100644 --- a/src/assay.h +++ b/src/assay.h @@ -57,7 +57,7 @@ class Assay { bool normalize) const; private: - Rcpp::NumericMatrix table; // Access to raw sample counts in R's memory + const Rcpp::NumericMatrix & table; // Access to raw sample counts in R's memory std::vector get_sample_counts(); diff --git a/src/propmap.h b/src/propmap.h index 3bf3df900..bda0c02a1 100644 --- a/src/propmap.h +++ b/src/propmap.h @@ -32,6 +32,30 @@ class PropMap { uint32_t defaultsize; }; +// Helper class +// To allow chunked processing, stores PropMap with vecsize-sized vectors +class PropMapMulti { +public: + PropMapMulti(uint32_t _vecsize) + : vecsize(_vecsize) + , multi(get_num_stacks(), PropMap(DEF_VEC_SIZE)) // round up + {} + ~PropMapMulti(){} + + // Number of stacks = number of def_sizes that go in vecsize + // Rounding up ensures that there are always enough stacks for full vecsize + uint32_t get_num_stacks() const {return (vecsize + (DEF_VEC_SIZE-1)) / DEF_VEC_SIZE;} + // These are used only for passing the value to set_prop_range and embed_prop_range + uint32_t get_start(uint32_t idx) const {return idx*DEF_VEC_SIZE;} + uint32_t get_end(uint32_t idx) const {return std::min((idx+1)*DEF_VEC_SIZE, vecsize);} + PropMap & get_prop_map(uint32_t idx) {return multi[idx];} + +protected: + const uint32_t vecsize; // equal to number of samples + static const uint32_t DEF_VEC_SIZE = 1024; // size of the sub-vectors, small enough to fit in L1 cache + std::vector multi; // Holds a StripeMap for each chunk +}; + std::vector set_proportions(const BPTree & tree, uint32_t node, const Assay & table, PropMap & pm, @@ -45,6 +69,8 @@ std::vector set_proportions_range(const su::BPTree & tree, PropMap & pm, bool normalize = true); + + } #endif /* __FAITH_PROPMAP_H */ diff --git a/src/stripemap.h b/src/stripemap.h index b30721364..11f839f96 100644 --- a/src/stripemap.h +++ b/src/stripemap.h @@ -16,9 +16,11 @@ namespace su { +// StripeMap is just used to replace a vector of double pointers + class StripeMap { public: - StripeMap(uint32_t n_samples); + StripeMap(uint32_t vecsize); virtual ~StripeMap(); void clear(uint32_t i); void update(uint32_t i, std::vector vec); @@ -27,7 +29,7 @@ class StripeMap { private: std::unordered_map> stripe_map; - uint32_t vecsize; // equal to number of samples + uint32_t vecsize; uint32_t n_stripes; }; diff --git a/src/unifrac.cpp b/src/unifrac.cpp index 727b1e630..c1904f272 100644 --- a/src/unifrac.cpp +++ b/src/unifrac.cpp @@ -7,8 +7,6 @@ * See LICENSE file for more details */ -#include - #include "unifrac.h" #include "propmap.h" #include "stripemap.h" @@ -36,15 +34,14 @@ su::mat_t su::one_off(const su::Assay & table, task.n_samples = table.n_samples; - //This could potentially be threaded - //Wasn't in the code because doesn't work with openacc/openmp? - + su::unifrac(std::ref(table), std::ref(tree), std::ref(dm_stripes), std::ref(dm_stripes_total), weighted, task); + su::mat_t result; result.n_samples = table.n_samples; @@ -102,7 +99,7 @@ inline void su::unifracTT(const su::Assay & table, const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1) / UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up - su::PropMap propmap(table.n_samples); + su::PropMapMulti propmap_multi(table.n_samples); const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; @@ -155,55 +152,66 @@ inline void su::unifracTT(const su::Assay & table, * (see C) but that is small over large N. */ - unsigned int k = 0; // index in tree - const unsigned int max_k = (tree.nparens / 2) - 1; + unsigned int k = 0; // index in tree + const unsigned int max_k = (tree.nparens / 2) - 1; + const unsigned int num_prop_chunks = propmap_multi.get_num_stacks(); // num_prop_chunks = 1 + while (k node_proportions = su::set_proportions_range( - tree, - node, - table, - tstart, - tend, - propmap); - - if(task_p.bypass_tips && tree.isleaf(node)) - continue; + su::PropMap & propmap = propmap_multi.get_prop_map(ck); + const unsigned int tstart = propmap_multi.get_start(ck); + const unsigned int tend = propmap_multi.get_end(ck); + + unsigned int my_filled_emb = 0; + unsigned int my_k=k_start; - lengths[filled_emb] = tree.lengths[node]; - filled_emb++; + while ((my_filled_emb node_proportions = su::set_proportions_range( + tree, + node, + table, + tstart, + tend, + propmap); + + if(task_p.bypass_tips && tree.isleaf(node)) + continue; + + + if (ck==0) { // they all do the same thing, so enough for the first to update the global state + lengths[filled_emb] = tree.lengths[node]; + filled_emb++; + } + + taskObj.embed_proportions_range(node_proportions, + tstart, + tend, + my_filled_emb); + my_filled_emb++; + } - taskObj.embed_proportions_range(node_proportions, - tstart, - tend, - my_filled_emb); - my_filled_emb++; + if (ck==0) { // they all do the same thing, so enough for the first to update the global state + k=my_k; + } } - - k=my_k; taskObj._run(filled_emb,lengths); - filled_emb=0; + } //want_total is used if you want the results as a percentage of the total? diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index 30909f176..398416f62 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -19,7 +19,6 @@ #include "unifrac.h" - // Calculate Unifrac // // This function calculates Unifrac distances for a given assay and rowTree, diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp index 23dd0e12b..8b16e5d3e 100644 --- a/src/unifrac_task.cpp +++ b/src/unifrac_task.cpp @@ -11,15 +11,13 @@ #include #include #include -#include - -#include +#include #include "tree.h" #include "unifrac_task.h" void su::UnifracUnweightedTask::_run(unsigned int filled_embs, - std::vector lengths){ + const std::vector & lengths){ const uint64_t start_idx = this->task_p.start; const uint64_t stop_idx = this->task_p.stop; const uint64_t n_samples = this->task_p.n_samples; @@ -73,8 +71,8 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, // we are summing we have enough buffer in sums const uint64_t emb8 = emb_el*8+sub8; - // compute all the combinations for this block, set to 0 any past the limit - // as above + // compute all the combinations for this block, set to 0 any past + // the limit as above for (uint64_t b8_i=0; b8_i<0x100; b8_i++) { double val= 0; for (uint64_t li=(emb8*8); li lengths) { +void su::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, const std::vector & lengths) { + //Task parameters determine stuff const uint64_t start_idx = this->task_p.start; const uint64_t stop_idx = this->task_p.stop; const uint64_t n_samples = this->task_p.n_samples; const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - // bool * const __restrict__ zcheck = this->zcheck; - // TFloat * const __restrict__ sums = this->sums; - const uint64_t step_size = su::UnifracUnnormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // How many steps in sample_size? round up // check for zero values and pre-compute single column sums @@ -186,11 +181,10 @@ void su::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, std::ve zcheck[k] = all_zeros; } - // now do the real compute - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { + for(uint64_t sk = 0; sk < sample_steps ; sk++) { for(uint64_t ik = 0; ik < step_size ; ik++) { // within-stripe index (0:n_samples-1) @@ -209,6 +203,7 @@ void su::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, std::ve double my_stripe; if (allzero_k || allzero_l1) { + // one side has all zeros // we can use the distributed property, and use the pre-computed values @@ -221,22 +216,18 @@ void su::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, std::ve } else { // both sides non zero, use the explicit but slow approach my_stripe = 0.0; - for (uint64_t emb=0; emb() : @@ -115,7 +115,7 @@ namespace su { su::task_parameters task_p; const unsigned int max_embs; - std::vector embedded_proportions; //Continuous vector - each stripe has n_samples_r elements, for complex reasons? + std::vector embedded_proportions; //Continuous vector - each stripe has n_samples_r elements //Has at most max_embs stripes - when filled, results stored in task _run() and embeds cleared to continue UnifracTaskBase(su::StripeMap & _dm_stripes, @@ -164,8 +164,8 @@ namespace su { // Just copy from one buffer to another - std::vector embed_proportions_range_straight( - std::vector out, + void embed_proportions_range_straight( + std::vector & out, const std::vector & in, unsigned int start, unsigned int end, @@ -188,7 +188,6 @@ namespace su { out[offset + i] = 0.0; } } - return out; } @@ -252,7 +251,7 @@ namespace su { unsigned int end, unsigned int emb ) { - embedded_proportions = embed_proportions_range_straight(embedded_proportions,in,start,end,emb); + embed_proportions_range_straight(embedded_proportions,in,start,end,emb); } template<> inline unsigned int UnifracTaskBase::get_emb_els( @@ -305,9 +304,9 @@ namespace su { template class UnifracTask : public UnifracTaskBase { protected: - // Use one cache line on CPU - // On GPU, sharing a cache line is actually a good thing - static const unsigned int step_size = 16*4/sizeof(double); + // The number of doubles that can fit in a 64-bit cache line + // This is used to optimize L1 cache access during loops? + static const unsigned int step_size = 4; public: @@ -316,11 +315,10 @@ namespace su { virtual ~UnifracTask() {} - //Probably should return a vector? - virtual void run(unsigned int filled_embs, std::vector lengths) = 0; + virtual void run(unsigned int filled_embs, const std::vector & lengths) = 0; protected: - static const unsigned int RECOMMENDED_MAX_EMBS_STRAIGHT = 128-16; // a little less to leave a bit of space of maxed-out L1 + static const unsigned int RECOMMENDED_MAX_EMBS_STRAIGHT = 64-16; // a little less to leave a bit of space of maxed-out L1 // packed uses 32x less memory,so this should be 32x larger than straight... but there are additional structures, so use half of that static const unsigned int RECOMMENDED_MAX_EMBS_BOOL = 64*32; @@ -345,9 +343,9 @@ namespace su { virtual ~UnifracUnweightedTask() {} - virtual void run(unsigned int filled_embs, std::vector lengths) {_run(filled_embs, lengths);} + virtual void run(unsigned int filled_embs, const std::vector & lengths) {_run(filled_embs, lengths);} - void _run(unsigned int filled_embs, std::vector lengths); + void _run(unsigned int filled_embs, const std::vector & lengths); private: std::vector sums; // temp buffer }; @@ -372,9 +370,9 @@ namespace su { { } - virtual void run(unsigned int filled_embs, std::vector lengths) {_run(filled_embs, lengths);} + virtual void run(unsigned int filled_embs, const std::vector & lengths) {_run(filled_embs, lengths);} - void _run(unsigned int filled_embs, std::vector lengths); + void _run(unsigned int filled_embs, const std::vector & lengths); protected: // temp buffers std::vector zcheck; From e5eb464c6f21ba1f40935637a69870ad0eb1384e Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 25 May 2026 01:28:04 +0300 Subject: [PATCH 41/48] Remove unnecessary exit point --- src/unifrac.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/unifrac.cpp b/src/unifrac.cpp index c1904f272..f30cc713e 100644 --- a/src/unifrac.cpp +++ b/src/unifrac.cpp @@ -89,12 +89,6 @@ inline void su::unifracTT(const su::Assay & table, su::StripeMap & dm_stripes_total, const su::task_parameters & task_p) { - - if(table.n_samples != task_p.n_samples) { - fprintf(stderr, "Task and table n_samples not equal\n"); - exit(EXIT_FAILURE); - } - const unsigned int n_samples = task_p.n_samples; const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1) / UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up From 7468ed6073c4710d6a742018850d400c5c4367dd Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 25 May 2026 17:40:07 +0300 Subject: [PATCH 42/48] Improve formatting --- src/assay.cpp | 3 +- src/propmap.cpp | 36 +++- src/propmap.h | 38 ++-- src/stripemap.cpp | 11 +- src/stripemap.h | 8 +- src/tree.cpp | 3 +- src/unifrac.cpp | 163 ++++++++-------- src/unifrac_R.cpp | 2 +- src/unifrac_task.cpp | 220 ++++++++++++++++----- src/unifrac_task.h | 449 +++++++++++++++++-------------------------- 10 files changed, 486 insertions(+), 447 deletions(-) diff --git a/src/assay.cpp b/src/assay.cpp index 68c71c1ed..7056ac325 100644 --- a/src/assay.cpp +++ b/src/assay.cpp @@ -38,8 +38,7 @@ Assay::Assay(const Rcpp::NumericMatrix & assay): sample_counts = get_sample_counts(); } -Assay::~Assay(){ -} +Assay::~Assay(){} void Assay::create_id_index(std::vector &ids, std::unordered_map &map){ diff --git a/src/propmap.cpp b/src/propmap.cpp index 1f2e798a6..5fd3de59e 100644 --- a/src/propmap.cpp +++ b/src/propmap.cpp @@ -22,8 +22,7 @@ PropMap::PropMap(uint32_t vecsize) prop_map.reserve(1000); } -PropMap::~PropMap() { -} +PropMap::~PropMap(){} std::vector PropMap::get(uint32_t i){ if( prop_map.count(i) > 0 ){ @@ -41,6 +40,36 @@ void PropMap::update(uint32_t node, std::vector vec){ prop_map[node] = vec; } + + +PropMapMulti::PropMapMulti(uint32_t _vecsize) + : vecsize(_vecsize) + , multi(get_num_stacks(), PropMap(DEF_VEC_SIZE)) {} + +PropMapMulti::~PropMapMulti(){} + +// Number of stacks = number of def_sizes that go in vecsize +// Rounding up ensures that there are always enough stacks for full vecsize +uint32_t PropMapMulti::get_num_stacks() const { + return (vecsize + (DEF_VEC_SIZE-1)) / DEF_VEC_SIZE; +} + +// get_start and get_end are used only for passing the values to set_prop_range +// and embed_prop_range +uint32_t PropMapMulti::get_start(uint32_t idx) const { + return idx*DEF_VEC_SIZE; +} + +uint32_t PropMapMulti::get_end(uint32_t idx) const { + return std::min((idx+1)*DEF_VEC_SIZE, vecsize); +} + +PropMap & PropMapMulti::get_prop_map(uint32_t idx){ + return multi[idx]; +} + + + std::vector su::set_proportions(const BPTree & tree, uint32_t node, const Assay & table, @@ -75,14 +104,13 @@ std::vector su::set_proportions(const BPTree & tree, return(props); } - std::vector su::set_proportions_range(const su::BPTree & tree, uint32_t node, const su::Assay & table, unsigned int start, unsigned int end, PropMap & pm, - bool normalize) { + bool normalize){ const unsigned int els = end-start; std::vector props = std::vector(els, 0.0); if(tree.isleaf(node)) { diff --git a/src/propmap.h b/src/propmap.h index bda0c02a1..62dbca527 100644 --- a/src/propmap.h +++ b/src/propmap.h @@ -32,28 +32,22 @@ class PropMap { uint32_t defaultsize; }; -// Helper class -// To allow chunked processing, stores PropMap with vecsize-sized vectors +// Helper class that splits the full proportions vector into smaller chunks of +// pre-defined size class PropMapMulti { -public: - PropMapMulti(uint32_t _vecsize) - : vecsize(_vecsize) - , multi(get_num_stacks(), PropMap(DEF_VEC_SIZE)) // round up - {} - ~PropMapMulti(){} - - // Number of stacks = number of def_sizes that go in vecsize - // Rounding up ensures that there are always enough stacks for full vecsize - uint32_t get_num_stacks() const {return (vecsize + (DEF_VEC_SIZE-1)) / DEF_VEC_SIZE;} - // These are used only for passing the value to set_prop_range and embed_prop_range - uint32_t get_start(uint32_t idx) const {return idx*DEF_VEC_SIZE;} - uint32_t get_end(uint32_t idx) const {return std::min((idx+1)*DEF_VEC_SIZE, vecsize);} - PropMap & get_prop_map(uint32_t idx) {return multi[idx];} - -protected: - const uint32_t vecsize; // equal to number of samples - static const uint32_t DEF_VEC_SIZE = 1024; // size of the sub-vectors, small enough to fit in L1 cache - std::vector multi; // Holds a StripeMap for each chunk + public: + PropMapMulti(uint32_t _vecsize); + ~PropMapMulti(); + + uint32_t get_num_stacks() const; + uint32_t get_start(uint32_t idx) const; + uint32_t get_end(uint32_t idx) const; + PropMap & get_prop_map(uint32_t idx); + + private: + const uint32_t vecsize; // Size of the full vector, equal to n_samples + static const uint32_t DEF_VEC_SIZE = 1024; // size of the sub-vectors + std::vector multi; }; std::vector set_proportions(const BPTree & tree, uint32_t node, @@ -69,8 +63,6 @@ std::vector set_proportions_range(const su::BPTree & tree, PropMap & pm, bool normalize = true); - - } #endif /* __FAITH_PROPMAP_H */ diff --git a/src/stripemap.cpp b/src/stripemap.cpp index 2a73909ff..a478857d2 100644 --- a/src/stripemap.cpp +++ b/src/stripemap.cpp @@ -7,24 +7,21 @@ * See LICENSE file for more details */ -#include "tree.h" -#include "assay.h" #include "stripemap.h" using namespace su; -StripeMap::StripeMap(uint32_t n_samples) +StripeMap::StripeMap(uint32_t _n_samples) : stripe_map() - , vecsize(n_samples) + , n_samples(_n_samples) { n_stripes = (n_samples + 1) / 2; for( unsigned int i = 0; i < n_stripes; i++ ){ - this->update(i, std::vector(vecsize, 0.0)); + this->update(i, std::vector(n_samples, 0.0)); } } -StripeMap::~StripeMap(){ -} +StripeMap::~StripeMap(){} std::vector StripeMap::get(uint32_t i){ if( stripe_map.count(i) > 0 ){ diff --git a/src/stripemap.h b/src/stripemap.h index 11f839f96..879e8e203 100644 --- a/src/stripemap.h +++ b/src/stripemap.h @@ -13,15 +13,15 @@ #include #include #include +#include namespace su { -// StripeMap is just used to replace a vector of double pointers - class StripeMap { public: - StripeMap(uint32_t vecsize); + StripeMap(uint32_t n_samples); virtual ~StripeMap(); + void clear(uint32_t i); void update(uint32_t i, std::vector vec); std::vector get(uint32_t i); @@ -29,7 +29,7 @@ class StripeMap { private: std::unordered_map> stripe_map; - uint32_t vecsize; + uint32_t n_samples; uint32_t n_stripes; }; diff --git a/src/tree.cpp b/src/tree.cpp index 480d92cfd..a7a042390 100644 --- a/src/tree.cpp +++ b/src/tree.cpp @@ -167,8 +167,7 @@ BPTree BPTree::collapse() { return this->mask(collapsemask, new_lengths); } -BPTree::~BPTree(){ -} +BPTree::~BPTree(){} void BPTree::index_and_cache(){ // Should probably do the open/close in here too diff --git a/src/unifrac.cpp b/src/unifrac.cpp index f30cc713e..0db135aa1 100644 --- a/src/unifrac.cpp +++ b/src/unifrac.cpp @@ -15,7 +15,7 @@ su::mat_t su::one_off(const su::Assay & table, const su::BPTree & tree, bool weighted, - bool bypass_tips) { + bool bypass_tips){ //Number of stripes to be used, basically half of samples const unsigned int stripe_stop = (table.n_samples + 1) / 2; @@ -56,23 +56,19 @@ su::mat_t su::one_off(const su::Assay & table, return result; } - - - void su::unifrac(const su::Assay &table, const su::BPTree &tree, su::StripeMap & dm_stripes, su::StripeMap & dm_stripes_total, bool weighted, - const su::task_parameters task_p) -{ - //unweighted - if (weighted == false) { + const su::task_parameters task_p){ + // unweighted + if (weighted == false){ unifracTT( table, tree, true, dm_stripes, dm_stripes_total, task_p ); } - //weighted unnormalized + // weighted unnormalized else { unifracTT( table, tree, false, dm_stripes, dm_stripes_total, @@ -80,15 +76,13 @@ void su::unifrac(const su::Assay &table, } } - template inline void su::unifracTT(const su::Assay & table, - const su::BPTree & tree, - const bool want_total, - su::StripeMap & dm_stripes, - su::StripeMap & dm_stripes_total, - const su::task_parameters & task_p) -{ + const su::BPTree & tree, + const bool want_total, + su::StripeMap & dm_stripes, + su::StripeMap & dm_stripes_total, + const su::task_parameters & task_p){ const unsigned int n_samples = task_p.n_samples; const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1) / UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up @@ -102,51 +96,49 @@ inline void su::unifracTT(const su::Assay & table, std::vector lengths = std::vector(max_emb); /* - * The values in the example vectors correspond to index positions of an - * element in the resulting distance matrix. So, in the example below, - * the following can be interpreted: - * - * [0 1 2] - * [1 2 3] - * - * As comparing the sample for row 0 against the sample for col 1, the - * sample for row 1 against the sample for col 2, the sample for row 2 - * against the sample for col 3. - * - * In other words, we're computing stripes of a distance matrix. In the - * following example, we're computing over 6 samples requiring 3 - * stripes. - * - * A; stripe == 0 - * [0 1 2 3 4 5] - * [1 2 3 4 5 0] - * - * B; stripe == 1 - * [0 1 2 3 4 5] - * [2 3 4 5 0 1] - * - * C; stripe == 2 - * [0 1 2 3 4 5] - * [3 4 5 0 1 2] - * - * The stripes end up computing the following positions in the distance - * matrix. - * - * x A B C x x - * x x A B C x - * x x x A B C - * C x x x A B - * B C x x x A - * A B C x x x - * - * However, we store those stripes as vectors, ie - * [ A A A A A A ] - * - * We end up performing N / 2 redundant calculations on the last stripe - * (see C) but that is small over large N. - */ - - + * The values in the example vectors correspond to index positions of an + * element in the resulting distance matrix. So, in the example below, + * the following can be interpreted: + * + * [0 1 2] + * [1 2 3] + * + * As comparing the sample for row 0 against the sample for col 1, the + * sample for row 1 against the sample for col 2, the sample for row 2 + * against the sample for col 3. + * + * In other words, we're computing stripes of a distance matrix. In the + * following example, we're computing over 6 samples requiring 3 + * stripes. + * + * A; stripe == 0 + * [0 1 2 3 4 5] + * [1 2 3 4 5 0] + * + * B; stripe == 1 + * [0 1 2 3 4 5] + * [2 3 4 5 0 1] + * + * C; stripe == 2 + * [0 1 2 3 4 5] + * [3 4 5 0 1 2] + * + * The stripes end up computing the following positions in the distance + * matrix. + * + * x A B C x x + * x x A B C x + * x x x A B C + * C x x x A B + * B C x x x A + * A B C x x x + * + * However, we store those stripes as vectors, ie + * [ A A A A A A ] + * + * We end up performing N / 2 redundant calculations on the last stripe + * (see C) but that is small over large N. + */ unsigned int k = 0; // index in tree const unsigned int max_k = (tree.nparens / 2) - 1; @@ -154,14 +146,12 @@ inline void su::unifracTT(const su::Assay & table, const unsigned int num_prop_chunks = propmap_multi.get_num_stacks(); // num_prop_chunks = 1 - while (k node_proportions = su::set_proportions_range( - tree, - node, - table, - tstart, - tend, - propmap); + std::vector node_proportions + = su::set_proportions_range(tree, + node, + table, + tstart, + tend, + propmap); - if(task_p.bypass_tips && tree.isleaf(node)) + if (task_p.bypass_tips && tree.isleaf(node)){ continue; + } - - if (ck==0) { // they all do the same thing, so enough for the first to update the global state + // they all do the same thing, so enough for the first to update + // the global state + if (ck==0){ lengths[filled_emb] = tree.lengths[node]; filled_emb++; } @@ -198,18 +190,17 @@ inline void su::unifracTT(const su::Assay & table, my_filled_emb++; } - if (ck==0) { // they all do the same thing, so enough for the first to update the global state + // they all do the same thing, so enough for the first to update the + // global state + if (ck==0){ k=my_k; } } - taskObj._run(filled_emb,lengths); filled_emb=0; - } - //want_total is used if you want the results as a percentage of the total? - if(want_total) { + if(want_total){ const uint64_t start_idx = task_p.start; const uint64_t stop_idx = task_p.stop; @@ -233,21 +224,19 @@ std::vector su::stripes_to_condensed_form(su::StripeMap & stripes, uint64_t comb_N = comb_2(n); std::vector cf = std::vector(comb_N, 0.0); - for(unsigned int stripe = start; stripe < stop; stripe++) { - //Does stripemap contain all the stripes or just one thread's stripes? + for(unsigned int stripe = start; stripe < stop; stripe++){ std::vector dm_stripe = stripes.get(stripe); // compute the (i, j) position of each element in each stripe uint64_t i = 0; uint64_t j = stripe + 1; - for(uint64_t k = 0; k < n; k++, i++, j++) { - if(j == n) { + for(uint64_t k = 0; k < n; k++, i++, j++){ + if(j == n){ i = 0; j = n - (stripe + 1); } // determine the position in the condensed form vector for a given // (i, j) - // based off of - // https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.squareform.html + // based on https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.squareform.html uint64_t comb_N_minus_i = comb_2(n - i); cf[comb_N - comb_N_minus_i + (j - i - 1)] = dm_stripe[k]; } diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index 398416f62..fb5a244dc 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -67,7 +67,7 @@ Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, unsigned int n = results.condensed_form.size(); Rcpp::NumericVector unifrac = Rcpp::NumericVector(n); - for( unsigned int i = 0; i < n; i++ ){ + for(unsigned int i = 0; i < n; i++){ unifrac[i] = results.condensed_form[i]; } diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp index 8b16e5d3e..205fcc9ed 100644 --- a/src/unifrac_task.cpp +++ b/src/unifrac_task.cpp @@ -16,14 +16,119 @@ #include "tree.h" #include "unifrac_task.h" -void su::UnifracUnweightedTask::_run(unsigned int filled_embs, - const std::vector & lengths){ +using namespace su; + +UnifracTaskVector::UnifracTaskVector(su::StripeMap & _dm_stripes, + const su::task_parameters _task_p) + : dm_stripes(_dm_stripes) + , start_idx(_task_p.start) + , n_samples(_task_p.n_samples) + , n_samples_r(((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK) + , task_p(_task_p) +{ + + // The buffer is only needed if the stripes are non-empty + buf = (dm_stripes.is_empty(start_idx)) ? std::vector() : + std::vector(n_samples_r* + (task_p.stop-start_idx), 0.0); + // Copy stripe values to buffer + if (!buf.empty()){ + for(unsigned int stripe=start_idx; stripe < task_p.stop; stripe++) { + std::vector dm_stripe = dm_stripes.get(stripe); + for(unsigned int k=0; k < dm_stripe.size(); k++) { + buf[ (stripe-start_idx)*n_samples_r + k ] = dm_stripe[k]; + } + } + } +} + +UnifracTaskVector::~UnifracTaskVector(){ + // If the buffer isn't empty, copy its values back to the stripes + if (!buf.empty()){ + for(unsigned int stripe=start_idx; stripe < task_p.stop; stripe++) { + std::vector dm_stripe = dm_stripes.get(stripe); + for(unsigned int k=0; k < dm_stripe.size(); k++) { + dm_stripe[k] = buf[ (stripe-start_idx)*n_samples_r + k ]; + } + dm_stripes.update(stripe, dm_stripe); + } + } +} + + + +template +UnifracTaskBase::UnifracTaskBase(su::StripeMap & _dm_stripes, + su::StripeMap & _dm_stripes_total, + unsigned int _max_embs, + su::task_parameters _task_p) + : dm_stripes(_dm_stripes,_task_p) + , dm_stripes_total(_dm_stripes_total,_task_p) + , task_p(_task_p) + , max_embs(_max_embs) +{ + embedded_proportions = initialize_embedded(dm_stripes.n_samples_r, + _max_embs); +} + +template +UnifracTaskBase::~UnifracTaskBase(){} + +template +std::vector UnifracTaskBase::initialize_embedded( + const uint64_t n_samples_r, + unsigned int max_embs){ + uint64_t bsize = n_samples_r * get_emb_els(max_embs); + return std::vector(bsize); +} + +template +void UnifracTaskBase::embed_proportions(const std::vector & in, + unsigned int emb){ + embed_proportions_range(in,0,dm_stripes.n_samples,emb); +} + + +template +UnifracTask::UnifracTask(su::StripeMap & _dm_stripes, + su::StripeMap & _dm_stripes_total, + unsigned int _max_embs, + su::task_parameters _task_p) + : UnifracTaskBase(_dm_stripes, + _dm_stripes_total, + _max_embs, + _task_p){} + +template +UnifracTask::~UnifracTask(){} + + + +UnifracUnweightedTask::UnifracUnweightedTask(su::StripeMap & _dm_stripes, + su::StripeMap & _dm_stripes_total, + unsigned int _max_embs, + su::task_parameters _task_p) + : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) +{ + const unsigned int bsize = _max_embs*32; + sums = std::vector(bsize, 0.0); +} + +UnifracUnweightedTask::~UnifracUnweightedTask(){} + +void UnifracUnweightedTask::run(unsigned int filled_embs, + const std::vector & lengths){ + _run(filled_embs, lengths); +} + +void UnifracUnweightedTask::_run(unsigned int filled_embs, + const std::vector & lengths){ const uint64_t start_idx = this->task_p.start; const uint64_t stop_idx = this->task_p.stop; const uint64_t n_samples = this->task_p.n_samples; const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - const uint64_t step_size = su::UnifracUnweightedTask::step_size; + const uint64_t step_size = UnifracUnweightedTask::step_size; const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; const uint64_t filled_embs_els = filled_embs/64; @@ -34,11 +139,11 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, // pre-compute sums of length elements, since they are likely to be accessed // many times // We will use a 8-bit map, to keep it small enough to keep in L1 cache - for (uint64_t emb_el=0; emb_el pl = std::vector(8); + std::vector pl = std::vector(8); uint64_t len_off = emb8*8; @@ -51,7 +156,7 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, // ... // psum[255] = pl[1] +.. + pl[7] // + 0*pl[0] // psum[255] = pl[0] +pl[1] +.. + pl[7] - for (uint64_t b8_i=0; b8_i<0x100; b8_i++){ + for(uint64_t b8_i=0; b8_i<0x100; b8_i++){ sums[(emb8<<8) + b8_i] = (((b8_i >> 0) & 1) * lengths[len_off + 0]) + (((b8_i >> 1) & 1) * lengths[len_off + 1]) + @@ -67,15 +172,15 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, if (filled_embs_rem>0){ // add also the overflow elements const uint64_t emb_el=filled_embs_els; - for (uint64_t sub8=0; sub8<8; sub8++) { + for(uint64_t sub8=0; sub8<8; sub8++){ // we are summing we have enough buffer in sums const uint64_t emb8 = emb_el*8+sub8; // compute all the combinations for this block, set to 0 any past // the limit as above - for (uint64_t b8_i=0; b8_i<0x100; b8_i++) { + for(uint64_t b8_i=0; b8_i<0x100; b8_i++){ double val= 0; - for (uint64_t li=(emb8*8); li> (li-(emb8*8))) & 1) * lengths[li]; } sums[(emb8<<8) + b8_i] = val; @@ -86,13 +191,12 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, // point of thread for(uint64_t sk = 0; sk < sample_steps ; sk++){ - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++){ - for(uint64_t ik = 0; ik < step_size ; ik++){ - - const uint64_t k = sk*step_size + ik; // within-stripe index (0:n_samples-1) - const uint64_t idx = (stripe-start_idx) * n_samples_r; //n_samples_r seems to relate to continuous buffer shenanigans + // within-stripe index (0:n_samples-1) + const uint64_t k = sk*step_size + ik; + //buffer index + const uint64_t idx = (stripe-start_idx) * n_samples_r; if (k>=n_samples) continue; // past the limit @@ -102,9 +206,8 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, double my_stripe = 0.0; double my_stripe_total = 0.0; - //This is the main calculation phase - - for (uint64_t emb_el=0; emb_el> 8) & 0xff)] + @@ -142,8 +247,8 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, } } - if (did_update) { - dm_stripes.buf[idx + k] += my_stripe; + if (did_update){ + dm_stripes.buf[idx + k] += my_stripe; dm_stripes_total.buf[idx + k] += my_stripe_total; } } @@ -152,24 +257,43 @@ void su::UnifracUnweightedTask::_run(unsigned int filled_embs, } -void su::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, const std::vector & lengths) { - - //Task parameters determine stuff + + +UnifracUnnormalizedWeightedTask::UnifracUnnormalizedWeightedTask( + su::StripeMap & _dm_stripes, + su::StripeMap & _dm_stripes_total, + unsigned int _max_embs, + su::task_parameters _task_p) + : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) +{ + const unsigned int n_samples = this->task_p.n_samples; + zcheck = std::vector(n_samples, 0); + sums = std::vector(n_samples, 0.0); +} + +UnifracUnnormalizedWeightedTask::~UnifracUnnormalizedWeightedTask(){} + +void UnifracUnnormalizedWeightedTask::run(unsigned int filled_embs, + const std::vector & lengths){ + _run(filled_embs, lengths); +} + +void UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, + const std::vector & lengths){ const uint64_t start_idx = this->task_p.start; const uint64_t stop_idx = this->task_p.stop; const uint64_t n_samples = this->task_p.n_samples; const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - const uint64_t step_size = su::UnifracUnnormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // How many steps in sample_size? round up + const uint64_t step_size = UnifracUnnormalizedWeightedTask::step_size; + const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // check for zero values and pre-compute single column sums - - for(uint64_t k=0; k the number of samples being processed - * start the first stripe to process - * stop the last stripe to process - * tid the thread identifier - * bypass_tips ignore tips on compute, reduces compute by ~50% - * g_unifrac_alpha an alpha value for generalized unifrac - */ - struct task_parameters { - uint32_t n_samples; // number of samples - unsigned int start; // starting stripe - unsigned int stop; // stopping stripe - unsigned int tid; // thread ID - bool bypass_tips; // avoid compute at tips - - // task specific arguments below - double g_unifrac_alpha; // generalized unifrac alpha - }; +/* task specific compute parameters +* +* n_samples the number of samples being processed +* start the first stripe to process +* stop the last stripe to process +* tid the thread identifier +* bypass_tips ignore tips on compute, reduces compute by ~50% +* g_unifrac_alpha an alpha value for generalized unifrac +*/ - // Note: This adds a copy, which is suboptimal - // But was the easiest way to get a contiguous buffer - // And it does allow for fp32 compute, when desired - - class UnifracTaskVector { - private: - const su::task_parameters task_p; +struct task_parameters { + uint32_t n_samples; // number of samples + unsigned int start; // starting stripe + unsigned int stop; // stopping stripe + unsigned int tid; // thread ID + bool bypass_tips; // avoid compute at tips +}; +// Helper class that manages stripes +class UnifracTaskVector { public: - su::StripeMap & dm_stripes; - const unsigned int start_idx; - const unsigned int n_samples; - const uint64_t n_samples_r; - std::vector buf; - - UnifracTaskVector(su::StripeMap & _dm_stripes, - const su::task_parameters _task_p) - : task_p(_task_p), dm_stripes(_dm_stripes) - , start_idx(task_p.start), n_samples(task_p.n_samples) - , n_samples_r(((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK) // round up - //buf is just a new array with as many stripes as called for in task_p - //n_samples_r : n_samples rounded up to a multiple of UNIFRAC_BLOCK - //Originally this was a null comparison, we might need to check what it does specifically - , buf((dm_stripes.is_empty(start_idx)) ? - std::vector() : - std::vector(n_samples_r*(task_p.stop-start_idx), 0.0)) // dm_stripes could be null, in which case keep it null - { - - if (!buf.empty()) { - //Initialize buffer to dm_stripe values - for(unsigned int stripe=start_idx; stripe < task_p.stop; stripe++) { - std::vector dm_stripe = dm_stripes.get(stripe); - //copy stripe to appropriate segment of buffer - //The stripes themselves have n_samples elements, - //but in the buffer each stripe gets n_samples_r elements? - std::copy(std::begin(dm_stripe), std::end(dm_stripe), - std::begin(buf) + ((stripe-start_idx)*n_samples_r) ); - } - } - } - - //Destructor copies the buffer values back into dm_stripe - ~UnifracTaskVector() - { - if (!buf.empty()) { - for(unsigned int stripe=start_idx; stripe < task_p.stop; stripe++) { - std::vector vec = dm_stripes.get(stripe); - std::copy( std::begin(buf) + ((stripe-start_idx)*n_samples_r), - std::begin(buf) + ((stripe-start_idx)*n_samples_r) + n_samples, - std::begin(vec) ); - dm_stripes.update(stripe, vec); - } - } + su::StripeMap & dm_stripes; + const unsigned int start_idx; + const unsigned int n_samples; + const uint64_t n_samples_r; + std::vector buf; - } + UnifracTaskVector(su::StripeMap & _dm_stripes, + const su::task_parameters _task_p); + + //Destructor copies the buffer values back into dm_stripes + ~UnifracTaskVector(); private: - UnifracTaskVector() = delete; - UnifracTaskVector operator=(const UnifracTaskVector&other) const = delete; - }; - - - - - /***********************************************/ - - - // Base task class to be shared by all tasks - template - class UnifracTaskBase { + const su::task_parameters task_p; +}; + +// Base task class to be shared by all tasks +// Templated to allow proportions to be embedded as either doubles (weighted) or +// packed bools (unweighted) +template +class UnifracTaskBase { public: - //Two taskvectors for stripes and total UnifracTaskVector dm_stripes; UnifracTaskVector dm_stripes_total; su::task_parameters task_p; const unsigned int max_embs; - std::vector embedded_proportions; //Continuous vector - each stripe has n_samples_r elements - //Has at most max_embs stripes - when filled, results stored in task _run() and embeds cleared to continue + //Continuous vector - each stripe has n_samples_r elements + //Has at most max_embs stripes - when filled, results stored by the + //task's _run() function and embeds are cleared for the next batch + std::vector embedded_proportions; UnifracTaskBase(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, - su::task_parameters _task_p) - : dm_stripes(_dm_stripes,_task_p), - dm_stripes_total(_dm_stripes_total,_task_p), - task_p(_task_p), - max_embs(_max_embs), - embedded_proportions(initialize_embedded(dm_stripes.n_samples_r, - _max_embs)) - {} + su::task_parameters _task_p); - virtual ~UnifracTaskBase() {} + virtual ~UnifracTaskBase(); + // Templated function used when initializing embeds static unsigned int get_emb_els(unsigned int max_embs); static std::vector initialize_embedded( - const uint64_t n_samples_r, - unsigned int max_embs ) - { - uint64_t bsize = n_samples_r * get_emb_els(max_embs); - return std::vector(bsize); - } - - //Need to return a vector? - void embed_proportions_range( - const std::vector & in, - unsigned int start, - unsigned int end, - unsigned int emb); - - void embed_proportions( - const std::vector & in, - unsigned int emb) - { - embed_proportions_range(in,0,dm_stripes.n_samples,emb); - } + const uint64_t n_samples_r, + unsigned int max_embs); + // Store proportions from in into embedded_proportions + void embed_proportions(const std::vector & in, + unsigned int emb); + void embed_proportions_range(const std::vector & in, + unsigned int start, + unsigned int end, + unsigned int emb); - // - // ===== Internal, do not use directly ======= - // - - // Just copy from one buffer to another - - void embed_proportions_range_straight( - std::vector & out, - const std::vector & in, - unsigned int start, - unsigned int end, - unsigned int emb) const - { + protected: + void embed_proportions_range_straight(std::vector & out, + const std::vector & in, + unsigned int start, + unsigned int end, + unsigned int emb) const { const unsigned int n_samples = dm_stripes.n_samples; const uint64_t n_samples_r = dm_stripes.n_samples_r; const uint64_t offset = emb * n_samples_r; @@ -178,62 +107,59 @@ namespace su { //Copy to stripe indicated by emb //Stripes are all contained in in/out in one mass //Start/end aren't necessarily the whole stripe? - for(unsigned int i = start; i < end; i++) { + for(unsigned int i = start; i < end; i++){ out[offset + i] = in[i-start]; } - if (end==n_samples) { + if (end==n_samples){ // avoid NaNs - for(unsigned int i = n_samples; i < n_samples_r; i++) { + for(unsigned int i = n_samples; i < n_samples_r; i++){ out[offset + i] = 0.0; } } } - - // packed bool // Compute (in[:]>0) on each element, and store only the boolean bit. - // The output values are stored in a multi-byte format, one bit per emb index, - // so it will likely take multiple passes to store all the values - // - // Note: assumes we are processing emb in increasing order, starting from 0 - - //Only used with uint64_t + // The output values are stored in a multi-byte format, one bit per emb + // index, so it will likely take multiple passes to store all the values + // Note: assumes we are processing emb in increasing order, starting + // from 0 std::vector embed_proportions_range_bool( - std::vector out, - const std::vector & in, - unsigned int start, - unsigned int end, - unsigned int emb) const - { - + std::vector out, + const std::vector & in, + unsigned int start, + unsigned int end, + unsigned int emb) const { const unsigned int n_packed = sizeof(uint64_t)*8; const unsigned int n_samples = dm_stripes.n_samples; const uint64_t n_samples_r = dm_stripes.n_samples_r; - // The output values are stored in a multi-byte format, one bit per emb index - // Compute the element to store the bit into, as well as whichbit in that element - unsigned int emb_block = emb/n_packed; // beginning of the element block + // The output values are stored in a multi-byte format, one bit per + // emb index + // Compute the element to store the bit into, as well as which bit + // in that element + unsigned int emb_block = emb/n_packed; // beginning of block unsigned int emb_bit = emb%n_packed; // bit inside the elements const uint64_t offset = emb_block * n_samples_r; - if (emb_bit == 0) { + if (emb_bit == 0){ // assign for emb_bit==0, so it clears the other bits - // assumes we processing emb in increasing order, starting from 0 - for(unsigned int i = start; i < end; i++) { + // assumes we processing emb in increasing order starting from 0 + for(unsigned int i = start; i < end; i++){ out[offset + i] = (in[i - start] > 0); } - if (end == n_samples) { + if (end == n_samples){ // avoid NaNs for(unsigned int i = n_samples; i < n_samples_r; i++) { out[offset + i] = 0; } } - } else { + } + else { // just update my bit - for(unsigned int i = start; i < end; i++) { + for(unsigned int i = start; i < end; i++){ out[offset + i] |= (uint64_t(in[i-start] > 0) << emb_bit); } @@ -241,143 +167,128 @@ namespace su { } return out; } - }; - - - - template<> inline void UnifracTaskBase::embed_proportions_range( - const std::vector & in, - unsigned int start, - unsigned int end, - unsigned int emb ) - { - embed_proportions_range_straight(embedded_proportions,in,start,end,emb); - } - - template<> inline unsigned int UnifracTaskBase::get_emb_els( - unsigned int max_embs ) - { - return max_embs; - } - - - - - template<> inline void UnifracTaskBase::embed_proportions_range( - const std::vector & in, - unsigned int start, - unsigned int end, - unsigned int emb ) - { - embedded_proportions = embed_proportions_range_bool(embedded_proportions,in,start,end,emb); - } - - template<> inline unsigned int UnifracTaskBase::get_emb_els( - unsigned int max_embs ) - { - return (max_embs+63)/64; - } - - - - - - - /***********************************************/ - - /* void unifrac tasks - * - * all methods utilize the same function signature. that signature is as follows: - * - * dm_stripes vector the stripes of the distance matrix being accumulated - * into for unique branch length - * dm_stripes vector the stripes of the distance matrix being accumulated - * into for total branch length (e.g., to normalize unweighted unifrac) - * embedded_proportions the proportions vector for a sample, or rather - * the counts vector normalized to 1. this vector is embedded as it is - * duplicated: if A, B and C are proportions for features A, B, and C, the - * vector will look like [A B C A B C]. - * length the branch length of the current node to its parent. - * task_p task specific parameters. - */ +}; + + +template<> inline void UnifracTaskBase::embed_proportions_range( + const std::vector & in, + unsigned int start, + unsigned int end, + unsigned int emb){ + embed_proportions_range_straight(embedded_proportions,in,start,end,emb); +} + +template<> inline unsigned int UnifracTaskBase::get_emb_els( + unsigned int max_embs){ + return max_embs; +} + +template<> inline void UnifracTaskBase::embed_proportions_range( + const std::vector & in, + unsigned int start, + unsigned int end, + unsigned int emb){ + embedded_proportions = embed_proportions_range_bool(embedded_proportions, + in, + start, + end, + emb); +} + +template<> inline unsigned int UnifracTaskBase::get_emb_els( + unsigned int max_embs){ + return (max_embs+63)/64; +} - template - class UnifracTask : public UnifracTaskBase { - protected: - // The number of doubles that can fit in a 64-bit cache line - // This is used to optimize L1 cache access during loops? - static const unsigned int step_size = 4; - + +/* void unifrac tasks +* +* all methods utilize the same function signature. that signature is as follows: +* +* dm_stripes vector the stripes of the distance matrix being accumulated +* into for unique branch length +* dm_stripes vector the stripes of the distance matrix being accumulated +* into for total branch length (e.g., to normalize unweighted unifrac) +* embedded_proportions the proportions vector for a sample, or rather +* the counts vector normalized to 1. this vector is embedded as it is +* duplicated: if A, B and C are proportions for features A, B, and C, the +* vector will look like [A B C A B C]. +* length the branch length of the current node to its parent. +* task_p task specific parameters. +*/ + +template +class UnifracTask : public UnifracTaskBase { public: + UnifracTask(su::StripeMap & _dm_stripes, + su::StripeMap & _dm_stripes_total, + unsigned int _max_embs, + su::task_parameters _task_p); - UnifracTask(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) - : UnifracTaskBase(_dm_stripes, _dm_stripes_total, _max_embs, _task_p) {} + virtual ~UnifracTask(); - virtual ~UnifracTask() {} - - virtual void run(unsigned int filled_embs, const std::vector & lengths) = 0; + virtual void run(unsigned int filled_embs, + const std::vector & lengths) = 0; protected: - static const unsigned int RECOMMENDED_MAX_EMBS_STRAIGHT = 64-16; // a little less to leave a bit of space of maxed-out L1 - // packed uses 32x less memory,so this should be 32x larger than straight... but there are additional structures, so use half of that - static const unsigned int RECOMMENDED_MAX_EMBS_BOOL = 64*32; + // Controls the size of inner loops in the calculation phase + static const unsigned int step_size = 4; - }; - - /***********************************************/ + // Max embs are theoretically optimized for cache performance + static const unsigned int RECOMMENDED_MAX_EMBS_STRAIGHT = 64-16; + static const unsigned int RECOMMENDED_MAX_EMBS_BOOL = 64*32; +}; - //Simplify all template stuff into doubles - class UnifracUnweightedTask : public UnifracTask { + +class UnifracUnweightedTask : public UnifracTask { public: - static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_BOOL; + static const unsigned int RECOMMENDED_MAX_EMBS + = UnifracTask::RECOMMENDED_MAX_EMBS_BOOL; // Note: _max_emb MUST be multiple of 64 - UnifracUnweightedTask(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) - { - const unsigned int bsize = _max_embs*32; - sums = std::vector(bsize, 0.0); - } + UnifracUnweightedTask(su::StripeMap & _dm_stripes, + su::StripeMap & _dm_stripes_total, + unsigned int _max_embs, + su::task_parameters _task_p); - virtual ~UnifracUnweightedTask() {} + virtual ~UnifracUnweightedTask(); - virtual void run(unsigned int filled_embs, const std::vector & lengths) {_run(filled_embs, lengths);} + virtual void run(unsigned int filled_embs, + const std::vector & lengths); - void _run(unsigned int filled_embs, const std::vector & lengths); + void _run(unsigned int filled_embs, + const std::vector & lengths); private: std::vector sums; // temp buffer - }; +}; - /***********************************************/ - class UnifracUnnormalizedWeightedTask : public UnifracTask { - public: - static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; - - UnifracUnnormalizedWeightedTask(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) - { - const unsigned int n_samples = this->task_p.n_samples; - - zcheck = std::vector(n_samples, 0); - sums = std::vector(n_samples, 0.0); - } - - virtual ~UnifracUnnormalizedWeightedTask() - { - } - - virtual void run(unsigned int filled_embs, const std::vector & lengths) {_run(filled_embs, lengths);} - - void _run(unsigned int filled_embs, const std::vector & lengths); - protected: +class UnifracUnnormalizedWeightedTask : public UnifracTask { + public: + static const unsigned int RECOMMENDED_MAX_EMBS + = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; + + UnifracUnnormalizedWeightedTask(su::StripeMap & _dm_stripes, + su::StripeMap & _dm_stripes_total, + unsigned int _max_embs, + su::task_parameters _task_p); + + virtual ~UnifracUnnormalizedWeightedTask(); + + virtual void run(unsigned int filled_embs, + const std::vector & lengths); + + void _run(unsigned int filled_embs, + const std::vector & lengths); + + protected: // temp buffers std::vector zcheck; std::vector sums; - }; +}; } From f2cf6c48bfc9de6ff3fcc87d8825cb39361aa15b Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 25 May 2026 17:47:43 +0300 Subject: [PATCH 43/48] Remove bypass_tips from function parameters --- R/calculateUnifrac.R | 2 +- src/unifrac_R.cpp | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/R/calculateUnifrac.R b/R/calculateUnifrac.R index dc6640137..f16d14c45 100644 --- a/R/calculateUnifrac.R +++ b/R/calculateUnifrac.R @@ -74,7 +74,7 @@ # Calculate unifrac. Use implementation from ecodive package if( new ){ - res <- .unifrac_cpp(x, tree, weighted, F) + res <- .unifrac_cpp(x, tree, weighted) } else { FUN <- if( weighted ) weighted_unifrac else unweighted_unifrac res <- FUN(t(x), tree = tree) diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index fb5a244dc..0c0d0c83b 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -51,8 +51,7 @@ // [[Rcpp::export(.unifrac_cpp)]] Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree, - bool weighted, - bool bypass_tips){ + bool weighted){ su::BPTree tree = su::BPTree(rowTree); su::Assay table = su::Assay(assay); @@ -62,7 +61,7 @@ Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - su::mat_t results = su::one_off(table, tree_sheared, weighted, bypass_tips); + su::mat_t results = su::one_off(table, tree_sheared, weighted, false); unsigned int n = results.condensed_form.size(); Rcpp::NumericVector unifrac = Rcpp::NumericVector(n); From 702c2435a37b1f54134cdfc24845c8cf8829b603 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 25 May 2026 18:02:46 +0300 Subject: [PATCH 44/48] Update documentation --- R/addDissimilarity.R | 10 +++++++--- R/calculateUnifrac.R | 11 ++--------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/R/addDissimilarity.R b/R/addDissimilarity.R index e4d8e7661..844d80478 100644 --- a/R/addDissimilarity.R +++ b/R/addDissimilarity.R @@ -91,9 +91,8 @@ #' all the abundances of features are equal between two samples, and 0 means #' that samples have completely different relative abundances. #' -#' Unifrac is calculated with \code{ -#' \link[ecodive:unweighted_unifrac]{ecodive:unweighted_unifrac()}} -#' or \code{\link[ecodive:weighted_unifrac]{ecodive:weighted_unifrac()}}. +#' Unifrac is calculated with a C++ implementation of the Striped Unifrac +#' algorithm (McDonald et al. 2021). #' #' If rarefaction is enabled, \code{\link[vegan:avgdist]{vegan:avgdist()}} is #' utilized. @@ -131,6 +130,11 @@ #' #' Lozupone C, Knight R. ``Unifrac: a new phylogenetic method for comparing #' microbial communities.'' Appl Environ Microbiol. 2005 71 (12):8228-35. +#' +#' McDonald D, Vázquez-Baeza Y, Koslicki D, McClelland J, Reeve N, Xu Z, +#' Gonzalez A, Knight R. ``Striped UniFrac: enabling microbiome analysis at +#' unprecedented scale.'' Nat Methods. 2018 15 (11):847-848. +#' doi: 10.1038/s41592-018-0187-8. #' #' For JSD dissimilarity: #' Jensen-Shannon Divergence and Hilbert space embedding. diff --git a/R/calculateUnifrac.R b/R/calculateUnifrac.R index f16d14c45..bb9320cd5 100644 --- a/R/calculateUnifrac.R +++ b/R/calculateUnifrac.R @@ -1,6 +1,4 @@ #' @importFrom ape drop.tip -#' @importFrom ecodive weighted_unifrac -#' @importFrom ecodive unweighted_unifrac .get_unifrac <- function( x, tree, new = FALSE, weighted = FALSE, node.label = nodeLab, nodeLab = NULL, ...){ # Transpose the matrix so that the orientation is the same as in other @@ -72,13 +70,8 @@ # multiple rows are linked to single tip. x <- .merge_assay_by_rows(x, node.label, ...) - # Calculate unifrac. Use implementation from ecodive package - if( new ){ - res <- .unifrac_cpp(x, tree, weighted) - } else { - FUN <- if( weighted ) weighted_unifrac else unweighted_unifrac - res <- FUN(t(x), tree = tree) - } + # Calculate unifrac with C++ algorithm + res <- .unifrac_cpp(x, tree, weighted) return(res) } From e8ee4858b9b99cef64c82b9ed9c18dbf4b476b16 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 25 May 2026 19:41:28 +0300 Subject: [PATCH 45/48] Turn return object into a distance matrix --- src/assay.cpp | 3 +++ src/unifrac_R.cpp | 15 ++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/assay.cpp b/src/assay.cpp index 7056ac325..d396fc8c3 100644 --- a/src/assay.cpp +++ b/src/assay.cpp @@ -27,6 +27,9 @@ Assay::Assay(const Rcpp::NumericMatrix & assay): Rcpp::StringVector rownames = Rcpp::rownames(table); obs_ids = Rcpp::as>(rownames); + Rcpp::StringVector colnames = Rcpp::colnames(table); + sample_ids = Rcpp::as>(colnames); + n_samples = table.ncol(); n_obs = obs_ids.size(); diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index 0c0d0c83b..7162b1a0f 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -65,11 +65,24 @@ Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, unsigned int n = results.condensed_form.size(); Rcpp::NumericVector unifrac = Rcpp::NumericVector(n); - + // + // Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, + // Rcpp::Named("is_upper_triangle") = result->is_upper_triangle, + // Rcpp::Named("cf_size") = result->cf_size, + // Rcpp::Named("c_form") = cf) + // for(unsigned int i = 0; i < n; i++){ unifrac[i] = results.condensed_form[i]; } + unifrac.attr("class") = "dist"; + Rcpp::StringVector labels(table.n_samples); + labels = table.sample_ids; + unifrac.attr("Labels") = labels; + unifrac.attr("Size") = table.n_samples; + unifrac.attr("Diag") = false; + unifrac.attr("Upper") = false; + return unifrac; } From dc7b8feee97f281450850d661e9f939cc3a04325 Mon Sep 17 00:00:00 2001 From: Jesse Pasanen Date: Mon, 25 May 2026 19:41:28 +0300 Subject: [PATCH 46/48] Turn return object into a distance matrix --- R/RcppExports.R | 4 ++-- src/RcppExports.cpp | 9 ++++----- src/assay.cpp | 3 +++ src/unifrac_R.cpp | 15 ++++++++++++++- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/R/RcppExports.R b/R/RcppExports.R index 5044dbcdb..1f301c944 100644 --- a/R/RcppExports.R +++ b/R/RcppExports.R @@ -9,7 +9,7 @@ .Call(`_mia_apply_transformation_difference_or_division`, mat, method) } -.unifrac_cpp <- function(assay, rowTree, weighted, bypass_tips) { - .Call(`_mia_unifrac_cpp`, assay, rowTree, weighted, bypass_tips) +.unifrac_cpp <- function(assay, rowTree, weighted) { + .Call(`_mia_unifrac_cpp`, assay, rowTree, weighted) } diff --git a/src/RcppExports.cpp b/src/RcppExports.cpp index da9b9ef92..ec36b8f9b 100644 --- a/src/RcppExports.cpp +++ b/src/RcppExports.cpp @@ -35,16 +35,15 @@ BEGIN_RCPP END_RCPP } // unifrac_cpp -Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix& assay, const Rcpp::List& rowTree, bool weighted, bool bypass_tips); -RcppExport SEXP _mia_unifrac_cpp(SEXP assaySEXP, SEXP rowTreeSEXP, SEXP weightedSEXP, SEXP bypass_tipsSEXP) { +Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix& assay, const Rcpp::List& rowTree, bool weighted); +RcppExport SEXP _mia_unifrac_cpp(SEXP assaySEXP, SEXP rowTreeSEXP, SEXP weightedSEXP) { BEGIN_RCPP Rcpp::RObject rcpp_result_gen; Rcpp::RNGScope rcpp_rngScope_gen; Rcpp::traits::input_parameter< const Rcpp::NumericMatrix& >::type assay(assaySEXP); Rcpp::traits::input_parameter< const Rcpp::List& >::type rowTree(rowTreeSEXP); Rcpp::traits::input_parameter< bool >::type weighted(weightedSEXP); - Rcpp::traits::input_parameter< bool >::type bypass_tips(bypass_tipsSEXP); - rcpp_result_gen = Rcpp::wrap(unifrac_cpp(assay, rowTree, weighted, bypass_tips)); + rcpp_result_gen = Rcpp::wrap(unifrac_cpp(assay, rowTree, weighted)); return rcpp_result_gen; END_RCPP } @@ -52,7 +51,7 @@ END_RCPP static const R_CallMethodDef CallEntries[] = { {"_mia_faith_cpp", (DL_FUNC) &_mia_faith_cpp, 2}, {"_mia_apply_transformation_difference_or_division", (DL_FUNC) &_mia_apply_transformation_difference_or_division, 2}, - {"_mia_unifrac_cpp", (DL_FUNC) &_mia_unifrac_cpp, 4}, + {"_mia_unifrac_cpp", (DL_FUNC) &_mia_unifrac_cpp, 3}, {NULL, NULL, 0} }; diff --git a/src/assay.cpp b/src/assay.cpp index 7056ac325..d396fc8c3 100644 --- a/src/assay.cpp +++ b/src/assay.cpp @@ -27,6 +27,9 @@ Assay::Assay(const Rcpp::NumericMatrix & assay): Rcpp::StringVector rownames = Rcpp::rownames(table); obs_ids = Rcpp::as>(rownames); + Rcpp::StringVector colnames = Rcpp::colnames(table); + sample_ids = Rcpp::as>(colnames); + n_samples = table.ncol(); n_obs = obs_ids.size(); diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index 0c0d0c83b..7162b1a0f 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -65,11 +65,24 @@ Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, unsigned int n = results.condensed_form.size(); Rcpp::NumericVector unifrac = Rcpp::NumericVector(n); - + // + // Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, + // Rcpp::Named("is_upper_triangle") = result->is_upper_triangle, + // Rcpp::Named("cf_size") = result->cf_size, + // Rcpp::Named("c_form") = cf) + // for(unsigned int i = 0; i < n; i++){ unifrac[i] = results.condensed_form[i]; } + unifrac.attr("class") = "dist"; + Rcpp::StringVector labels(table.n_samples); + labels = table.sample_ids; + unifrac.attr("Labels") = labels; + unifrac.attr("Size") = table.n_samples; + unifrac.attr("Diag") = false; + unifrac.attr("Upper") = false; + return unifrac; } From b7cb6381c2f99ccd14d68344a4b1cf848834bc75 Mon Sep 17 00:00:00 2001 From: Tuomas Borman Date: Fri, 12 Jun 2026 14:35:47 +0300 Subject: [PATCH 47/48] up --- DESCRIPTION | 4 +- NEWS | 1 + R/addAlpha.R | 16 +++--- src/propmap.cpp | 32 +++++------ src/unifrac_R.cpp | 19 +++---- src/unifrac_task.cpp | 101 +++++++++++++++------------------ tests/testthat/test-5Unifrac.R | 2 + 7 files changed, 82 insertions(+), 93 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 286480e04..50eae803e 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: mia Type: Package -Version: 1.21.1 +Version: 1.21.2 Authors@R: c(person(given = "Tuomas", family = "Borman", role = c("aut", "cre"), email = "tuomas.v.borman@utu.fi", @@ -77,7 +77,6 @@ Imports: MASS, MatrixGenerics, methods, - ecodive, rlang, S4Vectors, scater, @@ -93,6 +92,7 @@ Suggests: BiocStyle, biomformat, dada2, + ecodive, knitr, mediation, miaTime, diff --git a/NEWS b/NEWS index 6c4b0852f..49e8b2d2b 100644 --- a/NEWS +++ b/NEWS @@ -190,3 +190,4 @@ Changes in version 1.19.x Changes in version 1.21.x + Fixed explained percentages on PCA axes for jointRPCA and hardened rank-deficient SVD handling in joint-RPCA (1.21.1, 2026-05-12) ++ Implement Striped Unifrac algorithm (1.21.2, 2026-06-12) diff --git a/R/addAlpha.R b/R/addAlpha.R index e6ce9ca50..5b2e6d3af 100644 --- a/R/addAlpha.R +++ b/R/addAlpha.R @@ -39,7 +39,7 @@ #' whether to remove internal nodes when Faith's index is calculated. #' When \code{only.tips=TRUE}, those rows that are not tips of tree are #' removed. (Default: \code{FALSE}) -#' +#' #' \item \code{threshold}: (Coverage and all evenness indices). #' \code{Numeric scalar}. #' From \code{0 to 1}, determines the threshold for coverage and evenness @@ -244,9 +244,9 @@ #' evenly the abundances of different species are distributed. The following #' evenness indices are provided: #' -#' By default, four indices are returned, each taking into account different -#' aspects: richness (the number of observed unique features), -#' dominance (Berger-Parker), information (Shannon), and phylogenetics (Faith) +#' By default, four indices are returned, each taking into account different +#' aspects: richness (the number of observed unique features), +#' dominance (Berger-Parker), information (Shannon), and phylogenetics (Faith) #' (Cassol et al., 2025). #' #' The available evenness indices include the following (all in lowercase): @@ -354,7 +354,7 @@ #' Refer to Schloss (2024) for more details on rarefaction. #' #' @references -#' +#' #' Armstrong G. et al. (2021) #' Efficient computation of Faith's phylogenetic diversity with applications #' in characterizing microbiomes. @@ -452,9 +452,9 @@ #' A tribute to Claude Shannon (1916 –2001) and a plea for more rigorous use of #' species richness, species diversity and the ‘Shannon–Wiener’ Index. #' _Alpha Ecology & Biogeography_ 12, 177–197. -#' -#' Cassol, I., Ibañez, M. & Bustamante, J.P. (2025) -#' Key features and guidelines for the application of microbial alpha diversity +#' +#' Cassol, I., Ibañez, M. & Bustamante, J.P. (2025) +#' Key features and guidelines for the application of microbial alpha diversity #' metrics. _Sci Rep_ 15, 622. doi:10.1038/s41598-024-77864-y #' #' @seealso diff --git a/src/propmap.cpp b/src/propmap.cpp index 5fd3de59e..a177b101e 100644 --- a/src/propmap.cpp +++ b/src/propmap.cpp @@ -15,7 +15,7 @@ using namespace su; -PropMap::PropMap(uint32_t vecsize) +PropMap::PropMap(uint32_t vecsize) : prop_map() , defaultsize(vecsize) { @@ -28,8 +28,8 @@ std::vector PropMap::get(uint32_t i){ if( prop_map.count(i) > 0 ){ return prop_map.at(i); } else { - return(std::vector()); - } + return(std::vector()); + } } void PropMap::clear(uint32_t i){ @@ -40,14 +40,12 @@ void PropMap::update(uint32_t node, std::vector vec){ prop_map[node] = vec; } - - PropMapMulti::PropMapMulti(uint32_t _vecsize) : vecsize(_vecsize) , multi(get_num_stacks(), PropMap(DEF_VEC_SIZE)) {} PropMapMulti::~PropMapMulti(){} - + // Number of stacks = number of def_sizes that go in vecsize // Rounding up ensures that there are always enough stacks for full vecsize uint32_t PropMapMulti::get_num_stacks() const { @@ -66,10 +64,8 @@ uint32_t PropMapMulti::get_end(uint32_t idx) const { PropMap & PropMapMulti::get_prop_map(uint32_t idx){ return multi[idx]; -} - +} - std::vector su::set_proportions(const BPTree & tree, uint32_t node, const Assay & table, @@ -87,19 +83,19 @@ std::vector su::set_proportions(const BPTree & tree, } else { unsigned int current = tree.leftchild(node); unsigned int right = tree.rightchild(node); - + while( current <= right && current != 0 ){ std::vector vec = pm.get(current); // Pull from prop map pm.clear(current); // Remove from prop map - + for( unsigned int i = 0; i < table.n_samples; i++ ){ props[i] = props[i] + vec[i]; } - + current = tree.rightsibling(current); } } - + pm.update(node, props); return(props); } @@ -119,19 +115,19 @@ std::vector su::set_proportions_range(const su::BPTree & tree, } else { unsigned int current = tree.leftchild(node); unsigned int right = tree.rightchild(node); - + while(current <= right && current != 0) { std::vector vec = pm.get(current); // pull from prop map pm.clear(current); // remove from prop map, place back on stack - + for(unsigned int i = 0; i < els; i++){ props[i] = props[i] + vec[i]; } - + current = tree.rightsibling(current); } } - + pm.update(node, props); return props; -} \ No newline at end of file +} diff --git a/src/unifrac_R.cpp b/src/unifrac_R.cpp index 7162b1a0f..5b882e155 100644 --- a/src/unifrac_R.cpp +++ b/src/unifrac_R.cpp @@ -52,29 +52,29 @@ Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, const Rcpp::List & rowTree, bool weighted){ - + su::BPTree tree = su::BPTree(rowTree); su::Assay table = su::Assay(assay); - + std::unordered_set to_keep(table.obs_ids.begin(), table.obs_ids.end()); - + su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - + su::mat_t results = su::one_off(table, tree_sheared, weighted, false); - + unsigned int n = results.condensed_form.size(); Rcpp::NumericVector unifrac = Rcpp::NumericVector(n); - // + // // Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, // Rcpp::Named("is_upper_triangle") = result->is_upper_triangle, // Rcpp::Named("cf_size") = result->cf_size, // Rcpp::Named("c_form") = cf) - // + // for(unsigned int i = 0; i < n; i++){ unifrac[i] = results.condensed_form[i]; } - + unifrac.attr("class") = "dist"; Rcpp::StringVector labels(table.n_samples); labels = table.sample_ids; @@ -82,7 +82,6 @@ Rcpp::NumericVector unifrac_cpp(const Rcpp::NumericMatrix & assay, unifrac.attr("Size") = table.n_samples; unifrac.attr("Diag") = false; unifrac.attr("Upper") = false; - + return unifrac; } - diff --git a/src/unifrac_task.cpp b/src/unifrac_task.cpp index 205fcc9ed..c35d32a51 100644 --- a/src/unifrac_task.cpp +++ b/src/unifrac_task.cpp @@ -26,7 +26,7 @@ UnifracTaskVector::UnifracTaskVector(su::StripeMap & _dm_stripes, , n_samples_r(((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK) , task_p(_task_p) { - + // The buffer is only needed if the stripes are non-empty buf = (dm_stripes.is_empty(start_idx)) ? std::vector() : std::vector(n_samples_r* @@ -55,8 +55,6 @@ UnifracTaskVector::~UnifracTaskVector(){ } } - - template UnifracTaskBase::UnifracTaskBase(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, @@ -102,13 +100,11 @@ UnifracTask::UnifracTask(su::StripeMap & _dm_stripes, template UnifracTask::~UnifracTask(){} - - UnifracUnweightedTask::UnifracUnweightedTask(su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) + : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) { const unsigned int bsize = _max_embs*32; sums = std::vector(bsize, 0.0); @@ -127,27 +123,27 @@ void UnifracUnweightedTask::_run(unsigned int filled_embs, const uint64_t stop_idx = this->task_p.stop; const uint64_t n_samples = this->task_p.n_samples; const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - + const uint64_t step_size = UnifracUnweightedTask::step_size; const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; - + const uint64_t filled_embs_els = filled_embs/64; - const uint64_t filled_embs_rem = filled_embs%64; - + const uint64_t filled_embs_rem = filled_embs%64; + const uint64_t filled_embs_els_round = (filled_embs+63)/64; - + // pre-compute sums of length elements, since they are likely to be accessed // many times // We will use a 8-bit map, to keep it small enough to keep in L1 cache for(uint64_t emb_el=0; emb_el pl = std::vector(8); - - + + uint64_t len_off = emb8*8; - + // compute all the combinations for this block (8-bits total) // psum[0] = 0.0 // +0*pl[0]+0*pl[1]+0*pl[2]+... // psum[1] = pl[0] // +0*pl[1]+0*pl[2]+... @@ -169,13 +165,13 @@ void UnifracUnweightedTask::_run(unsigned int filled_embs, } } } - + if (filled_embs_rem>0){ // add also the overflow elements const uint64_t emb_el=filled_embs_els; for(uint64_t sub8=0; sub8<8; sub8++){ // we are summing we have enough buffer in sums const uint64_t emb8 = emb_el*8+sub8; - + // compute all the combinations for this block, set to 0 any past // the limit as above for(uint64_t b8_i=0; b8_i<0x100; b8_i++){ @@ -185,50 +181,50 @@ void UnifracUnweightedTask::_run(unsigned int filled_embs, } sums[(emb8<<8) + b8_i] = val; } - + } } - + // point of thread for(uint64_t sk = 0; sk < sample_steps ; sk++){ for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++){ for(uint64_t ik = 0; ik < step_size ; ik++){ // within-stripe index (0:n_samples-1) - const uint64_t k = sk*step_size + ik; + const uint64_t k = sk*step_size + ik; //buffer index - const uint64_t idx = (stripe-start_idx) * n_samples_r; - + const uint64_t idx = (stripe-start_idx) * n_samples_r; + if (k>=n_samples) continue; // past the limit - + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - + bool did_update = false; double my_stripe = 0.0; double my_stripe_total = 0.0; - + //Main calculation phase for(uint64_t emb_el=0; emb_el> 8) & 0xff)] + sums[sums_off + 0x200+((x1 >> 16) & 0xff)] + sums[sums_off + 0x300+((x1 >> 24) & 0xff)] + @@ -246,7 +242,7 @@ void UnifracUnweightedTask::_run(unsigned int filled_embs, sums[sums_off + 0x700+((o1 >> 56) )]; } } - + if (did_update){ dm_stripes.buf[idx + k] += my_stripe; dm_stripes_total.buf[idx + k] += my_stripe_total; @@ -256,15 +252,12 @@ void UnifracUnweightedTask::_run(unsigned int filled_embs, } } - - - UnifracUnnormalizedWeightedTask::UnifracUnnormalizedWeightedTask( su::StripeMap & _dm_stripes, su::StripeMap & _dm_stripes_total, unsigned int _max_embs, su::task_parameters _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) + : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) { const unsigned int n_samples = this->task_p.n_samples; zcheck = std::vector(n_samples, 0); @@ -284,59 +277,59 @@ void UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, const uint64_t stop_idx = this->task_p.stop; const uint64_t n_samples = this->task_p.n_samples; const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - + const uint64_t step_size = UnifracUnnormalizedWeightedTask::step_size; const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; - + // check for zero values and pre-compute single column sums for(uint64_t k=0; k=n_samples) continue; // past the limit - + const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - + const bool allzero_k = zcheck[k]; const bool allzero_l1 = zcheck[l1]; - + if (allzero_k && allzero_l1) { // nothing to do, would have to add 0 } else { double my_stripe; - + if (allzero_k || allzero_l1){ // one side has all zeros // we can use the distributed property, and use the // pre-computed values - + const uint64_t ridx = (allzero_k) ? l1 : k; // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 - + my_stripe = sums[ridx]; - + } else { // both sides non zero, use the explicit but slow @@ -350,7 +343,7 @@ void UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, double length = lengths[emb]; my_stripe += fabs(diff1) * length; } // for emb - + } const uint64_t idx = (stripe-start_idx)*n_samples_r; dm_stripes.buf[idx + k] += my_stripe; @@ -359,5 +352,3 @@ void UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, } // for stripe } // for sk } - - diff --git a/tests/testthat/test-5Unifrac.R b/tests/testthat/test-5Unifrac.R index 74006b6a4..7ea9b7ba0 100644 --- a/tests/testthat/test-5Unifrac.R +++ b/tests/testthat/test-5Unifrac.R @@ -65,6 +65,8 @@ test_that("Unifrac beta diversity", { weighted = FALSE, tree.name = "tree2") ) unifrac_mia <- as.matrix(unifrac_mia) + + skip_if_not(requireNamespace("ecodive", quietly = TRUE)) unifrac_ecodive <- as.matrix(ecodive::unweighted_unifrac(t(assay(tse_ref)), rowTree(tse_ref))) expect_equal(unifrac_mia, unifrac_ecodive) From e7d5bac292edb61c82cb51b43accc02711ed33dd Mon Sep 17 00:00:00 2001 From: Tuomas Borman Date: Fri, 12 Jun 2026 15:03:12 +0300 Subject: [PATCH 48/48] up --- DESCRIPTION | 2 +- NAMESPACE | 2 -- NEWS | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 50eae803e..2fcc714b0 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: mia Type: Package -Version: 1.21.2 +Version: 1.21.3 Authors@R: c(person(given = "Tuomas", family = "Borman", role = c("aut", "cre"), email = "tuomas.v.borman@utu.fi", diff --git a/NAMESPACE b/NAMESPACE index b0f050187..44e47052a 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -430,8 +430,6 @@ importFrom(dplyr,select) importFrom(dplyr,summarise) importFrom(dplyr,sym) importFrom(dplyr,tally) -importFrom(ecodive,unweighted_unifrac) -importFrom(ecodive,weighted_unifrac) importFrom(rlang,":=") importFrom(rlang,sym) importFrom(scater,calculateMDS) diff --git a/NEWS b/NEWS index 96e44c6cd..314fa77a3 100644 --- a/NEWS +++ b/NEWS @@ -191,4 +191,4 @@ Changes in version 1.21.x + Fixed explained percentages on PCA axes for jointRPCA and hardened rank-deficient SVD handling in joint-RPCA (1.21.1, 2026-05-12) * Generalise agglomerateByModule to non-binary numerical modules (1.21.2, 2026-05-31) -+ Implement Striped Unifrac algorithm (1.21.2, 2026-06-12) ++ Implement Striped Unifrac algorithm (1.21.3, 2026-06-12)