Fix(indicators): Fix Forest Inventory Indicators

backend/configs/config.json

@@ -49,37 +49,43 @@
       "columns": {
         "id": "_id",
         "geom": "gps.merge().centroid()",
-        "for": "loc2",
+        "for": "int(loc2)",
         "cod": "cod",
-        "ecos": "ecos",
-        "taille_placette": "20^2*pi()/10000",
+        "isPublic": "int(conf)",
+        "ecos": "int(ecos)",
+        "plot_size": "20^2*pi()/10000",
         "project": "proj",
         "tree_pop": "count(1 if adu.cond = 1)",
         "biomass_volume": "adu.sum(0.0673 * (for_sp.dens * (meas1 + meas2 + meas3 + meas4 + meas5 + meas6 + meas7 + meas8 + meas9 + meas10)^2 * hgt_perp)^0.976 if cond = 1) / (20^2 * pi() / 10000) / 1000",
         "tree_density": "count(1 if adu.cond = 1) / (20^2 * pi() / 10000)",
         "richness": "count(unique(adu.adu_sp))",
         "relative_abundance": "histogram(adu.adu_sp)",
+        "deadwoord": "adu.count(1 if cond=2 or cond=3)",
         "bio_idx_tree_density": "count(1 if adu.cond = 1) / (20^2 * pi() / 10000) / 100",
-        "bio_idx_deadWood": "((adu.sum(0.0673 * (for_dw.dens * (meas1 + meas2 + meas3 + meas4 + meas5 + meas6 + meas7 + meas8 + meas9 + meas10)^2 * hgt_perp)^0.976 if cond = 2) + adu.sum(pi() / 3 * ((meas_b / 2)^2 + ((meas_e / 2)^2 + meas_b / 2 + meas_e / 2) * lght * for_dw.dens) if cond = 3)) / adu.sum(0.0673 * (for_sp.dens * (meas1 + meas2 + meas3 + meas4 + meas5 + meas6 + meas7 + meas8 + meas9 + meas10)^2 * hgt_perp)^0.976 if cond = 1)) * 10",
+        "bio_idx_deadWood": "((adu.sum(0.0673 * (for_dw.dens * (meas1 + meas2 + meas3 + meas4 + meas5 + meas6 + meas7 + meas8 + meas9 + meas10)^2 * hgt_perp)^0.976 if cond = 2) + adu.sum(pi() / 3 * ((meas_b / 2)^2 + ((meas_e / 2)^2 + meas_b / 2 + meas_e / 2) * lght * for_dw.dens) if cond = 3)) / adu.sum(0.0673 * (for_sp.dens * (meas1 + meas2 + meas3 + meas4 + meas5 + meas6 + meas7 + meas8 + meas9 + meas10)^2 * hgt_perp)^0.976 if cond = 1)) / 30 * 10",
         "bio_idx_tree_diversity": "shannon(adu.adu_sp) / 5 * 10",
-        "bio_idx_spatial_distribution": "adu.categorical_gini(area) * 10",
+        "test_gini": "adu.gini(int(area) if cond = 1) * 10",
+        "bio_idx_spatial_distribution": "gini_arr(list_concat(list(adu.count(1 if cond=1 and area=1)), list(adu.count(1 if cond=1 and area=2)), list(adu.count(1 if cond=1 and area=3)), list(adu.count(1 if cond=1 and area=4)))) * 10",
         "bio_idx_diametric_distribution": "adu.gini(meas1 + meas2 + meas3 + meas4 + meas5 + meas6 + meas7 + meas8 + meas9 + meas10) * 10",
         "bio_idx_vertical_distribution": "gini(adu.hgt_perp) * 10",
-        "bio_idx_dominant_height": "adu.avg(hgt_perp if hgt_perp > percentile(hgt_perp, 80)) * 10 / 40",
-        "bio_idx_microhabitats": "adu.list(trem).flatten().list_unique() * 10 / trem_group_max",
-        "soil_structure_idx": "(thk1*float(not1) + thk2*float(not2) + thk3*float(not3) + thk4*float(not4) + thk5*float(not5)) / (thk1 + thk2 + thk3 + thk4 + thk5) * 2",
-        "soil_composition": "count(1)",
+        "bio_idx_dominant_height": "adu.avg(hgt_perp if hgt_perp >= percentile(hgt_perp, 80)) * 10 / 40",
+        "bio_idx_microhabitats": "adu.list(trem).flatten().filter( x -> x != 'None').list_unique() * 10 / trem_group_max",
+        "soil_structure_idx": "(thk1*float(not1) + thk2*float(not2) + thk3*float(not3) + thk4*float(not4) + thk5*float(not5)) / (thk1 + thk2 + thk3 + thk4 + thk5)",
         "soil_eros_rainfall_and_wind": "for_weath.concat_ws('-', rain, wind)",
         "soil_eros_slope": "int(10 - slop if slop < 10 else 0)",
         "soil_eros_cover": "veg",
-        "soil_eros_stability": "(int(slak1) + int(slak2) + int(slak3)) / 3",
-        "soil_eros_water_infiltration": "600 * 300 / ( (epoch(interval(beer1)) + epoch(interval(beer2)) + epoch(interval(beer3)) + epoch(interval(beer4)) + epoch(interval(beer5)) + epoch(interval(beer6)) + epoch(interval(beer7)) + epoch(interval(beer8)) + epoch(interval(beer9)) + epoch(interval(beer10)) ) / 10 * pi() * 9^2) if beer1 != '' else 0",
+        "soil_eros_stability": "((int(slak1) + int(slak2) + int(slak3) ) / 3 if slak1 != '' )",
+        "soil_eros_water_infiltration": "600 * 300 / ( (epoch(interval(beer1)) + ( epoch(interval(beer2)) if beer2!='' ) + ( epoch(interval(beer3)) if beer3!='' ) + ( epoch(interval(beer4)) if beer4!='' ) + ( epoch(interval(beer5)) if beer5!='' ) + ( epoch(interval(beer6)) if beer6!='' ) + ( epoch(interval(beer7)) if beer7!='' ) + ( epoch(interval(beer8)) if beer8!='' ) + ( epoch(interval(beer9)) if beer9!='' ) + ( epoch(interval(beer10)) if beer10!='' ) ) / non_null_count(beer1, beer2, beer3, beer4, beer5, beer6, beer7, beer8, beer9, beer10) * pi() * 9^2) if beer1 != ''",
         "soil_fauna_density": "sum(tsbf_001.tsbf_no) / tsbf_samp",
-        "soil_fauna_diversity": "list_unique(list(nullif(tsbf_001.concat_ws('-', tsbf_tax1, tsbf_tax2, tsbf_tax3), '')))",
-        "soil_fauna_abundance": "list_aggregate(list(tsbf_001.concat_ws('-', tsbf_tax1, tsbf_tax2, tsbf_tax3)), 'histogram')",
+        "soil_fauna_diversity": "count(tsbf_001.tsbf_tax1)",
+        "soil_fauna_abundance": "list_aggregate(list(nullif(tsbf_001.concat_ws('-', tsbf_tax1, tsbf_tax2, tsbf_tax3), '')), 'histogram')",
+        "soil_fauna_abundance_pop": "list_concat(list(tsbf_001.concat_ws(':', concat_ws('-',tsbf_tax1, tsbf_tax2, tsbf_tax3), tsbf_no) if tsbf_001.tsbf_no > 0))",
+        "soil_fauna_total_pop": "int(sum(tsbf_001.tsbf_no))",
         "soil_surface_fauna_density": "(sum(barba_001.barbA_no) + sum(barbb_001.barbB_no) + sum(barbc_001.barbC_no) + sum(barbd_001.barbD_no)) / barb_samp",
-        "soil_surface_fauna_diversity": "list_unique(list_concat(list(nullif(barba_001.concat_ws('-', barbA_tax1, barbA_tax2, barbA_tax3), '')), list(nullif(barbb_001.concat_ws('-', barbB_tax1, barbB_tax2, barbB_tax3), '')), list(nullif(barbc_001.concat_ws('-', barbC_tax1, barbC_tax2, barbC_tax3), '')), list(nullif(barbd_001.concat_ws('-', barbD_tax1, barbD_tax2, barbD_tax3), ''))))",
-        "soil_surface_fauna_abundance": "list_aggregate(list_concat(list(barba_001.concat_ws('-', barbA_tax1, barbA_tax2, barbA_tax3)), list(barbb_001.concat_ws('-', barbB_tax1, barbB_tax2, barbB_tax3)), list(barbc_001.concat_ws('-', barbC_tax1, barbC_tax2, barbC_tax3)), list(barbd_001.concat_ws('-', barbD_tax1, barbD_tax2, barbD_tax3))), 'histogram')"
+        "soil_surface_fauna_diversity": "(count(barba_001.barbA_no) + count(barbb_001.barbB_no) + count(barbc_001.barbC_no) + count(barbd_001.barbD_no) ) / 4",
+        "soil_surface_fauna_abundance": "list_aggregate(list_concat(list(nullif(barba_001.concat_ws('-', barbA_tax1, barbA_tax2, barbA_tax3), '')), list(nullif(barbb_001.concat_ws('-', barbB_tax1, barbB_tax2, barbB_tax3), '')), list(nullif(barbc_001.concat_ws('-', barbC_tax1, barbC_tax2, barbC_tax3), '')), list(nullif(barbd_001.concat_ws('-', barbD_tax1, barbD_tax2, barbD_tax3), ''))), 'histogram')",
+        "soil_surface_fauna_abundance_pop": "list_concat(list(barba_001.concat_ws(':', concat_ws('-',barbA_tax1, barbA_tax2, barbA_tax3), barbA_no) if barba_001.barbA_no > 0), list(barbb_001.concat_ws(':', concat_ws('-',barbB_tax1, barbB_tax2, barbB_tax3), barbB_no) if barbb_001.barbB_no > 0), list(barbc_001.concat_ws(':', concat_ws('-',barbC_tax1, barbC_tax2, barbC_tax3), barbC_no) if barbc_001.barbC_no > 0), list(barbd_001.concat_ws(':', concat_ws('-',barbD_tax1, barbD_tax2, barbD_tax3), barbD_no) if barbd_001.barbD_no > 0))",
+        "soil_surface_fauna_total_pop": "int(sum(barba_001.barbA_no) + sum(barbb_001.barbB_no) + sum(barbc_001.barbC_no) + sum(barbd_001.barbD_no))"
       },
       "popup": {
         "trigger": "click"
@@ -104,7 +110,7 @@
         "population": "sum(res)",
         "fuel_sources": "list_aggregate(flatten(list(ener)), 'histogram')",
         "wood_fuel_cons": "sum( 12 * ( ifnull(fw_qty * enq_fw.fw_vol * fw_freq, 0) + ifnull(coal_qty * enq_coal.coal_kg * coal_freq * enq_coal.coal_fw, 0) / enq_coal.coal_dens )) / sum(res)",
-        "firewood_collec_time": "ifnull(avg(coll_freq * coll_dur)/30, 0)",
+        "firewood_collec_time": "avg(ifnull(coll_freq, 0) * ifnull(coll_dur, 0) / 30)",
         "firewood_satis": "histogram(wf_satis if int(wf_satis) > 0)",
         "firewood_satis_tot": "count(1 if int(wf_satis) > 0)",
         "firewood_satis1": "count(1 if wf_satis = '1') / count(1 if int(wf_satis) > 0) * 100",
@@ -122,16 +128,13 @@
         "lean_period": "avg(len(lean) if lean_yn = 1)",
         "food_diversity": "list_aggregate(flatten(list(food_cons)), 'histogram')",
         "food_diversity_score": "avg(list_unique(food_cons) / int(food_cons_max) * 10)",
-        "food_self_suff_score": "avg(list_unique(food_self) / list_unique(food_cons) * 100)",
+        "food_self_suff_score": "avg(ifnull(list_unique(food_self), 0) / list_unique(food_cons) * 100)",
         "living_cond_perception": "histogram(cond_evol)",
         "revenue_change": "avg(0 - inc_dif if inc_evol = '2' else ifnull(inc_dif, 0))",
-        "assets_idx": "10 / int(ass_tot_max) * avg( ( ( int(hous_typ) + int(hous_size)) * int(hous_own) + list_unique(ass) + int(lvst_yn) * ( int(lvst_s) + int(lvst_l) ) + int( land_own ) + int( land_titl ) + int( land_size ) ) if ( ( int(hous_typ) + int(hous_size)) * int(hous_own) + list_unique(ass) + int(lvst_yn) * ( int(lvst_s) + int(lvst_l) ) + int( land_own ) + int( land_titl ) + int( land_size ) ) <= int(ass_tot_max) )",
+        "assets_idx_raw": "avg( ( ( int(hous_typ) + int(hous_size)) * int(hous_own) + ifnull(list_unique(ass),0) + int(lvst_yn) * ( int(lvst_s) + int(lvst_l) ) + ( int( land_own ) + int( land_titl ) + int( land_size ) if int(land_own) > 0 else 0) ) if ( ( int(hous_typ) + int(hous_size)) * int(hous_own) + list_unique(ass) + int(lvst_yn) * ( int(lvst_s) + int(lvst_l) ) + int( land_own ) + int( land_titl ) + int( land_size ) ) <= int(ass_tot_max) )",
+        "assets_idx": "10 / int(ass_tot_max) * avg( ( ( int(hous_typ) + int(hous_size)) * int(hous_own) + ifnull(list_unique(ass),0) + int(lvst_yn) * ( int(lvst_s) + int(lvst_l) ) + ( int( land_own ) + int( land_titl ) + int( land_size ) if int(land_own) > 0 else 0) ) if ( ( int(hous_typ) + int(hous_size)) * int(hous_own) + list_unique(ass) + int(lvst_yn) * ( int(lvst_s) + int(lvst_l) ) + int( land_own ) + int( land_titl ) + int( land_size ) ) <= int(ass_tot_max) )",
         "nb_income_sources" : "avg(int(inc_no))",
         "nb_additional_incomes": "'N/A'",
-        "sector_benef": "'N/A'",
-        "sector_support_amount": "'N/A'",
-        "nb_improved_skills": "'N/A'",
-        "stak_typ": "'N/A'",
         "benef_pract1": "count(1 if int(pract1) = 2) / count(1) * 100",
         "benef_pract2": "count(1 if int(pract2) = 2) / count(1) * 100",
         "benef_pract3": "count(1 if int(pract3) = 2) / count(1) * 100",

backend/requirements.txt

@@ -1,6 +1,6 @@
 asgiref==3.8.1
 chardet==7.4.3
-coordo @ git+https://github.com/dataforgoodfr/Coordonnees.git@0.6.2#subdirectory=coordo-py
+coordo @ git+https://github.com/dataforgoodfr/Coordonnees.git@test-0.7.1#subdirectory=coordo-py
 Django>=4.2.27
 djangorestframework==3.16.0
 djangorestframework_simplejwt==5.5.1

webapp/package.json

@@ -40,6 +40,7 @@
     "chart.js": "^4.5.1",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
+    "color2k": "^2.0.4",
     "coordo": "github:dataforgoodfr/Coordonnees#0.6.0",
     "i18next": "^25.8.0",
     "i18next-browser-languagedetector": "^8.2.0",

webapp/src/app/styles/all4trees.css

@@ -97,10 +97,10 @@
   --ring: var(--a4t-color-vert-kelly);
 
   --chart-1: var(--a4t-color-vert-kelly);
-  --chart-2: var(--a4t-color-citrouille);
-  --chart-3: var(--a4t-color-bleu);
-  --chart-4: #895bf5;
-  --chart-5: #f04646;
+  --chart-2: var(--a4t-color-bleu);
+  --chart-3: var(--a4t-color-citrouille);
+  --chart-4: var(--a4t-color-vert-de-gris);
+  --chart-5: #895bf5;
   --chart-6: var(--a4t-color-onyx);
 
   --sidebar: var(--a4t-color-alabaster);

webapp/src/features/charts/biodiversity/chart-relative-abundance.tsx

@@ -2,15 +2,14 @@ import { findLabelInExternalData } from "@features/indicators/utils";
 import type { ExternalData } from "@features/popup/forest-inventory/types";
 
 import { useTranslation } from "@shared/i18n";
-import { precise } from "@shared/lib/utils";
 import type { ChartConfig } from "@shared/ui/chart";
 
 import type { ChartComponentType } from "../components/chart-component";
 import { PieChartCategorical } from "../components/pie-chart-categorical";
 
 type PieChartProps = {
   project: string;
-  data: [string, number][];
+  data: Record<string, number>;
   externalData: ExternalData;
 };
 
@@ -20,19 +19,18 @@ export const ChartRelativeAbundance: ChartComponentType<PieChartProps> = ({
   externalData,
 }) => {
   const { t } = useTranslation("all4trees");
-  const smallCategoriesSum = Number(
-    precise(
-      data
-        .filter(([_, value]) => value < 5)
-        .reduce((acc, [_, value]) => acc + value, 0),
-    ),
-  );
-  const chartData = data
-    .filter(([name, value]) => name !== "0" && (data.length < 6 || value >= 5))
-    .map((element, index) => ({
+  const smallCategoriesSum = Object.values(data)
+    .filter((value) => value < 5)
+    .reduce((acc, value) => acc + value, 0);
+  const chartData = Object.entries(data)
+    .filter(
+      ([name, value]) =>
+        name !== "0" && (Object.keys(data).length < 6 || value >= 5),
+    )
+    .map(([name, value], index) => ({
       fill: `var(--chart-${(index % 5) + 1})`,
-      name: element[0],
-      value: element[1],
+      name,
+      value,
     }));
 
   let chartConfig: ChartConfig = {};

webapp/src/features/charts/soil/lib/sunburst.ts

@@ -1,3 +1,5 @@
+import { darken, lighten } from "color2k";
+
 import type { ExternalData } from "@features/popup/forest-inventory/types";
 
 import { getChartPalette } from "@shared/lib/palette";
@@ -10,59 +12,51 @@ export function buildSunburstNodes(
   metadata: ExternalData,
   project: string,
 ): SunburstNode[] {
-  const nodes: SunburstNode[] = [];
-  const seen = new Set<string>();
-
+  const nodes = new Map<string, SunburstNode>();
   for (const [key, value] of dataEntries) {
     const parts = key.split("-");
-    nodes.push({
+    nodes.set(key, {
+      depth: parts.length - 1,
       id: key,
       label: formatTaxonLevelLabel(key, metadata, project),
       parent: parts.slice(0, -1).join("-") || "",
       value,
     });
-    seen.add(key);
 
-    for (let i = 0; i < parts.length - 1; i++) {
-      const parentPath = parts.slice(0, i + 1).join("-");
-      if (!seen.has(parentPath)) {
-        nodes.push({
+    for (let i = 1; i < parts.length; i++) {
+      const parentPath = parts.slice(0, i).join("-");
+      const parentNode = nodes.get(parentPath);
+      if (parentNode) {
+        parentNode.value += value;
+        nodes.set(parentPath, parentNode);
+      } else {
+        nodes.set(parentPath, {
+          depth: parentPath.split("-").length - 1,
           id: parentPath,
           label: formatTaxonLevelLabel(parentPath, metadata, project),
-          parent: parts.slice(0, i).join("-") || "",
-          value: 0,
+          parent: parts.slice(0, i - 1).join("-") || "",
+          value: value,
         });
-        seen.add(parentPath);
       }
     }
   }
 
-  const nodeMap = new Map(nodes.map((n) => [n.id, n]));
-  nodes.forEach((node) => {
-    if (node.parent) {
-      const parent = nodeMap.get(node.parent);
-      if (parent) parent.value += node.value;
-    }
-  });
-
-  return nodes;
+  return Array.from(nodes.values());
 }
 
 export const getLevelPalettes = () => {
   const palette = getChartPalette();
-
-  return [
-    palette.slice(0, 3),
-    [palette[3], palette[4], palette[0]],
-    [palette[1], palette[2], palette[3]],
-  ];
+  const lvl1Palette = palette.map((color) => darken(color, 0.1));
+  const lvl3Palette = palette.map((color) => lighten(color, 0.1));
+  return [lvl1Palette, palette, lvl3Palette];
 };
 
 export function buildNodeColors(nodes: SunburstNode[]) {
   const palettes = getLevelPalettes();
-  return nodes.map((node, index) => {
-    const depth = node.id.split("-").length;
-    const palette = palettes[depth - 1] ?? palettes[0];
-    return palette[index % palette.length];
-  });
+  return nodes
+    .sort((a, b) => a.depth - b.depth)
+    .map((node, index) => {
+      const palette = palettes[node.depth];
+      return palette[index % palette.length];
+    });
 }

webapp/src/features/charts/soil/types.ts

@@ -11,17 +11,20 @@ export type SunburstNode = {
   label: string;
   parent: string;
   value: number;
+  depth: number;
 };
 
 export type SunburstTrace = {
   type: "sunburst";
   branchvalues: "total";
   extendsunburstcolorway: boolean;
+  hoverinfo?: string;
+  hovertext?: string[];
   ids: string[];
   labels: string[];
   parents: string[];
   values: number[];
-  marker: { colors: string[]; line: { width: number } };
+  marker: { colors: string[]; line: { width: number; color: string } };
   leaf: { opacity: number };
   sunburstcolorway: string[];
 };

webapp/src/features/charts/soil/ui/chart-taxon-abundance.tsx

@@ -32,15 +32,20 @@ export const ChartTaxonAbundance: ChartComponentType<PieChartProps> = ({
     );
     const nodes = buildSunburstNodes(filteredDataEntries, metadata, project);
     const nodeColors = buildNodeColors(nodes);
+    const hoverText = nodes.map(
+      (node) => `${node.label}<br>${node.value.toFixed(2)} %`,
+    );
 
     sunburstData = [
       {
         branchvalues: "total",
         extendsunburstcolorway: true,
+        hoverinfo: "text",
+        hovertext: hoverText,
         ids: nodes.map((n) => n.id),
         labels: nodes.map((n) => n.label),
-        leaf: { opacity: 0.4 },
-        marker: { colors: nodeColors, line: { width: 2 } },
+        leaf: { opacity: 1 },
+        marker: { colors: nodeColors, line: { color: "#fff", width: 2 } },
         parents: nodes.map((n) => n.parent),
         sunburstcolorway: nodeColors,
         type: "sunburst",

webapp/src/features/indicators/biodiversity/format-data.ts

@@ -1,12 +1,13 @@
 import { useTranslation } from "react-i18next";
 
 import {
+  convertDictToPercentage,
   preciseNumericIndicators,
   UNITS,
   useFormatterWithUnit,
 } from "@features/indicators/utils";
+import type { ForestInventoryData } from "@features/popup/forest-inventory/types";
 
-import { precise } from "@shared/lib/utils";
 import type { NumericKeys } from "@shared/types";
 
 import type { BiodiversityData } from "./types";
@@ -25,22 +26,10 @@ const indicatorKeys: NumericKeys<BiodiversityData>[] = [
   "bio_idx_microhabitats",
 ];
 
-const formatRelativeAbundance = (
-  relativeAbundance: BiodiversityData["relative_abundance"],
-  treePop: number,
-) =>
-  Object.entries(relativeAbundance).map(
-    ([key, value]) =>
-      [key, Number(precise((Number(value) * 100) / treePop))] as [
-        string,
-        number,
-      ],
-  );
-
 /**
  * Return data in a convenient way for UI rendering, handling units and fixing
  */
-export const useFormatBiodiversityData = (data: BiodiversityData) => {
+export const useFormatBiodiversityData = (data: ForestInventoryData) => {
   const { t } = useTranslation("common");
   const { formatWithUnit } = useFormatterWithUnit();
 
@@ -59,9 +48,10 @@ export const useFormatBiodiversityData = (data: BiodiversityData) => {
       volume: formatWithUnit(safeData.biomass_volume, UNITS.tonPerHectare),
     },
     treeDiversity: {
-      relative_abundance: formatRelativeAbundance(
+      relative_abundance: convertDictToPercentage(
         data.relative_abundance,
         data.tree_pop,
+        "0",
       ),
       speciesRichness: formatWithUnit(safeData.richness, UNITS.essenceCount),
       tree_pop: data.tree_pop,