From f536caa5e3aa94f69d179e6de73ce2d33ba621be Mon Sep 17 00:00:00 2001 From: varisd Date: Sat, 4 Jul 2026 20:44:58 +0200 Subject: [PATCH 1/8] webapp: Metric Pair Correlation graph scaling and value transforms --- frontend/package-lock.json | 249 +++----------- frontend/package.json | 7 +- frontend/src/components/GraphConfigurator.tsx | 40 +++ .../src/components/graphs/ScatterGraph.tsx | 307 +++++++++++------- frontend/src/types.ts | 10 + 5 files changed, 301 insertions(+), 312 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e2031c5..3289e61 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,14 +8,15 @@ "name": "tokcollate-frontend", "version": "0.1.0", "dependencies": { - "axios": "^1.6.0", + "chart.js": "^4.5.1", + "chartjs-plugin-zoom": "^2.2.0", "cors": "^2.8.5", "express": "^4.18.2", + "hammerjs": "^2.0.8", "html2canvas": "^1.4.1", - "jszip": "^3.10.1", "multer": "^1.4.5-lts.1", - "npyjs": "^1.0.4", "react": "^18.2.0", + "react-chartjs-2": "^5.3.1", "react-dom": "^18.2.0", "recharts": "^2.10.0" }, @@ -713,6 +714,11 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@kurkle/color": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.4.tgz", + "integrity": "sha512-M5UknZPHRu3DEDWoipU6sE8PdkZ6Z/S+v4dD+Ke8IaNlpdSQah50lz1KtcFBa2vsdOnwbbnxJwVM4wty6udA5w==" + }, "node_modules/@rolldown/pluginutils": { "version": "1.0.0-beta.27", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", @@ -1145,6 +1151,11 @@ "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", "dev": true }, + "node_modules/@types/hammerjs": { + "version": "2.0.46", + "resolved": "https://registry.npmjs.org/@types/hammerjs/-/hammerjs-2.0.46.tgz", + "integrity": "sha512-ynRvcq6wvqexJ9brDMS4BnBLzmr0e14d6ZJTEShTBWKymQiHwlAyGu0ZPEFI2Fh1U53F7tN9ufClWM5KvqkKOw==" + }, "node_modules/@types/node": { "version": "20.19.27", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.27.tgz", @@ -1336,21 +1347,6 @@ "node": ">=12" } }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" - }, - "node_modules/axios": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.2.tgz", - "integrity": "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA==", - "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.4", - "proxy-from-env": "^1.1.0" - } - }, "node_modules/base64-arraybuffer": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz", @@ -1533,6 +1529,29 @@ "node": ">=18" } }, + "node_modules/chart.js": { + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", + "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", + "dependencies": { + "@kurkle/color": "^0.3.0" + }, + "engines": { + "pnpm": ">=8" + } + }, + "node_modules/chartjs-plugin-zoom": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/chartjs-plugin-zoom/-/chartjs-plugin-zoom-2.2.0.tgz", + "integrity": "sha512-in6kcdiTlP6npIVLMd4zXZ08PDUXC52gZ4FAy5oyjk1zX3gKarXMAof7B9eFiisf9WOC3bh2saHg+J5WtLXZeA==", + "dependencies": { + "@types/hammerjs": "^2.0.45", + "hammerjs": "^2.0.8" + }, + "peerDependencies": { + "chart.js": ">=3.2.0" + } + }, "node_modules/check-error": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", @@ -1550,17 +1569,6 @@ "node": ">=6" } }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/concat-stream": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", @@ -1630,14 +1638,6 @@ "node": ">= 0.10" } }, - "node_modules/cross-fetch": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.1.0.tgz", - "integrity": "sha512-uKm5PU+MHTootlWEY+mZ4vvXoCn4fLQxT9dSc1sXVMSFkINTJVN8cAQROpwcKm8bJ/c7rgZVIBWzH5T78sNZZw==", - "dependencies": { - "node-fetch": "^2.7.0" - } - }, "node_modules/css-line-break": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/css-line-break/-/css-line-break-2.1.0.tgz", @@ -1792,14 +1792,6 @@ "node": ">=6" } }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -1902,20 +1894,6 @@ "node": ">= 0.4" } }, - "node_modules/es-set-tostringtag": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", - "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "dependencies": { - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.6", - "has-tostringtag": "^1.0.2", - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/esbuild": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", @@ -2095,40 +2073,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, - "node_modules/follow-redirects": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", - "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, - "node_modules/form-data": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", - "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -2222,6 +2166,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hammerjs": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/hammerjs/-/hammerjs-2.0.8.tgz", + "integrity": "sha512-tSQXBXS/MWQOn/RKckawJ61vvsDpCom87JgxiYdGwHdOa0ht0vzUWDlfioofFCRU0L+6NGDt6XzbgoJvZkMeRQ==", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -2233,20 +2185,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/has-tostringtag": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", - "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dependencies": { - "has-symbols": "^1.0.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -2300,11 +2238,6 @@ "node": ">=0.10.0" } }, - "node_modules/immediate": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==" - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -2360,25 +2293,6 @@ "node": ">=6" } }, - "node_modules/jszip": { - "version": "3.10.1", - "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", - "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", - "dependencies": { - "lie": "~3.3.0", - "pako": "~1.0.2", - "readable-stream": "~2.3.6", - "setimmediate": "^1.0.5" - } - }, - "node_modules/lie": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", - "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", - "dependencies": { - "immediate": "~3.0.5" - } - }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", @@ -2811,42 +2725,12 @@ "node": ">= 0.6" } }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/node-releases": { "version": "2.0.27", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", "dev": true }, - "node_modules/npyjs": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/npyjs/-/npyjs-1.0.4.tgz", - "integrity": "sha512-+hcBtLieQ8E+MdCb7dPzqC8Co1FCn1RBHbTA9zO6JYIYfY1ai0NzIyOrplnCOtGKU06WURiqqKFC/IjiQrqX/g==", - "dependencies": { - "cross-fetch": "^4.1.0" - }, - "engines": { - "node": ">=18" - } - }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -2877,11 +2761,6 @@ "node": ">= 0.8" } }, - "node_modules/pako": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" - }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -2976,11 +2855,6 @@ "node": ">= 0.10" } }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" - }, "node_modules/qs": { "version": "6.14.1", "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", @@ -3028,6 +2902,15 @@ "node": ">=0.10.0" } }, + "node_modules/react-chartjs-2": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/react-chartjs-2/-/react-chartjs-2-5.3.1.tgz", + "integrity": "sha512-h5IPXKg9EXpjoBzUfyWJvllMjG2mQ4EiuHQFhms/AjUm0XSZHhyRy2xVmLXHKrtcdrPO4mnGqRtYoD0vp95A0A==", + "peerDependencies": { + "chart.js": "^4.1.1", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/react-dom": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", @@ -3267,11 +3150,6 @@ "node": ">= 0.8.0" } }, - "node_modules/setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" - }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -3461,11 +3339,6 @@ "node": ">=0.6" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -3736,20 +3609,6 @@ } } }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/why-is-node-running": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index a31e54b..948d8bc 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -3,14 +3,15 @@ "version": "0.1.0", "private": true, "dependencies": { - "axios": "^1.6.0", + "chart.js": "^4.5.1", + "chartjs-plugin-zoom": "^2.2.0", "cors": "^2.8.5", "express": "^4.18.2", + "hammerjs": "^2.0.8", "html2canvas": "^1.4.1", - "jszip": "^3.10.1", "multer": "^1.4.5-lts.1", - "npyjs": "^1.0.4", "react": "^18.2.0", + "react-chartjs-2": "^5.3.1", "react-dom": "^18.2.0", "recharts": "^2.10.0" }, diff --git a/frontend/src/components/GraphConfigurator.tsx b/frontend/src/components/GraphConfigurator.tsx index 58e8991..ba64a5f 100644 --- a/frontend/src/components/GraphConfigurator.tsx +++ b/frontend/src/components/GraphConfigurator.tsx @@ -361,6 +361,7 @@ const GraphConfigurator: React.FC = ({ // Keep boolean flag in sync for any legacy consumers showTrendline: trendlineMode !== 'none', sentenceRange: cfg.sentenceRange, + axisTransforms: (cfg as any).axisTransforms, }; // Always propagate the current configuration to the active figure. @@ -567,6 +568,45 @@ const GraphConfigurator: React.FC = ({ + {/* Axis transform controls */} + {(['x', 'y'] as const).map((axis) => { + const metricLabel = axis === 'x' ? 'X-axis' : 'Y-axis'; + const tx = (config as any).axisTransforms?.[axis] || {}; + const setTx = (patch: Record) => { + const newConfig = { + ...config, + axisTransforms: { + ...((config as any).axisTransforms || {}), + [axis]: { ...tx, ...patch }, + }, + }; + setConfig(newConfig); + validateConfig(newConfig); + }; + return ( +
+ + + +
+ ); + })} ) : config.typeId !== 'metric-table' && config.typeId !== 'tokenized-text' && (
diff --git a/frontend/src/components/graphs/ScatterGraph.tsx b/frontend/src/components/graphs/ScatterGraph.tsx index cbc9d9d..ffadf10 100644 --- a/frontend/src/components/graphs/ScatterGraph.tsx +++ b/frontend/src/components/graphs/ScatterGraph.tsx @@ -1,53 +1,21 @@ import React from 'react'; import { FigureConfig, VisualizationData } from '../../types'; import { - Scatter, - XAxis, - YAxis, - CartesianGrid, + Chart as ChartJS, + LinearScale, + LogarithmicScale, + PointElement, + LineElement, + LineController, Tooltip, Legend, - ResponsiveContainer, - ComposedChart, - Line, -} from 'recharts'; +} from 'chart.js'; +import { Scatter } from 'react-chartjs-2'; +import zoomPlugin from 'chartjs-plugin-zoom'; import { buildLanguageLabelMap, getDisplayLanguageLabel, getDisplayLanguagePairLabel } from '../../utils/languageLabels'; -import { getColorForMetric } from './graphColors'; +import { getColorForMetric, GRAPH_COLORS } from './graphColors'; -// --------------------------------------------------------------------------- -// Tooltip -// --------------------------------------------------------------------------- - -const ScatterTooltip: React.FC<{ - active?: boolean; - payload?: any[]; - metricX: string; - metricY: string; - formatLanguage?: (label: string, isPair: boolean) => string; -}> = ({ active, payload, metricX, metricY, formatLanguage }) => { - if (!active || !payload || payload.length === 0) return null; - - const pt = payload[0]?.payload || {}; - const hasLanguagePair = pt.languagePair !== undefined && pt.languagePair !== null; - const rawLanguageLabel = hasLanguagePair ? pt.languagePair : pt.language; - const languageLabel = - rawLanguageLabel !== undefined && formatLanguage - ? formatLanguage(String(rawLanguageLabel), hasLanguagePair) - : rawLanguageLabel; - const languageTitle = hasLanguagePair ? 'Language pair' : 'Language'; - const clusterSize = typeof pt.clusterSize === 'number' ? pt.clusterSize : undefined; - const formatVal = (v: any) => (typeof v === 'number' ? v.toFixed(2) : v); - - return ( -
-

Tokenizer: {pt.tokenizer ?? 'N/A'}

- {languageLabel !== undefined &&

{languageTitle}: {languageLabel}

} - {clusterSize !== undefined && clusterSize > 1 &&

Points in cluster: {clusterSize}

} -

{metricX}: {formatVal(pt[metricX])}

-

{metricY}: {formatVal(pt[metricY])}

-
- ); -}; +ChartJS.register(LinearScale, LogarithmicScale, PointElement, LineElement, LineController, Tooltip, Legend, zoomPlugin); // --------------------------------------------------------------------------- // Downsampling @@ -133,6 +101,18 @@ function computeTrend( return { m, b, minX, maxX }; } +// --------------------------------------------------------------------------- +// hex color → rgba helper +// --------------------------------------------------------------------------- + +function hexToRgba(hex: string, alpha: number): string { + const h = hex.replace('#', ''); + const r = parseInt(h.substring(0, 2), 16); + const g = parseInt(h.substring(2, 4), 16); + const b = parseInt(h.substring(4, 6), 16); + return `rgba(${r},${g},${b},${alpha})`; +} + // --------------------------------------------------------------------------- // Component // --------------------------------------------------------------------------- @@ -147,6 +127,27 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) const metricX = config.metrics[0]; const metricY = config.metrics[1]; const groupBy = config.groupBy || 'tokenizer'; + const chartRef = React.useRef>(null); + + const txX = config.axisTransforms?.x; + const txY = config.axisTransforms?.y; + const scaleX = txX?.scale ?? 'linear'; + const scaleY = txY?.scale ?? 'linear'; + const flipX = txX?.flip ?? false; + const flipY = txY?.flip ?? false; + + // Apply per-axis transforms to a raw data value + const applyX = (v: number) => (flipX ? -v : v); + const applyY = (v: number) => (flipY ? -v : v); + + // Build axis label with active transform annotations + const axisLabel = (metric: string, scale: string, flip: boolean) => { + const parts: string[] = []; + if (flip) parts.push('−'); + parts.push(metric); + if (scale === 'log') parts.push('[log]'); + return parts.join(''); + }; const allLanguages = data.metadata?.languages || []; const languageLabelMap = React.useMemo( @@ -195,7 +196,11 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) const isValidPoint = (pt: any): boolean => { const x = pt[metricX], y = pt[metricY]; - return typeof x === 'number' && typeof y === 'number' && !Number.isNaN(x) && !Number.isNaN(y) && isFinite(x) && isFinite(y); + if (typeof x !== 'number' || typeof y !== 'number' || Number.isNaN(x) || Number.isNaN(y) || !isFinite(x) || !isFinite(y)) return false; + const tx = applyX(x), ty = applyY(y); + if (scaleX === 'log' && tx <= 0) return false; + if (scaleY === 'log' && ty <= 0) return false; + return true; }; const allPoints = (Array.isArray(chartData) ? chartData : []).filter(isValidPoint); @@ -209,87 +214,161 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) const groupNames = Array.from(groupsMap.keys()); const trendlineMode: 'none' | 'global' | 'groups' = - (config as any).trendlineMode || (config.showTrendline ? 'global' : 'none'); + config.trendlineMode || (config.showTrendline ? 'global' : 'none'); - const buildTrendData = (trend: { m: number; b: number; minX: number; maxX: number }) => [ - { [metricX]: trend.minX, [metricY]: trend.m * trend.minX + trend.b }, - { [metricX]: trend.maxX, [metricY]: trend.m * trend.maxX + trend.b }, - ]; + // Build Chart.js datasets + const datasets: any[] = []; - type TrendLineDef = { key: string; name: string; color: string; data: any[] }; - const trendLines: TrendLineDef[] = []; + groupNames.forEach((name, idx) => { + const color = getColorForMetric(idx); + const pts = downsampleGroupPoints(groupsMap.get(name) || [], metricX, metricY, MAX_SCATTER_POINTS_PER_GROUP); + datasets.push({ + type: 'scatter' as const, + label: name, + data: pts.map((pt) => ({ x: applyX(pt[metricX]), y: applyY(pt[metricY]), _raw: pt })), + backgroundColor: hexToRgba(color, 0.7), + borderColor: color, + borderWidth: 1, + pointRadius: 4, + pointHoverRadius: 6, + }); + }); + + // Trendlines as line datasets — computed on transformed values + const trendPointsForGroup = (name: string) => + (groupsMap.get(name) || []) + .filter(isValidPoint) + .map((pt) => ({ [metricX]: applyX(pt[metricX]), [metricY]: applyY(pt[metricY]) })); + const allTransformedPoints = allPoints.map((pt) => ({ [metricX]: applyX(pt[metricX]), [metricY]: applyY(pt[metricY]) })); if (trendlineMode === 'global') { - const trend = computeTrend(allPoints, metricX, metricY); - if (trend) trendLines.push({ key: 'global-trend', name: 'Trend (global)', color: '#444', data: buildTrendData(trend) }); + const trend = computeTrend(allTransformedPoints, metricX, metricY); + if (trend) { + datasets.push({ + type: 'line' as const, + label: 'Trend (global)', + data: [ + { x: trend.minX, y: trend.m * trend.minX + trend.b }, + { x: trend.maxX, y: trend.m * trend.maxX + trend.b }, + ], + borderColor: '#444', + borderDash: [4, 2], + borderWidth: 2, + pointRadius: 0, + fill: false, + tension: 0, + }); + } } else if (trendlineMode === 'groups') { groupNames.forEach((name, idx) => { - const trend = computeTrend(groupsMap.get(name) || [], metricX, metricY); - if (trend) trendLines.push({ key: `trend-${name}`, name: `${name} trend`, color: getColorForMetric(idx), data: buildTrendData(trend) }); + const trend = computeTrend(trendPointsForGroup(name), metricX, metricY); + if (trend) { + const color = getColorForMetric(idx); + datasets.push({ + type: 'line' as const, + label: `${name} trend`, + data: [ + { x: trend.minX, y: trend.m * trend.minX + trend.b }, + { x: trend.maxX, y: trend.m * trend.maxX + trend.b }, + ], + borderColor: color, + borderDash: [4, 2], + borderWidth: 2, + pointRadius: 0, + fill: false, + tension: 0, + }); + } }); } - const displayGroupsMap: Map = new Map(); - groupNames.forEach((name) => { - displayGroupsMap.set(name, downsampleGroupPoints(groupsMap.get(name) || [], metricX, metricY, MAX_SCATTER_POINTS_PER_GROUP)); - }); + const options: any = { + animation: false, + responsive: true, + maintainAspectRatio: false, + parsing: false, + scales: { + x: { + type: (scaleX === 'log' ? 'logarithmic' : 'linear') as 'linear' | 'logarithmic', + title: { display: true, text: axisLabel(metricX, scaleX, flipX) }, + }, + y: { + type: (scaleY === 'log' ? 'logarithmic' : 'linear') as 'linear' | 'logarithmic', + title: { display: true, text: axisLabel(metricY, scaleY, flipY) }, + }, + }, + plugins: { + legend: { + position: 'bottom' as const, + }, + tooltip: { + callbacks: { + label: (context: any) => { + const raw = context.raw as any; + const pt = raw?._raw || raw; + const hasLanguagePair = pt?.languagePair !== undefined && pt?.languagePair !== null; + const rawLabel = hasLanguagePair ? pt.languagePair : pt.language; + const displayLabel = + rawLabel !== undefined + ? hasLanguagePair + ? getDisplayLanguagePairLabel(String(rawLabel), languageLabelMap) + : getDisplayLanguageLabel(String(rawLabel), languageLabelMap) + : undefined; + const languageTitle = hasLanguagePair ? 'Language pair' : 'Language'; + const clusterSize = typeof pt?.clusterSize === 'number' ? pt.clusterSize : undefined; + const fmt = (v: any) => (typeof v === 'number' ? v.toFixed(4) : String(v)); + const lines = [ + `Tokenizer: ${pt?.tokenizer ?? context.dataset.label ?? 'N/A'}`, + ...(displayLabel !== undefined ? [`${languageTitle}: ${displayLabel}`] : []), + ...(clusterSize !== undefined && clusterSize > 1 ? [`Points in cluster: ${clusterSize}`] : []), + `${metricX}: ${fmt(raw.x)}`, + `${metricY}: ${fmt(raw.y)}`, + ]; + return lines; + }, + }, + }, + zoom: { + pan: { + enabled: true, + mode: 'xy' as const, + modifierKey: undefined, + }, + zoom: { + wheel: { enabled: true }, + pinch: { enabled: true }, + mode: 'xy' as const, + }, + }, + }, + }; + + const handleResetZoom = () => { + chartRef.current?.resetZoom(); + }; return ( - - - - - - - isPair - ? getDisplayLanguagePairLabel(label, languageLabelMap) - : getDisplayLanguageLabel(label, languageLabelMap) - } - /> - )} - /> - - {trendLines.map((t) => ( - - ))} - {groupNames.map((name, idx) => ( - - ))} - - +
+ + +
); }; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index b2556de..34697a9 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -42,6 +42,16 @@ export interface FigureConfig { trendlineMode?: 'none' | 'global' | 'groups'; // For tokenized-text type: 1-based inclusive range [start, end] sentenceRange?: [number, number]; + // Per-axis transforms for metric-pair-correlation + axisTransforms?: { + x?: AxisTransform; + y?: AxisTransform; + }; +} + +export interface AxisTransform { + scale?: 'linear' | 'log'; + flip?: boolean; } export interface MetricDimensionality { From 8a05eb6714ee888de5f14d5156d653742ae69273 Mon Sep 17 00:00:00 2001 From: varisd Date: Sat, 4 Jul 2026 21:04:21 +0200 Subject: [PATCH 2/8] webapp (ScatterGraph): improved Log-scale transform (includes negative part of the axis now) --- .../src/components/graphs/ScatterGraph.tsx | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/frontend/src/components/graphs/ScatterGraph.tsx b/frontend/src/components/graphs/ScatterGraph.tsx index ffadf10..f65cb56 100644 --- a/frontend/src/components/graphs/ScatterGraph.tsx +++ b/frontend/src/components/graphs/ScatterGraph.tsx @@ -17,6 +17,24 @@ import { getColorForMetric, GRAPH_COLORS } from './graphColors'; ChartJS.register(LinearScale, LogarithmicScale, PointElement, LineElement, LineController, Tooltip, Legend, zoomPlugin); +// --------------------------------------------------------------------------- +// Symlog transform: sign(x) * log10(1 + |x|) — handles negative values +// --------------------------------------------------------------------------- + +function symlog(x: number): number { + return Math.sign(x) * Math.log10(1 + Math.abs(x)); +} + +function isymlog(y: number): number { + return Math.sign(y) * (Math.pow(10, Math.abs(y)) - 1); +} + +function fmtSymlogTick(v: number): string { + const orig = isymlog(v); + if (orig === 0) return '0'; + return parseFloat(orig.toPrecision(3)).toString(); +} + // --------------------------------------------------------------------------- // Downsampling // --------------------------------------------------------------------------- @@ -137,15 +155,16 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) const flipY = txY?.flip ?? false; // Apply per-axis transforms to a raw data value - const applyX = (v: number) => (flipX ? -v : v); - const applyY = (v: number) => (flipY ? -v : v); + // For log scale we use symlog so negative values are preserved + const applyX = (v: number) => { const f = flipX ? -v : v; return scaleX === 'log' ? symlog(f) : f; }; + const applyY = (v: number) => { const f = flipY ? -v : v; return scaleY === 'log' ? symlog(f) : f; }; // Build axis label with active transform annotations const axisLabel = (metric: string, scale: string, flip: boolean) => { const parts: string[] = []; if (flip) parts.push('−'); parts.push(metric); - if (scale === 'log') parts.push('[log]'); + if (scale === 'log') parts.push('[symlog]'); return parts.join(''); }; @@ -197,9 +216,6 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) const isValidPoint = (pt: any): boolean => { const x = pt[metricX], y = pt[metricY]; if (typeof x !== 'number' || typeof y !== 'number' || Number.isNaN(x) || Number.isNaN(y) || !isFinite(x) || !isFinite(y)) return false; - const tx = applyX(x), ty = applyY(y); - if (scaleX === 'log' && tx <= 0) return false; - if (scaleY === 'log' && ty <= 0) return false; return true; }; @@ -289,12 +305,14 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) parsing: false, scales: { x: { - type: (scaleX === 'log' ? 'logarithmic' : 'linear') as 'linear' | 'logarithmic', + type: 'linear' as const, title: { display: true, text: axisLabel(metricX, scaleX, flipX) }, + ticks: scaleX === 'log' ? { callback: (v: any) => fmtSymlogTick(v) } : {}, }, y: { - type: (scaleY === 'log' ? 'logarithmic' : 'linear') as 'linear' | 'logarithmic', + type: 'linear' as const, title: { display: true, text: axisLabel(metricY, scaleY, flipY) }, + ticks: scaleY === 'log' ? { callback: (v: any) => fmtSymlogTick(v) } : {}, }, }, plugins: { @@ -317,12 +335,15 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) const languageTitle = hasLanguagePair ? 'Language pair' : 'Language'; const clusterSize = typeof pt?.clusterSize === 'number' ? pt.clusterSize : undefined; const fmt = (v: any) => (typeof v === 'number' ? v.toFixed(4) : String(v)); + // Show original (pre-transform) values from _raw when available + const dispX = pt?.[metricX] !== undefined ? fmt(pt[metricX]) : fmt(raw.x); + const dispY = pt?.[metricY] !== undefined ? fmt(pt[metricY]) : fmt(raw.y); const lines = [ `Tokenizer: ${pt?.tokenizer ?? context.dataset.label ?? 'N/A'}`, ...(displayLabel !== undefined ? [`${languageTitle}: ${displayLabel}`] : []), ...(clusterSize !== undefined && clusterSize > 1 ? [`Points in cluster: ${clusterSize}`] : []), - `${metricX}: ${fmt(raw.x)}`, - `${metricY}: ${fmt(raw.y)}`, + `${metricX}: ${dispX}`, + `${metricY}: ${dispY}`, ]; return lines; }, From cb2249835dad724d39451361cf53f8b5a09badfe Mon Sep 17 00:00:00 2001 From: varisd Date: Sat, 4 Jul 2026 21:15:04 +0200 Subject: [PATCH 3/8] webapp: split metric correlation visulationa to mono and bilingual metric visualizer --- frontend/src/components/Graph.tsx | 2 +- frontend/src/components/GraphConfigurator.tsx | 4 +- frontend/src/utils/graphTypes.ts | 155 ++++++++++-------- 3 files changed, 94 insertions(+), 67 deletions(-) diff --git a/frontend/src/components/Graph.tsx b/frontend/src/components/Graph.tsx index 03d2f83..07852d7 100644 --- a/frontend/src/components/Graph.tsx +++ b/frontend/src/components/Graph.tsx @@ -54,7 +54,7 @@ const Graph: React.FC = ({ config, data }) => { } const renderContent = () => { - if (config.typeId === 'metric-pair-correlation') { + if (config.typeId === 'metric-pair-correlation-mono' || config.typeId === 'metric-pair-correlation-bili') { return ; } if (config.typeId === 'metric-table') { diff --git a/frontend/src/components/GraphConfigurator.tsx b/frontend/src/components/GraphConfigurator.tsx index ba64a5f..62bac42 100644 --- a/frontend/src/components/GraphConfigurator.tsx +++ b/frontend/src/components/GraphConfigurator.tsx @@ -193,7 +193,7 @@ const GraphConfigurator: React.FC = ({ // Defaults for metrics depend on graph type and constraints let defaultMetrics: string[] = []; - if (newTypeId === 'metric-pair-correlation') { + if (newTypeId === 'metric-pair-correlation-mono' || newTypeId === 'metric-pair-correlation-bili') { // Choose first for X, second for Y if (filteredMetrics.length >= 2) { defaultMetrics = [filteredMetrics[0], filteredMetrics[1]]; @@ -513,7 +513,7 @@ const GraphConfigurator: React.FC = ({
- {config.typeId === 'metric-pair-correlation' ? ( + {(config.typeId === 'metric-pair-correlation-mono' || config.typeId === 'metric-pair-correlation-bili') ? (
= ({
diff --git a/frontend/src/components/graphs/ScatterGraph.tsx b/frontend/src/components/graphs/ScatterGraph.tsx index f65cb56..501e918 100644 --- a/frontend/src/components/graphs/ScatterGraph.tsx +++ b/frontend/src/components/graphs/ScatterGraph.tsx @@ -209,6 +209,7 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) const groupKeyForPoint = (pt: any): string => { if (groupBy === 'tokenizer') return pt.tokenizer || 'unknown'; if (groupBy === 'language') return pt.language || 'unknown'; + if (groupBy === 'languagePair') return pt.languagePair || 'unknown'; if (groupBy === 'family') return getFamilyForLanguage(pt.language || ''); return 'unknown'; }; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 34697a9..0887985 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -35,7 +35,7 @@ export interface FigureConfig { languages: string[]; metrics: string[]; filters?: Record; - groupBy?: 'tokenizer' | 'language' | 'family'; + groupBy?: 'tokenizer' | 'language' | 'languagePair' | 'family'; // Deprecated boolean flag kept for backward compatibility showTrendline?: boolean; // New, more expressive trendline mode diff --git a/frontend/src/utils/__tests__/graphTypes.test.ts b/frontend/src/utils/__tests__/graphTypes.test.ts index 1ceab63..f276096 100644 --- a/frontend/src/utils/__tests__/graphTypes.test.ts +++ b/frontend/src/utils/__tests__/graphTypes.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from 'vitest'; import { - MetricPairCorrelationGraphType, + MonolingualMetricPairCorrelationGraphType, + BilingualMetricPairCorrelationGraphType, MetricTableGraphType, TokenizedTextGraphType, } from '../graphTypes'; @@ -25,11 +26,11 @@ function dims(map: Record = {}): MetricDimensionality { } // --------------------------------------------------------------------------- -// MetricPairCorrelationGraphType +// MonolingualMetricPairCorrelationGraphType // --------------------------------------------------------------------------- -describe('MetricPairCorrelationGraphType.validate()', () => { - const gType = new MetricPairCorrelationGraphType(); +describe('MonolingualMetricPairCorrelationGraphType.validate()', () => { + const gType = new MonolingualMetricPairCorrelationGraphType(); it('passes with two 2D metrics', () => { const result = gType.validate( @@ -40,18 +41,10 @@ describe('MetricPairCorrelationGraphType.validate()', () => { expect(result.errors).toHaveLength(0); }); - it('passes with two 3D metrics of the same dimensionality', () => { - const result = gType.validate( - cfg({ metrics: ['pmi', 'js_div'] }), - dims({ pmi: 3, js_div: 3 }), - ); - expect(result.valid).toBe(true); - }); - it('fails when fewer than 2 metrics provided', () => { const result = gType.validate(cfg({ metrics: ['seq_len'] }), dims({ seq_len: 2 })); expect(result.valid).toBe(false); - expect(result.errors.some((e) => /minimum/i.test(e))).toBe(true); + expect(result.errors.some((e: string) => /minimum/i.test(e))).toBe(true); }); it('fails when more than 2 metrics provided', () => { @@ -60,25 +53,24 @@ describe('MetricPairCorrelationGraphType.validate()', () => { dims({ a: 2, b: 2, c: 2 }), ); expect(result.valid).toBe(false); - expect(result.errors.some((e) => /maximum/i.test(e))).toBe(true); + expect(result.errors.some((e: string) => /maximum/i.test(e))).toBe(true); }); - it('fails when a metric is 1D', () => { + it('fails when a metric is not 2D', () => { const result = gType.validate( cfg({ metrics: ['freq', 'seq_len'] }), dims({ freq: 1, seq_len: 2 }), ); expect(result.valid).toBe(false); - expect(result.errors.some((e) => /2D or 3D/i.test(e))).toBe(true); + expect(result.errors.some((e: string) => /2D/i.test(e))).toBe(true); }); - it('fails when metrics have different dimensionalities', () => { + it('fails when a 3D metric is provided', () => { const result = gType.validate( cfg({ metrics: ['seq_len', 'pmi'] }), dims({ seq_len: 2, pmi: 3 }), ); expect(result.valid).toBe(false); - expect(result.errors.some((e) => /different dimensionality/i.test(e))).toBe(true); }); it('fails when no tokenizers are selected', () => { @@ -87,7 +79,7 @@ describe('MetricPairCorrelationGraphType.validate()', () => { dims({ seq_len: 2, vocab_size: 2 }), ); expect(result.valid).toBe(false); - expect(result.errors.some((e) => /tokenizer/i.test(e))).toBe(true); + expect(result.errors.some((e: string) => /tokenizer/i.test(e))).toBe(true); }); it('fails when no languages are selected', () => { @@ -96,17 +88,69 @@ describe('MetricPairCorrelationGraphType.validate()', () => { dims({ seq_len: 2, vocab_size: 2 }), ); expect(result.valid).toBe(false); - expect(result.errors.some((e) => /language/i.test(e))).toBe(true); + expect(result.errors.some((e: string) => /language/i.test(e))).toBe(true); }); it('accumulates multiple errors at once', () => { const result = gType.validate(cfg({ metrics: [] }), dims()); expect(result.valid).toBe(false); - // At minimum: too few metrics error expect(result.errors.length).toBeGreaterThanOrEqual(1); }); }); +// --------------------------------------------------------------------------- +// BilingualMetricPairCorrelationGraphType +// --------------------------------------------------------------------------- + +describe('BilingualMetricPairCorrelationGraphType.validate()', () => { + const gType = new BilingualMetricPairCorrelationGraphType(); + + it('passes with two 3D metrics', () => { + const result = gType.validate( + cfg({ metrics: ['pmi', 'js_div'], languages: ['eng_Latn', 'deu_Latn'] }), + dims({ pmi: 3, js_div: 3 }), + ); + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it('fails when fewer than 2 metrics provided', () => { + const result = gType.validate( + cfg({ metrics: ['pmi'], languages: ['eng_Latn', 'deu_Latn'] }), + dims({ pmi: 3 }), + ); + expect(result.valid).toBe(false); + expect(result.errors.some((e: string) => /minimum/i.test(e))).toBe(true); + }); + + it('fails when a 2D metric is provided', () => { + const result = gType.validate( + cfg({ metrics: ['pmi', 'seq_len'], languages: ['eng_Latn', 'deu_Latn'] }), + dims({ pmi: 3, seq_len: 2 }), + ); + expect(result.valid).toBe(false); + expect(result.errors.some((e: string) => /3D/i.test(e))).toBe(true); + }); + + it('fails when no tokenizers are selected', () => { + const result = gType.validate( + cfg({ metrics: ['pmi', 'js_div'], tokenizers: [], languages: ['eng_Latn', 'deu_Latn'] }), + dims({ pmi: 3, js_div: 3 }), + ); + expect(result.valid).toBe(false); + expect(result.errors.some((e: string) => /tokenizer/i.test(e))).toBe(true); + }); + + it('fails when fewer than 2 languages are selected', () => { + const result = gType.validate( + cfg({ metrics: ['pmi', 'js_div'], languages: ['eng_Latn'] }), + dims({ pmi: 3, js_div: 3 }), + ); + expect(result.valid).toBe(false); + expect(result.errors.some((e: string) => /language/i.test(e))).toBe(true); + }); +}); + // --------------------------------------------------------------------------- // MetricTableGraphType // --------------------------------------------------------------------------- @@ -223,29 +267,47 @@ describe('GraphType base validate – tokenizer and language count edges', () => // getCompatibleMetrics // --------------------------------------------------------------------------- -describe('MetricPairCorrelationGraphType.getCompatibleMetrics()', () => { - const gType = new MetricPairCorrelationGraphType(); +describe('MonolingualMetricPairCorrelationGraphType.getCompatibleMetrics()', () => { + const gType = new MonolingualMetricPairCorrelationGraphType(); const allMetrics = ['seq_len', 'vocab_size', 'pmi', 'freq']; const dimensionality = { seq_len: 2, vocab_size: 2, pmi: 3, freq: 1 } as Record; - it('returns only 2D and 3D metrics', () => { + it('returns only 2D metrics', () => { const result = gType.getCompatibleMetrics(allMetrics, dimensionality); - expect(result).toEqual(['seq_len', 'vocab_size', 'pmi']); + expect(result).toEqual(['seq_len', 'vocab_size']); }); - it('excludes 1D metrics', () => { + it('excludes 1D and 3D metrics', () => { const result = gType.getCompatibleMetrics(allMetrics, dimensionality); expect(result).not.toContain('freq'); + expect(result).not.toContain('pmi'); }); - it('returns empty array when all metrics are 1D', () => { - const result = gType.getCompatibleMetrics(['freq'], { freq: 1 }); + it('returns empty array when no 2D metrics', () => { + const result = gType.getCompatibleMetrics(['freq', 'pmi'], { freq: 1, pmi: 3 }); expect(result).toEqual([]); }); +}); - it('returns all metrics when all are 2D or 3D', () => { - const result = gType.getCompatibleMetrics(['a', 'b'], { a: 2, b: 3 }); - expect(result).toEqual(['a', 'b']); +describe('BilingualMetricPairCorrelationGraphType.getCompatibleMetrics()', () => { + const gType = new BilingualMetricPairCorrelationGraphType(); + const allMetrics = ['seq_len', 'vocab_size', 'pmi', 'freq']; + const dimensionality = { seq_len: 2, vocab_size: 2, pmi: 3, freq: 1 } as Record; + + it('returns only 3D metrics', () => { + const result = gType.getCompatibleMetrics(allMetrics, dimensionality); + expect(result).toEqual(['pmi']); + }); + + it('excludes 1D and 2D metrics', () => { + const result = gType.getCompatibleMetrics(allMetrics, dimensionality); + expect(result).not.toContain('freq'); + expect(result).not.toContain('seq_len'); + }); + + it('returns empty array when no 3D metrics', () => { + const result = gType.getCompatibleMetrics(['freq', 'seq_len'], { freq: 1, seq_len: 2 }); + expect(result).toEqual([]); }); }); diff --git a/frontend/src/utils/graphTypes.ts b/frontend/src/utils/graphTypes.ts index 97435bd..52ea284 100644 --- a/frontend/src/utils/graphTypes.ts +++ b/frontend/src/utils/graphTypes.ts @@ -9,7 +9,7 @@ type TypedArray = Float64Array | Float32Array | Int32Array | Uint32Array | Uint8 export interface ConstraintRange { min: number; max: number; - dimension?: 1 | 2 | 'both'; // For metrics: 1D, 2D, or both + dimension?: 1 | 2 | 3 | 'both'; // For metrics: 1D, 2D, 3D, or both } export interface VisualizationConstraints { @@ -219,7 +219,7 @@ export class MonolingualMetricPairCorrelationGraphType extends GraphType { * Scatter plot visualization for correlating two 3D (bilingual) metrics */ export class BilingualMetricPairCorrelationGraphType extends GraphType { - typeId = 'metric-pair-correlation-bili'; + typeId = 'metric-pair-correlation-bi'; displayName = 'Metric Pair Correlation (Bilingual)'; description = 'Scatterplot showing the relationship between two 3D (per-language-pair) metrics. Choose X and Y axes, tokenizers, and languages.'; @@ -268,6 +268,7 @@ export class BilingualMetricPairCorrelationGraphType extends GraphType { const l1Idx = allLanguages.indexOf(lang1); if (l1Idx < 0) continue; for (const lang2 of config.languages) { + if (lang1 === lang2) continue; const l2Idx = allLanguages.indexOf(lang2); if (l2Idx < 0) continue; const xVal = xData[tokIdx * xShape[1] * xShape[2] + l1Idx * xShape[2] + l2Idx]; From ee7e97b848ea8c9aa934fe94ee2aa00fb32ea8fc Mon Sep 17 00:00:00 2001 From: varisd Date: Sun, 5 Jul 2026 00:30:47 +0200 Subject: [PATCH 5/8] webapp: properly grouping language pairs based on their language family inclusion --- frontend/src/components/graphs/ScatterGraph.tsx | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/graphs/ScatterGraph.tsx b/frontend/src/components/graphs/ScatterGraph.tsx index 501e918..18de4ca 100644 --- a/frontend/src/components/graphs/ScatterGraph.tsx +++ b/frontend/src/components/graphs/ScatterGraph.tsx @@ -206,11 +206,26 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) return 'unknown'; }; + const getFamilyGroupKey = (pt: any): string => { + if (pt.languagePair) { + const dashIdx = pt.languagePair.indexOf('-'); + if (dashIdx > 0) { + const lang1 = pt.languagePair.slice(0, dashIdx); + const lang2 = pt.languagePair.slice(dashIdx + 1); + const fam1 = getFamilyForLanguage(lang1); + const fam2 = getFamilyForLanguage(lang2); + const sorted = [fam1, fam2].sort(); + return `${sorted[0]} × ${sorted[1]}`; + } + } + return getFamilyForLanguage(pt.language || ''); + }; + const groupKeyForPoint = (pt: any): string => { if (groupBy === 'tokenizer') return pt.tokenizer || 'unknown'; if (groupBy === 'language') return pt.language || 'unknown'; if (groupBy === 'languagePair') return pt.languagePair || 'unknown'; - if (groupBy === 'family') return getFamilyForLanguage(pt.language || ''); + if (groupBy === 'family') return getFamilyGroupKey(pt); return 'unknown'; }; From 27bcd22181d1d5da167e805a454725fadce5473b Mon Sep 17 00:00:00 2001 From: varisd Date: Sun, 5 Jul 2026 00:50:17 +0200 Subject: [PATCH 6/8] webapp: added confidence band to the trendline --- .../src/components/graphs/ScatterGraph.tsx | 115 +++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/graphs/ScatterGraph.tsx b/frontend/src/components/graphs/ScatterGraph.tsx index 18de4ca..316975a 100644 --- a/frontend/src/components/graphs/ScatterGraph.tsx +++ b/frontend/src/components/graphs/ScatterGraph.tsx @@ -7,6 +7,7 @@ import { PointElement, LineElement, LineController, + Filler, Tooltip, Legend, } from 'chart.js'; @@ -15,7 +16,7 @@ import zoomPlugin from 'chartjs-plugin-zoom'; import { buildLanguageLabelMap, getDisplayLanguageLabel, getDisplayLanguagePairLabel } from '../../utils/languageLabels'; import { getColorForMetric, GRAPH_COLORS } from './graphColors'; -ChartJS.register(LinearScale, LogarithmicScale, PointElement, LineElement, LineController, Tooltip, Legend, zoomPlugin); +ChartJS.register(LinearScale, LogarithmicScale, PointElement, LineElement, LineController, Filler, Tooltip, Legend, zoomPlugin); // --------------------------------------------------------------------------- // Symlog transform: sign(x) * log10(1 + |x|) — handles negative values @@ -98,11 +99,22 @@ function downsampleGroupPoints( // Trend computation // --------------------------------------------------------------------------- +interface TrendResult { + m: number; + b: number; + minX: number; + maxX: number; + n: number; + meanX: number; + sxx: number; // sum of squared deviations of x + se: number; // residual standard error +} + function computeTrend( pts: any[], metricX: string, metricY: string, -): { m: number; b: number; minX: number; maxX: number } | null { +): TrendResult | null { if (!pts || pts.length < 2) return null; let sumX = 0, sumY = 0, sumXY = 0, sumXX = 0, minX = Infinity, maxX = -Infinity; for (const p of pts) { @@ -116,7 +128,49 @@ function computeTrend( if (denom === 0) return null; const m = (n * sumXY - sumX * sumY) / denom; const b = (sumY - m * sumX) / n; - return { m, b, minX, maxX }; + const meanX = sumX / n; + const sxx = sumXX - n * meanX * meanX; + // Residual sum of squares + let rss = 0; + for (const p of pts) { + const x = p[metricX], y = p[metricY]; + const residual = y - (m * x + b); + rss += residual * residual; + } + const se = n > 2 ? Math.sqrt(rss / (n - 2)) : 0; + return { m, b, minX, maxX, n, meanX, sxx, se }; +} + +// Approximate t critical value for 95% CI (two-tailed). +// Lookup table of [max_df, t_value] pairs, checked in ascending order. +const T_CRITICAL_95: [number, number][] = [ + [1, 12.706], [2, 4.303], [3, 3.182], [4, 2.776], [5, 2.571], + [6, 2.447], [7, 2.365], [8, 2.306], [9, 2.262], [10, 2.228], + [15, 2.131], [20, 2.086], [30, 2.042], [60, 2.000], [120, 1.980], +]; + +function tCritical95(df: number): number { + if (df <= 0) return Infinity; + return T_CRITICAL_95.find(([maxDf]) => df <= maxDf)?.[1] ?? 1.960; +} + +/** Compute upper and lower 95% CI band lines for a trendline. */ +function buildConfidenceBand( + trend: TrendResult, + steps = 60, +): { upper: { x: number; y: number }[]; lower: { x: number; y: number }[] } { + const { m, b, minX, maxX, n, meanX, sxx, se } = trend; + if (se === 0 || n <= 2) return { upper: [], lower: [] }; + const t = tCritical95(n - 2); + const upper: { x: number; y: number }[] = []; + const lower: { x: number; y: number }[] = []; + for (let i = 0; i <= steps; i++) { + const x = minX + (maxX - minX) * (i / steps); + const margin = t * se * Math.sqrt(1 / n + Math.pow(x - meanX, 2) / sxx); + upper.push({ x, y: m * x + b + margin }); + lower.push({ x, y: m * x + b - margin }); + } + return { upper, lower }; } // --------------------------------------------------------------------------- @@ -276,6 +330,33 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) if (trendlineMode === 'global') { const trend = computeTrend(allTransformedPoints, metricX, metricY); if (trend) { + const { upper, lower } = buildConfidenceBand(trend); + if (upper.length > 0) { + // Upper bound — fills down to the next dataset (lower bound) + datasets.push({ + type: 'line' as const, + label: 'Trend (global) 95% CI upper', + data: upper, + borderColor: 'transparent', + backgroundColor: 'rgba(68,68,68,0.15)', + borderWidth: 0, + pointRadius: 0, + fill: '+1', + tension: 0, + }); + // Lower bound + datasets.push({ + type: 'line' as const, + label: 'Trend (global) 95% CI lower', + data: lower, + borderColor: 'transparent', + backgroundColor: 'transparent', + borderWidth: 0, + pointRadius: 0, + fill: false, + tension: 0, + }); + } datasets.push({ type: 'line' as const, label: 'Trend (global)', @@ -296,6 +377,31 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) const trend = computeTrend(trendPointsForGroup(name), metricX, metricY); if (trend) { const color = getColorForMetric(idx); + const { upper, lower } = buildConfidenceBand(trend); + if (upper.length > 0) { + datasets.push({ + type: 'line' as const, + label: `${name} trend 95% CI upper`, + data: upper, + borderColor: 'transparent', + backgroundColor: hexToRgba(color, 0.15), + borderWidth: 0, + pointRadius: 0, + fill: '+1', + tension: 0, + }); + datasets.push({ + type: 'line' as const, + label: `${name} trend 95% CI lower`, + data: lower, + borderColor: 'transparent', + backgroundColor: 'transparent', + borderWidth: 0, + pointRadius: 0, + fill: false, + tension: 0, + }); + } datasets.push({ type: 'line' as const, label: `${name} trend`, @@ -334,6 +440,9 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) plugins: { legend: { position: 'bottom' as const, + labels: { + filter: (item: any) => !item.text?.includes('95% CI'), + }, }, tooltip: { callbacks: { From be6abbdc3a55d8083fd19c1db168a87f62bb6a4e Mon Sep 17 00:00:00 2001 From: varisd Date: Sun, 5 Jul 2026 02:39:40 +0200 Subject: [PATCH 7/8] webapp: additional trendline configuration --- frontend/src/components/GraphConfigurator.tsx | 26 ++++++++++++++++++- .../src/components/graphs/ScatterGraph.tsx | 6 +++-- frontend/src/types.ts | 2 ++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/frontend/src/components/GraphConfigurator.tsx b/frontend/src/components/GraphConfigurator.tsx index 70fadbc..0179b86 100644 --- a/frontend/src/components/GraphConfigurator.tsx +++ b/frontend/src/components/GraphConfigurator.tsx @@ -66,6 +66,7 @@ const GraphConfigurator: React.FC = ({ languages: [], metrics: [], trendlineMode: 'none', + trendlineUncertainty: 'none', }); const [validationErrors, setValidationErrors] = useState([]); @@ -358,6 +359,7 @@ const GraphConfigurator: React.FC = ({ filters: {}, groupBy: (cfg as any).groupBy || 'tokenizer', trendlineMode, + trendlineUncertainty: (cfg as any).trendlineUncertainty ?? 'none', // Keep boolean flag in sync for any legacy consumers showTrendline: trendlineMode !== 'none', sentenceRange: cfg.sentenceRange, @@ -560,7 +562,11 @@ const GraphConfigurator: React.FC = ({ value={(config as any).trendlineMode || ((config as any).showTrendline ? 'global' : 'none')} onChange={(e) => { const mode = e.target.value as 'none' | 'global' | 'groups'; - const newConfig = { ...config, trendlineMode: mode }; + const newConfig = { + ...config, + trendlineMode: mode, + ...(mode === 'none' ? { trendlineUncertainty: 'none' as const } : {}), + }; setConfig(newConfig); validateConfig(newConfig); }} @@ -571,6 +577,24 @@ const GraphConfigurator: React.FC = ({ + {((config as any).trendlineMode || 'none') !== 'none' && ( +
+ + +
+ )} {/* Axis transform controls */} {(['x', 'y'] as const).map((axis) => { const metricLabel = axis === 'x' ? 'X-axis' : 'Y-axis'; diff --git a/frontend/src/components/graphs/ScatterGraph.tsx b/frontend/src/components/graphs/ScatterGraph.tsx index 316975a..541cc90 100644 --- a/frontend/src/components/graphs/ScatterGraph.tsx +++ b/frontend/src/components/graphs/ScatterGraph.tsx @@ -327,11 +327,13 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) .map((pt) => ({ [metricX]: applyX(pt[metricX]), [metricY]: applyY(pt[metricY]) })); const allTransformedPoints = allPoints.map((pt) => ({ [metricX]: applyX(pt[metricX]), [metricY]: applyY(pt[metricY]) })); + const showConfidenceBand = (config.trendlineUncertainty ?? 'none') === 'confidence-band'; + if (trendlineMode === 'global') { const trend = computeTrend(allTransformedPoints, metricX, metricY); if (trend) { const { upper, lower } = buildConfidenceBand(trend); - if (upper.length > 0) { + if (showConfidenceBand && upper.length > 0) { // Upper bound — fills down to the next dataset (lower bound) datasets.push({ type: 'line' as const, @@ -378,7 +380,7 @@ const ScatterGraph: React.FC = ({ config, data, chartData }) if (trend) { const color = getColorForMetric(idx); const { upper, lower } = buildConfidenceBand(trend); - if (upper.length > 0) { + if (showConfidenceBand && upper.length > 0) { datasets.push({ type: 'line' as const, label: `${name} trend 95% CI upper`, diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 0887985..a7daabb 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -40,6 +40,8 @@ export interface FigureConfig { showTrendline?: boolean; // New, more expressive trendline mode trendlineMode?: 'none' | 'global' | 'groups'; + // Uncertainty visualization for trendline + trendlineUncertainty?: 'none' | 'confidence-band'; // For tokenized-text type: 1-based inclusive range [start, end] sentenceRange?: [number, number]; // Per-axis transforms for metric-pair-correlation From fff9699369a5f88dcca267792612c647801b107c Mon Sep 17 00:00:00 2001 From: varisd Date: Sun, 5 Jul 2026 02:56:08 +0200 Subject: [PATCH 8/8] webapp: GraphConfigurator refactor - moving some logical parts into separate source files --- frontend/src/components/GraphConfigurator.tsx | 250 ++---------------- .../src/components/LanguageFilterPanel.tsx | 150 +++++++++++ .../scatter/ScatterGraphControls.tsx | 123 +++++++++ .../components/scatter/TrendlineControls.tsx | 50 ++++ 4 files changed, 348 insertions(+), 225 deletions(-) create mode 100644 frontend/src/components/LanguageFilterPanel.tsx create mode 100644 frontend/src/components/scatter/ScatterGraphControls.tsx create mode 100644 frontend/src/components/scatter/TrendlineControls.tsx diff --git a/frontend/src/components/GraphConfigurator.tsx b/frontend/src/components/GraphConfigurator.tsx index 0179b86..2a54a82 100644 --- a/frontend/src/components/GraphConfigurator.tsx +++ b/frontend/src/components/GraphConfigurator.tsx @@ -3,6 +3,8 @@ import { FigureConfig, MetricDimensionality } from '../types'; import { getAvailableGraphTypes, getGraphType } from '../utils/graphTypes'; import { buildLanguageLabelMap, getDisplayLanguageLabel } from '../utils/languageLabels'; import { useLanguageFilters, lookupLanguageInfo } from '../utils/useLanguageFilters'; +import LanguageFilterPanel from './LanguageFilterPanel'; +import ScatterGraphControls from './scatter/ScatterGraphControls'; import './GraphConfigurator.css'; /** Check whether two string arrays contain the same set of values. */ @@ -263,29 +265,6 @@ const GraphConfigurator: React.FC = ({ validateConfig(newConfig); }; - // For metric pair correlation: separate X and Y axis selectors - const handleMetricXChange = (e: React.ChangeEvent) => { - const metricX = e.target.value; - const metrics = [metricX, config.metrics?.[1] || ''].filter(Boolean); - const newConfig = { ...config, metrics }; - setConfig(newConfig); - validateConfig(newConfig); - }; - const handleMetricYChange = (e: React.ChangeEvent) => { - const metricY = e.target.value; - const metrics = [config.metrics?.[0] || '', metricY].filter(Boolean); - const newConfig = { ...config, metrics }; - setConfig(newConfig); - validateConfig(newConfig); - }; - - const handleGroupByChange = (e: React.ChangeEvent) => { - const groupBy = e.target.value as 'tokenizer' | 'language' | 'family'; - const newConfig = { ...config, groupBy }; - setConfig(newConfig); - validateConfig(newConfig); - }; - const handleLanguageChange = (e: React.ChangeEvent) => { const selectedOptions = Array.from(e.target.selectedOptions, (option) => option.value); const newConfig = { @@ -516,125 +495,16 @@ const GraphConfigurator: React.FC = ({ {(config.typeId === 'metric-pair-correlation-mono' || config.typeId === 'metric-pair-correlation-bi') ? ( -
- - - - -
- - -
-
- - -
- {((config as any).trendlineMode || 'none') !== 'none' && ( -
- - -
- )} - {/* Axis transform controls */} - {(['x', 'y'] as const).map((axis) => { - const metricLabel = axis === 'x' ? 'X-axis' : 'Y-axis'; - const tx = (config as any).axisTransforms?.[axis] || {}; - const setTx = (patch: Record) => { - const newConfig = { - ...config, - axisTransforms: { - ...((config as any).axisTransforms || {}), - [axis]: { ...tx, ...patch }, - }, - }; - setConfig(newConfig); - validateConfig(newConfig); - }; - return ( -
- - - -
- ); - })} -
+ { + const newConfig = { ...config, ...patch }; + setConfig(newConfig); + validateConfig(newConfig); + }} + /> ) : config.typeId !== 'metric-table' && config.typeId !== 'tokenized-text' && (