diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 16a00790..407c8143 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -5,7 +5,7 @@ }, "metadata": { "description": "A marketplace for skills and tools for agentic Power BI development.", - "version": "26.20" + "version": "26.24" }, "plugins": [ { diff --git a/README.md b/README.md index e9962419..2a948fd2 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@

- Version + Version Power BI Microsoft Fabric Tabular Editor @@ -156,6 +156,7 @@ Hook checks can be individually toggled via config files. Set any check to `fals |------|------|-------------| | Skill | [`bpa-rules`](plugins/tabular-editor/skills/bpa-rules/) | Create and improve Best Practice Analyzer rules for models | | Skill | [`c-sharp-scripting`](plugins/tabular-editor/skills/c-sharp-scripting/) | C# scripting and macros for TE | +| Skill | [`te-cli`](plugins/tabular-editor/skills/te-cli/) | Cross-platform Tabular Editor CLI (`te`, preview) for semantic models from the terminal | | Skill | [`te2-cli`](plugins/tabular-editor/skills/te2-cli/) | Tabular Editor 2 CLI usage and automation (not TE3) | | Skill | [`te-docs`](plugins/tabular-editor/skills/te-docs/) | Tabular Editor documentation search, TE3 config files. Uses [`pbi-search`](https://github.com/data-goblin/pbi-search) CLI | | Command | [`/suggest-rule`](plugins/tabular-editor/commands/suggest-rule.md) | Generate BPA rules from descriptions | @@ -168,7 +169,7 @@ Hook checks can be individually toggled via config files. Set any check to `fals | Type | Name | Description | |------|------|-------------| -| Skill | [`connect-pbid`](plugins/pbi-desktop/skills/connect-pbid/) | Explore, query, and modify a model in Power BI Desktop | +| Skill | [`connect-pbid`](plugins/pbi-desktop/skills/connect-pbid/) | Explore, query, and modify a model in Power BI Desktop, and reload/screenshot the report canvas via the Desktop Bridge | | Agent | [`query-listener`](plugins/pbi-desktop/agents/query-listener.agent.md) | Capture DAX queries from Power BI Desktop visuals in real time | | Hook | DAX reference validation | Validates table, column, and measure references against the connected model; suggests corrections | | Hook | Measure metadata enforcement | Blocks adding measures without DisplayFolder, Description, and FormatString | @@ -205,7 +206,7 @@ Hook checks can be individually toggled via config files. Set any check to `fals | Skill | [`python-visuals`](plugins/reports/skills/python-visuals/) | Custom Python visuals in Power BI reports | | Skill | [`svg-visuals`](plugins/reports/skills/svg-visuals/) | SVG visuals via DAX measures in Power BI reports | | Skill | [`review-report`](plugins/reports/skills/review-report/) (WIP) | Review Power BI reports for usage metrics and best practices | -| Skill | [`pbir-cli`](plugins/reports/skills/pbir-cli/) | Programmatic report manipulation via the [`pbir` CLI](https://github.com/maxanatsko/pbir.tools) | +| Skill | [`pbir-cli`](plugins/reports/skills/pbir-cli/) | Programmatic report manipulation via the [`pbir` CLI](https://github.com/maxanatsko/pbir.tools), including live Power BI Desktop refresh and page screenshots | | Agent | [`deneb-reviewer`](plugins/reports/agents/deneb-reviewer.agent.md) | Review Deneb visual specs for Vega/Vega-Lite syntax and conventions | | Agent | [`svg-reviewer`](plugins/reports/agents/svg-reviewer.agent.md) | Review SVG DAX measures for syntax and design quality | | Agent | [`r-reviewer`](plugins/reports/agents/r-reviewer.agent.md) | Review R visual scripts (ggplot2) for Power BI conventions | @@ -218,9 +219,9 @@ Hook checks can be individually toggled via config files. Set any check to `fals | Type | Name | Description | |------|------|-------------| +| Skill | [`semantic-model`](plugins/semantic-models/skills/semantic-model/) | Design, build, refresh, and review semantic models through a `te`-first tool cascade | | Skill | [`standardize-naming-conventions`](plugins/semantic-models/skills/standardize-naming-conventions/) | Audit and standardize naming conventions in semantic models | -| Skill | [`review-semantic-model`](plugins/semantic-models/skills/review-semantic-model/) (Very WIP) | Review semantic models for quality, performance, AI readiness, and best practices | -| Skill | [`refreshing-semantic-model`](plugins/semantic-models/skills/refreshing-semantic-model/) | Trigger or troubleshoot refreshes | +| Skill | [`refresh-semantic-model`](plugins/semantic-models/skills/refresh-semantic-model/) | Trigger or troubleshoot refreshes | | Skill | [`lineage-analysis`](plugins/semantic-models/skills/lineage-analysis/) | Trace downstream reports from a semantic model across workspaces | | Skill | [`power-query`](plugins/semantic-models/skills/power-query/) | Write M expressions, debug query folding, execute M locally or via Fabric API | | Skill | [`dax`](plugins/semantic-models/skills/dax/) | Write, debug, and optimize DAX in semantic models. Contributed by [Justin Martin](https://daxnoob.blog) | diff --git a/plugins/fabric-admin/.claude-plugin/plugin.json b/plugins/fabric-admin/.claude-plugin/plugin.json index 70f7d0aa..8ba45882 100644 --- a/plugins/fabric-admin/.claude-plugin/plugin.json +++ b/plugins/fabric-admin/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "fabric-admin", - "version": "26.20", + "version": "26.24", "description": "Fabric and Power BI administration; tenant settings audits, governance, delegated overrides, and Entra security group investigation. Requires the fabric-cli plugin.", "author": { "name": "Kurt Buhler", diff --git a/plugins/fabric-admin/skills/audit-tenant-settings/SKILL.md b/plugins/fabric-admin/skills/audit-tenant-settings/SKILL.md index f452ada8..d669b2e3 100644 --- a/plugins/fabric-admin/skills/audit-tenant-settings/SKILL.md +++ b/plugins/fabric-admin/skills/audit-tenant-settings/SKILL.md @@ -1,6 +1,6 @@ --- name: audit-tenant-settings -version: 26.20 +version: 26.24 description: Automatically invoke this skill whenever the user asks about Fabric tenant settings or Power BI tenant settings or auditing tenant settings. You can use this skill if the user mentions "Fabric administration". --- diff --git a/plugins/fabric-cli/.claude-plugin/plugin.json b/plugins/fabric-cli/.claude-plugin/plugin.json index 1cc04ce3..cc3a82ab 100644 --- a/plugins/fabric-cli/.claude-plugin/plugin.json +++ b/plugins/fabric-cli/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "fabric-cli", - "version": "26.20", + "version": "26.24", "description": "Get this plugin to work with Fabric / Power BI service, by means of the fabric cli.", "author": { "name": "Kurt Buhler", diff --git a/plugins/fabric-cli/skills/fabric-cli/SKILL.md b/plugins/fabric-cli/skills/fabric-cli/SKILL.md index b3cf6e92..27740f7b 100644 --- a/plugins/fabric-cli/skills/fabric-cli/SKILL.md +++ b/plugins/fabric-cli/skills/fabric-cli/SKILL.md @@ -1,6 +1,6 @@ --- name: fabric-cli -version: 26.20 +version: 26.24 description: Expert guidance for using the Fabric CLI (`fab`) to fully interact with Fabric workspaces, items, and configuration. Automatically invoke this skill whenever the user mentions "Fabric" or "Power BI Service" or a "Fabric/Power BI workspace". --- diff --git a/plugins/pbi-desktop/.claude-plugin/plugin.json b/plugins/pbi-desktop/.claude-plugin/plugin.json index 65f29fbf..e06055f8 100644 --- a/plugins/pbi-desktop/.claude-plugin/plugin.json +++ b/plugins/pbi-desktop/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "pbi-desktop", - "version": "26.20", + "version": "26.24", "description": "Connect to Power BI Desktop's local Analysis Services instance via TOM and ADOMD.NET. No MCP server required.", "author": { "name": "Kurt Buhler", diff --git a/plugins/pbi-desktop/hooks/README.md b/plugins/pbi-desktop/hooks/README.md index 392f8e77..cc7328bb 100644 --- a/plugins/pbi-desktop/hooks/README.md +++ b/plugins/pbi-desktop/hooks/README.md @@ -6,11 +6,11 @@ PreToolUse and PostToolUse hooks that validate DAX references, enforce measure m | Hook | Event | Trigger (`if`) | Scope | |---|---|---|---| -| `validate-dax` | PreToolUse | `Bash(*tom_nuget*)` | DAX table/column/measure references in inline PowerShell commands | -| `validate-measure` | PreToolUse | `Bash(*Measures.Add*)` | Measure DisplayFolder, Description, FormatString when adding measures | -| `refresh-cache` | PostToolUse | `Bash(*AnalysisServices*)` | Auto-refresh model metadata cache on TOM connect or modification | -| `check-ri` | PostToolUse | `Bash(*model.SaveChanges*)` | Referential integrity; unmatched keys after relationship/column changes | -| `check-compat` | PostToolUse | `Bash(*AnalysisServices*)` | Compatibility level; lists features available by upgrading | +| `validate-dax` | PreToolUse | `Bash(*tom_nuget*)`, `Bash(* -File *.ps1*)` | DAX table/column/measure references in inline PowerShell commands and executed `.ps1` files | +| `validate-measure` | PreToolUse | `Bash(*Measures.Add*)`, `Bash(* -File *.ps1*)` | Measure DisplayFolder, Description, FormatString when adding measures | +| `refresh-cache` | PostToolUse | `Bash(* -File *.ps1*)` | Auto-refresh model metadata cache on TOM connect or modification | +| `check-ri` | PostToolUse | `Bash(*SaveChanges*)` | Referential integrity; unmatched keys after relationship/column changes | +| `check-compat` | PostToolUse | `Bash(* -File *.ps1*)` | Compatibility level; lists features available by upgrading | ## Checks @@ -56,7 +56,7 @@ That disables every hook in this plugin without touching individual check toggle ## Known limitations -- `.ps1` file execution hides content from `if` filters; hooks only validate inline PowerShell commands +- `if` glob patterns match only the raw command line, not `.ps1` file contents; the scripts compensate by reading executed `.ps1` files internally (`resolve_command_text`), gated by the `Bash(* -File *.ps1*)` triggers - `if` glob patterns are case-sensitive - UNC path conversion assumes `/Users//` prefix (macOS Parallels) @@ -65,5 +65,5 @@ That disables every hook in this plugin without touching individual check toggle ```bash echo '{"tool_name":"Bash","tool_input":{"command":"EVALUATE '"'"'FakeTable'"'"'[Col]"}}' | \ CLAUDE_PROJECT_DIR="$(pwd)" CLAUDE_PLUGIN_ROOT="$(pwd)/plugins/pbi-desktop" \ - bash plugins/pbi-desktop/hooks/run-pbi-hooks.sh validate-dax + bash plugins/pbi-desktop/hooks/pbi-hooks.sh validate-dax ``` diff --git a/plugins/pbi-desktop/hooks/pbi-hooks.sh b/plugins/pbi-desktop/hooks/pbi-hooks.sh index 5c679be5..c419edf4 100755 --- a/plugins/pbi-desktop/hooks/pbi-hooks.sh +++ b/plugins/pbi-desktop/hooks/pbi-hooks.sh @@ -83,17 +83,15 @@ extract_command() { echo "$STDIN_BUF" | jq -r '.tool_input.command // empty' 2>/dev/null } -resolve_command_text() { - # If the command is a -File .ps1 invocation, read the .ps1 file contents. - # Otherwise return the command text as-is. - # Handles UNC paths (\\Mac\Home\...) from Parallels by converting to macOS paths. +extract_ps1_path() { + # Extracts the .ps1 path from a -File .ps1 invocation. + # Outputs nothing if the command is not a -File invocation. local cmd="$1" local lower lower="$(echo "$cmd" | tr '[:upper:]' '[:lower:]')" if [[ "$lower" != *"-file"* ]]; then - echo "$cmd" - return + return 0 fi # Extract everything after -File, strip quotes and whitespace @@ -104,8 +102,32 @@ resolve_command_text() { after_file=$(printf '%s' "$after_file" | sed 's/^["]*//;s/["]*$//;s/^\\"//;s/\\"$//') # Trim to just the .ps1 path (stop at first .ps1) + printf '%s' "$after_file" | sed -n 's/\(.*\.ps1\).*/\1/p' +} + +is_bundled_connect_pbid_script() { + # Returns 0 if the command runs a .ps1 bundled with the connect-pbid skill. + # These scripts are exempt from content validation: load-tmdl.ps1 calls + # .Measures.Add without metadata by design, and doc strings like + # 'SUM(Sales[Amount])' would otherwise trip the DAX reference checks. + local cmd="$1" + local ps1_path + ps1_path="$(extract_ps1_path "$cmd")" + [[ -n "$ps1_path" ]] || return 1 + case "$ps1_path" in + *skills/connect-pbid/scripts*|*skills\\connect-pbid\\scripts*) return 0 ;; + esac + return 1 +} + +resolve_command_text() { + # If the command is a -File .ps1 invocation, read the .ps1 file contents. + # Otherwise return the command text as-is. + # Handles UNC paths (\\Mac\Home\...) from Parallels by converting to macOS paths. + local cmd="$1" + local ps1_path - ps1_path=$(printf '%s' "$after_file" | sed -n 's/\(.*\.ps1\).*/\1/p') + ps1_path="$(extract_ps1_path "$cmd")" if [[ -z "$ps1_path" ]]; then echo "$cmd" @@ -283,6 +305,9 @@ cmd_validate_dax() { raw_command="$(extract_command)" [[ -n "$raw_command" ]] || exit 0 + # Plugin-bundled connect-pbid scripts are exempt from content validation + is_bundled_connect_pbid_script "$raw_command" && exit 0 + local command_text command_text="$(resolve_command_text "$raw_command")" @@ -399,6 +424,9 @@ cmd_validate_measure() { raw_command="$(extract_command)" [[ -n "$raw_command" ]] || exit 0 + # Plugin-bundled connect-pbid scripts are exempt from content validation + is_bundled_connect_pbid_script "$raw_command" && exit 0 + local command_text command_text="$(resolve_command_text "$raw_command")" @@ -443,9 +471,12 @@ cmd_refresh_cache() { tool_name="$(extract_tool_name)" [[ "$tool_name" == "Bash" ]] || exit 0 + local raw_command + raw_command="$(extract_command)" + [[ -n "$raw_command" ]] || exit 0 + local command_text - command_text="$(extract_command)" - [[ -n "$command_text" ]] || exit 0 + command_text="$(resolve_command_text "$raw_command")" # Detect trigger local is_connect=false is_modification=false @@ -489,9 +520,12 @@ cmd_check_ri() { tool_name="$(extract_tool_name)" [[ "$tool_name" == "Bash" ]] || exit 0 + local raw_command + raw_command="$(extract_command)" + [[ -n "$raw_command" ]] || exit 0 + local command_text - command_text="$(extract_command)" - [[ -n "$command_text" ]] || exit 0 + command_text="$(resolve_command_text "$raw_command")" # Only run for relationship or column changes local relevant=false diff --git a/plugins/pbi-desktop/skills/connect-pbid/SKILL.md b/plugins/pbi-desktop/skills/connect-pbid/SKILL.md index 43bdcd4c..eb21101f 100644 --- a/plugins/pbi-desktop/skills/connect-pbid/SKILL.md +++ b/plugins/pbi-desktop/skills/connect-pbid/SKILL.md @@ -1,7 +1,7 @@ --- name: connect-pbid -version: 26.20 -description: TOM and ADOMD.NET guidance via PowerShell for connecting to Power BI Desktop's local Analysis Services instance. Covers model enumeration, DAX queries, metadata modification, annotations, calendar definitions, field parameters, query tracing, and DAX library package management (daxlib.org). Automatically invoke when the user mentions "Power BI Desktop", "Analysis Services port", "TOM", "ADOMD", "daxlib", "DAX library", "DAX UDF package", or asks to "connect to PBI Desktop", "query PBI Desktop with DAX", "modify PBI Desktop model", "add a measure to PBI", "capture visual queries", "create a field parameter", "validate DAX", "intercept DAX queries", "install daxlib", "add DAX SVG", "add IBCS". +version: 26.24 +description: TOM and ADOMD.NET guidance via PowerShell for connecting to Power BI Desktop's local Analysis Services instance. Covers model enumeration, DAX queries, metadata modification, annotations, calendar definitions, field parameters, query tracing, DAX library package management (daxlib.org), and the Desktop Bridge for reloading and screenshotting the report canvas. Automatically invoke when the user mentions "Power BI Desktop", "Analysis Services port", "TOM", "ADOMD", "daxlib", "DAX library", "DAX UDF package", or asks to "connect to PBI Desktop", "query PBI Desktop with DAX", "modify PBI Desktop model", "add a measure to PBI", "capture visual queries", "create a field parameter", "validate DAX", "intercept DAX queries", "install daxlib", "add DAX SVG", "add IBCS", "reload the report canvas", "screenshot a report page", "Desktop Bridge", or to work with the model and report in Power BI Desktop together. --- # Connect to Power BI Desktop (Local Analysis Services) @@ -19,12 +19,24 @@ Activate only when the Tabular Editor CLI or a Power BI MCP server is unavailabl **WARNING:** This skill does NOT support remote models via the XMLA endpoint. For Direct Lake models or models hosted in Fabric, use the Tabular Editor CLI or a Power BI MCP server instead; the local Analysis Services proxy does not expose Direct Lake databases to external TOM/ADOMD.NET connections. +## Model and report: routing + +Power BI Desktop exposes the model and the report as two separate local surfaces. This skill owns the model surface and report-canvas verification, and routes report authoring to the right skill: + +- **Model** (tables, columns, measures, relationships, roles, calculation groups, refresh): this skill, via TOM/ADOMD over the local Analysis Services instance. For model edits, prefer the `te` CLI or a model MCP when available; fall back to this skill's TOM when they are not (see "When to Use This Skill"). +- **Report-canvas verification** (reload after edits, screenshot pages): this skill, the raw Desktop Bridge named-pipe API (section 13). +- **Report authoring** (visuals, pages, formatting, filters, bookmarks, themes): the `pbir-cli` skill in the reports plugin (it drives the `pbir` CLI). The Desktop Bridge here only reloads and screenshots; it never edits visuals. Route every visual or page change to `pbir-cli`. +- **Report JSON edited directly** (only when `pbir` is unavailable): the `pbir-format` skill in the pbip plugin. + +Full loop on an open PBIP: change the model with TOM here, change visuals with `pbir-cli`, then reload and screenshot with the Desktop Bridge here to verify, and iterate. + ## Critical - Power BI Desktop must be open with a model loaded before connecting; if there are errors it is likely due to a "thin report" connected to a remote model, or a Direct Lake model (which uses a remote proxy that blocks external connections) - The local Analysis Services instance only accepts connections from `localhost` -- Multiple PBI Desktop files open means multiple `msmdsrv.exe` processes on different ports. Connect to each port, read `$server.Databases[0].Name`, and ask the user which model to work with if more than one is found +- Multiple PBI Desktop files open means multiple `msmdsrv.exe` processes on different ports. Connect to each port, read `$server.Databases[0].Name`, and ask the user which model to work with if more than one is found. When the `pbir` CLI is installed, prefer `pbir desktop list` to map each Desktop PID to the exact file it has open (see Section 2a) +- A workspace engine reporting `Databases: 0` belongs to a thin report (live connection to a remote model); there is no local model to connect to. Query thin reports through their remote model instead (`pbir model -q` routes there automatically) - Always use a timeout of 60000ms or higher for PowerShell commands via Bash - **Shell escaping**: Bash eats PowerShell `$` variables (`$env:TEMP`, `$server`, etc.) silently. Two options: (1) single-quote the `-Command` arg so Bash passes `$` literally to PowerShell; (2) write a `.ps1` file with a heredoc (single-quoted delimiter preserves `$`) and use `-File`. On macOS via Parallels, the `prlctl` -> `cmd.exe` -> `powershell.exe` chain adds extra escaping layers; `.ps1` files are more reliable for complex scripts but inline `-Command` with single quotes works for short commands. - **Always use `-ExecutionPolicy Bypass`** when running PowerShell commands or scripts. Windows blocks unsigned scripts by default. @@ -67,20 +79,29 @@ Packages install DLLs under `lib\net45\`. Load with `Add-Type -Path`. Find the port, load TOM, connect, enumerate -- in one script: ```powershell -# Find port +# Find ports (deduped; netstat lists IPv4 and IPv6 entries per port) $pids = (Get-Process msmdsrv -ErrorAction SilentlyContinue).Id $ports = netstat -ano | Select-String "LISTENING" | Where-Object { $pids -contains ($_ -split "\s+")[-1] } | - ForEach-Object { ($_ -split "\s+")[2] -replace ".*:" } + ForEach-Object { ($_ -split "\s+")[2] -replace ".*:" } | + Select-Object -Unique # Load TOM $basePath = "$env:TEMP\tom_nuget\Microsoft.AnalysisServices.retail.amd64\lib\net45" Add-Type -Path "$basePath\Microsoft.AnalysisServices.Core.dll" Add-Type -Path "$basePath\Microsoft.AnalysisServices.Tabular.dll" -# Connect to first port +# Connect to the first port that hosts a model; skip thin-report engines (0 databases) $server = New-Object Microsoft.AnalysisServices.Tabular.Server -$server.Connect("Data Source=localhost:$($ports[0])") +foreach ($p in $ports) { + $server.Connect("Data Source=localhost:$p") + if ($server.Databases.Count -eq 0) { + Write-Output "localhost:$p hosts no model (thin report); trying next port" + $server.Disconnect() + continue + } + break +} $model = $server.Databases[0].Model # Enumerate @@ -101,6 +122,22 @@ $server.Disconnect() | netstat | Any | `netstat -ano \| findstr LISTENING \| findstr ` | +## 2a. Correlating Ports to Reports (Multiple Instances) + +A port alone does not identify the report it serves; correlate before connecting to avoid modifying the wrong model. With the `pbir` CLI and Desktop's "external tool access" preview feature enabled, `pbir desktop list` shows each Desktop PID with the exact file it has open. Map ports to those PIDs through the process tree (each `msmdsrv.exe` is a child of its `PBIDesktop.exe`): + +```powershell +$conns = Get-NetTCPConnection -State Listen +foreach ($proc in Get-Process msmdsrv -ErrorAction SilentlyContinue) { + $port = ($conns | Where-Object OwningProcess -eq $proc.Id | Select-Object -First 1).LocalPort + $parent = (Get-WmiObject Win32_Process -Filter "ProcessId=$($proc.Id)").ParentProcessId + Write-Output "port $port -> msmdsrv $($proc.Id) -> PBIDesktop $parent" +} +``` + +An engine reporting `Databases: 0` is a thin report's workspace; no local model exists. Query the remote model instead (`pbir model -q` routes there automatically). + + ## 3. Loading TOM, Connecting, and Saving Changes ### Load Assemblies @@ -197,7 +234,7 @@ $conn.Open() All queries should preferably use `SUMMARIZECOLUMNS`. Check `dax.guide` online for information about DAX functions, if necessary. -**Important:** ADOMD.NET returns fully-qualified column names (e.g., `'Monsters'[Name]` not `Name`). Do not access columns by short name (`$reader["Name"]`) -- it fails silently and returns blank. Use `$reader.GetName($i)` to discover column names, then access by index: +**Important:** ADOMD.NET returns fully-qualified column names without quotes around the table name (e.g., `Brands[Brand Class]` not `Brand Class`; measure projections come back as `[@Alias]`). Do not access columns by short name (`$reader["Brand Class"]`) -- it fails silently and returns blank. Use `$reader.GetName($i)` to discover column names, then access by index: ```powershell $cmd = $conn.CreateCommand() @@ -426,9 +463,11 @@ foreach ($m in ($model.Tables | ForEach-Object { $_.Measures })) { ### Find the Open File Path -TOM does not expose the `.pbix`/`.pbip` file path directly. The most reliable method across all PBI Desktop install types is reading the `FileHistory` from `User.zip` in the PBI Desktop app data folder. +TOM does not expose the `.pbix`/`.pbip` file path directly. -**Primary method — FileHistory in User.zip (works for Store and non-Store):** +**Primary method — Desktop bridge:** `pbir desktop list` reports the exact file each running instance has open (requires the `pbir` CLI and the "external tool access" preview feature; see Section 2a). Use the methods below only when that is unavailable. + +**Fallback — FileHistory in User.zip (works for Store and non-Store):** ```powershell # Read the most recently opened file from PBI Desktop's settings @@ -487,17 +526,15 @@ For syntax, structure, and editing patterns for these files, load the relevant s - **`pbir-format`** -- `report.json`, `visual.json`, themes, filters, PBIR JSON schemas - **`tmdl`** -- TMDL syntax, measures, columns, roles, relationships -### No Hot-Reload — Close and Reopen Required - -**IMPORTANT:** Power BI Desktop does **not** watch for external file changes. If you edit metadata files on disk while the report is open, the changes will be silently ignored or overwritten when PBI Desktop next saves. +### Reloading External File Edits -To apply external file edits: +Power BI Desktop does **not** watch for external file changes; edits made on disk while a report is open are silently ignored or overwritten on the next Desktop save. To apply changes, in order of preference: -1. Close Power BI Desktop completely -2. Make your changes to the files on disk -3. Reopen the `.pbix` or `.pbip` file +1. **TOM modifications** (`$model.SaveChanges()`) apply to the running instance immediately. Prefer this for model metadata. +2. **PBIR report-definition edits** (pages, visuals) hot-reload into the open canvas with `pbir desktop refresh "Report.Report"` (PBIP/PBIR only, not `.pbix`; requires the preview feature). Theme JSON edits under StaticResources do NOT hot-reload; close and reopen instead. If the instance has unsaved changes, Desktop saves first and may overwrite the on-disk edit. +3. **Everything else** (TMDL edits on disk, theme files, `.pbix`): close Power BI Desktop, edit, reopen. -This is different from TOM modifications via `$model.SaveChanges()`, which apply immediately to the running instance without requiring a restart. +For **report** (PBIR) files specifically, the Desktop Bridge reloads on-disk edits into the open canvas without reopening (the `file.reload/v1` pipe method, with the `powerbi-desktop` npm CLI as a fallback); see section 13. Model (TMDL) on-disk edits still require close-and-reopen, or use live TOM `SaveChanges()` as above. ### Microsoft Documentation @@ -529,7 +566,7 @@ $job = Register-ObjectEvent -InputObject $trace -EventName "OnEvent" -MessageDat } ``` -**Always clear the VertiPaq cache** before debug queries; cached results prevent EVALUATEANDLOG from firing: +**Trace delivery is asynchronous**: `DAXEvaluationLog` events typically arrive 2-3.5 seconds after the query returns, so a short fixed sleep misses them. Poll the captured-event count (up to ~10s in 500ms steps) before reading results. Warm-cache runs still emit the event; do not rely on cache clearing to make it fire. Clear the VertiPaq cache only when cold-cache timings are needed: ```powershell $server.Execute('{ "clearCache": { "object": { "database": "' + $db.Name + '" } } }') | Out-Null @@ -571,6 +608,30 @@ Programmatic equivalent of DAX Studio's Server Timings. Subscribe to `QueryEnd`, For full setup, timing interpretation, sampling patterns, and PBIR-to-DAX translation, see [performance-profiling.md](./references/performance-profiling.md). +## 13. Working with the Report Canvas (Desktop Bridge) + +The TOM connection above drives the **model**: tables, measures, relationships, roles, refresh. It cannot touch the **report canvas** (pages and visuals). Power BI Desktop exposes a second, separate local API for that: the **Desktop Bridge**, a per-process JSON-RPC server on the Windows named pipe `\\.\pipe\pbi-desktop-bridge-`. Pair the two to change the model and immediately confirm the report re-renders. + +When the `pbir` CLI is installed, it wraps this same pipe; prefer it over driving the pipe raw: + +```powershell +pbir desktop list # PID + open file per instance +pbir model --% "Report.Report" -q "EVALUATE ROW(""Check"", [New Measure])" # engine-level check +pbir desktop refresh "Report.Report" # reload on-disk PBIR into the canvas +pbir desktop screenshot "Report.Report/Page Name.Page" -o verify.png # inspect rendering +``` + +The `--%` stop-parsing token prevents Windows PowerShell 5.1 from stripping the embedded quotes; omit it in bash or PowerShell 7+. + +Without `pbir`, drive the pipe raw from PowerShell, the same way this skill drives TOM/ADOMD. It requires the Desktop bridge **preview setting** enabled (File > Options and settings > Options > Preview features, then restart). Auto-discover the PID by enumerating the pipe directory; then over JSON-RPC: `application.state.get/v1` returns the open file path (`currentFilePath`, so the bridge can locate the PBIP on disk), `file.reload/v1` reloads the on-disk PBIR into the canvas, and `report.snapshot.capture/v1` returns a page PNG. + +Model-plus-report loop: edit the model with TOM and `$model.SaveChanges()` (applies live), then `reload` and `screenshot` the report to confirm visuals reflect the change (a renamed measure, a new format string, a repaired relationship). On-disk **report** (PBIR) edits are picked up by `reload`; on-disk **model** (TMDL) edits and theme files under StaticResources still need a reopen, so prefer live TOM for model changes. The bridge drives the Windows app, so on macOS run it inside the Parallels VM (see [parallels-macos.md](./references/parallels-macos.md)). + +For the full command set, PID selection, the JSON-RPC method surface (`bridge.manifest`, `application.state.get/v1`, `file.reload/v1`, `report.snapshot.capture/v1`), and how it complements the Analysis Services local API, see [desktop-bridge.md](./references/desktop-bridge.md). To CHANGE visuals, pages, formatting, filters, or bookmarks, route to the `pbir-cli` skill (reports plugin); the Desktop Bridge here only reloads and screenshots, it never edits the report. + +Alternative path (only if driving the raw pipe runs into trouble, framing, encoding, or a build that changed a param shape): fall back to the `powerbi-desktop` npm CLI, which wraps these same methods. See [desktop-bridge.md](./references/desktop-bridge.md). + + ## References **Skill references:** @@ -584,12 +645,14 @@ For full setup, timing interpretation, sampling patterns, and PBIR-to-DAX transl - [Performance Profiling](./references/performance-profiling.md) - DAX Server Timings via Trace API; FE/SE time split, cold/warm cache comparison, PBIR visual-to-DAX translation, trace event column compatibility - [Query Listener](./references/query-listener.md) - Capture live visual DAX queries via DMV polling; interpret query structure, timings, filter patterns - [Export Model](./references/export-model.md) - Export to BIM/TMDL via Tabular Editor CLI, fab CLI, or TOM serializer +- [Loading TMDL/BIM Files](./references/load-tmdl-files.md) - Load local TMDL folders or BIM files into TOM offline; inspect, modify, serialize back, deploy via fab CLI - [VertiPaq Statistics](./references/vertipaq-stats.md) - Column cardinality, dictionary/data size, memory by table, server timings via DMVs - [Refresh Model](./references/refresh-model.md) - All refresh methods (TMSL, TOM RequestRefresh, ADOMD.NET) - [macOS + Parallels Guide](./references/parallels-macos.md) - Connecting from macOS when PBI Desktop runs in a Parallels VM - [DAX Library Packages](./references/daxlib.md) - Installing reusable DAX UDF packages from daxlib.org; DaxLib.SVG, PowerofBI.IBCS, package structure, annotations +- [Desktop Bridge (report canvas)](./references/desktop-bridge.md) - Reload + screenshot the open report canvas over the raw named-pipe JSON-RPC API (PowerShell; `powerbi-desktop` npm CLI as fallback); pairing model (TOM) edits with report verification -**CLI tools in `bin/`:** +**CLI tools at the skill root:** - **`daxlib`** -- CLI for browsing, downloading, and installing DAX library packages from daxlib.org. Script at `daxlib.sh` (requires bash + jq). Model operations (add/update/remove) call `scripts/daxlib-tom/` via `dotnet run` (requires .NET 8 SDK). On macOS, model operations route through Parallels automatically. See [daxlib.md](./references/daxlib.md) for full command reference. @@ -606,6 +669,7 @@ For full setup, timing interpretation, sampling patterns, and PBIR-to-DAX transl - `modify-tom-objects.ps1` - Create table, rename measures, set folders/formats, hide columns, create relationship (with undo) - `create-field-parameter.ps1` - Create a field parameter table from a list of measures with all required metadata - `debug-dax.ps1` - Debug DAX with EVALUATEANDLOG trace capture and performance timings; auto-detects port, enumerates model measures, provides `Invoke-DebugQuery` helper +- `load-tmdl.ps1` - Load a local TMDL folder or BIM file into TOM offline (no running engine), enumerate the model, optionally add a measure and save back - `connect-from-mac.sh` - macOS wrapper that runs PowerShell scripts in a Parallels VM via `prlctl exec` **External references:** diff --git a/plugins/pbi-desktop/skills/connect-pbid/daxlib.sh b/plugins/pbi-desktop/skills/connect-pbid/daxlib.sh index 8d19730d..caf66609 100755 --- a/plugins/pbi-desktop/skills/connect-pbid/daxlib.sh +++ b/plugins/pbi-desktop/skills/connect-pbid/daxlib.sh @@ -12,6 +12,12 @@ set -euo pipefail +# jq is required for registry parsing; fail fast with a clear message. +# Skip the check for usage-only invocations. +if [[ $# -gt 0 && "$1" != "--help" && "$1" != "-h" ]]; then + command -v jq >/dev/null 2>&1 || { echo "Error: jq is required but not installed (https://jqlang.github.io/jq/)" >&2; exit 1; } +fi + # #region Constants @@ -793,6 +799,11 @@ parse_args() { exit 1 fi + if [[ "$1" == "--help" || "$1" == "-h" ]]; then + print_usage + exit 0 + fi + COMMAND="$1" shift diff --git a/plugins/pbi-desktop/skills/connect-pbid/references/desktop-bridge.md b/plugins/pbi-desktop/skills/connect-pbid/references/desktop-bridge.md new file mode 100644 index 00000000..fe845639 --- /dev/null +++ b/plugins/pbi-desktop/skills/connect-pbid/references/desktop-bridge.md @@ -0,0 +1,146 @@ +# Desktop Bridge: driving the report canvas over the raw named pipe + +Power BI Desktop exposes a second local API beside the Analysis Services engine: a +per-process JSON-RPC server on a Windows named pipe that controls the report canvas +(reload + snapshot). This skill rawdogs it the same way it rawdogs TOM/ADOMD: +PowerShell straight to the pipe, no wrapper CLI. (The `powerbi-desktop` npm CLI wraps +these same methods; the `pbir-format` and `pbir-cli` skills cover that wrapper path.) + +Use it together with the model API: change the model with TOM, then reload and +snapshot the report to confirm the visuals reflect the change. + +## The endpoint + +- Pipe: `\\.\pipe\pbi-desktop-bridge-`, one per Desktop process. Discover it by + enumerating the pipe directory for the prefix; `` is the PBIDesktop.exe id: + +```powershell +[System.IO.Directory]::GetFiles("\\.\pipe\") | + Where-Object { $_ -match 'pbi-desktop-bridge-(\d+)$' } +``` + + The pipe exists only when the bridge **preview setting is enabled**: in Power BI + Desktop, File > Options and settings > Options > Preview features, turn on the + developer-mode / report-bridge preview feature and restart Desktop. If no pipe is + found, that setting is off or Desktop is closed. +- Protocol: JSON-RPC 2.0 with LSP-style `Content-Length` framing (vscode-jsonrpc over + the pipe stream). No initialize handshake; connect and call. + +## Methods (params and returns, as the bridge defines them) + +```yaml +bridge.manifest: {} -> capability manifest (call first to confirm availability) +application.state.get/v1: {} -> { currentFilePath, hasUnsavedChanges } +file.reload/v1: { reloadModelDefinition: false } -> { success } # reloads on-disk PBIR into the live canvas +report.snapshot.capture/v1: { pageId, scale } -> { payload, encoding, pageId, pageDisplayName, mimeType } +``` + +- `pageId` is the PBIR section id (e.g. `ReportSection1a2b3c`), not the display name. +- `scale` is 1 to 3 (2 is a good default for readable review). +- `report.snapshot.capture/v1` returns the PNG as a base64 string in `payload` + (`encoding` is `base64`, `mimeType` is `image/png`). +- `file.reload/v1` with `reloadModelDefinition: false` reloads the report definition + only; this is the canvas refresh after a PBIR edit. + +## Raw PowerShell client + +```powershell +# 1. connect to the bridge pipe for the target Desktop process +$procId = (Get-Process PBIDesktop -ErrorAction Stop | Select-Object -First 1).Id +$pipe = New-Object System.IO.Pipes.NamedPipeClientStream(".", "pbi-desktop-bridge-$procId", + [System.IO.Pipes.PipeDirection]::InOut) +$pipe.Connect(5000) + +# 2. JSON-RPC over the pipe with LSP Content-Length framing +function Read-Frame($pipe) { + $win = ""; $one = New-Object byte[] 1; $hdr = New-Object Text.StringBuilder + while ($true) { + if ($pipe.Read($one, 0, 1) -le 0) { throw "pipe closed" } + $c = [char]$one[0]; [void]$hdr.Append($c) + $win = ($win + $c); if ($win.Length -gt 4) { $win = $win.Substring(1) } + if ($win -eq "`r`n`r`n") { break } + } + $len = [int]([regex]::Match($hdr.ToString(), 'Content-Length:\s*(\d+)').Groups[1].Value) + $buf = New-Object byte[] $len; $off = 0 + while ($off -lt $len) { $n = $pipe.Read($buf, $off, $len - $off); if ($n -le 0) { throw "eof" }; $off += $n } + [Text.Encoding]::UTF8.GetString($buf) | ConvertFrom-Json +} +function Invoke-Bridge($pipe, [int]$id, [string]$method, $params) { + $json = @{ jsonrpc = "2.0"; id = $id; method = $method; params = $params } | ConvertTo-Json -Compress -Depth 10 + $body = [Text.Encoding]::UTF8.GetBytes($json) + $header = [Text.Encoding]::ASCII.GetBytes("Content-Length: $($body.Length)`r`n`r`n") + $pipe.Write($header, 0, $header.Length); $pipe.Write($body, 0, $body.Length); $pipe.Flush() + Read-Frame $pipe +} + +# 3. calls +Invoke-Bridge $pipe 1 "bridge.manifest" @{} | Out-Null +$state = Invoke-Bridge $pipe 2 "application.state.get/v1" @{} # confirm $state.currentFilePath matches your PBIP +Invoke-Bridge $pipe 3 "file.reload/v1" @{ reloadModelDefinition = $false } # refresh the canvas after a PBIR edit +$shot = Invoke-Bridge $pipe 4 "report.snapshot.capture/v1" @{ pageId = "ReportSection1a2b3c"; scale = 2 } +[IO.File]::WriteAllBytes("page.png", [Convert]::FromBase64String($shot.payload)) +$pipe.Dispose() +``` + +The reader reads header bytes one at a time up to `\r\n\r\n`, then exactly +`Content-Length` body bytes; do not wrap the pipe in a buffering `StreamReader`, it +will swallow the next frame's bytes. + +## Model-and-report loop, rawdogged + +```powershell +# model edit via TOM (this skill's main body), applied live +$model.Tables["Sales"].Measures["Revenue"].FormatString = "\$#,0" +$model.SaveChanges() +# then confirm the report reflects it, no reopen +Invoke-Bridge $pipe 5 "file.reload/v1" @{ reloadModelDefinition = $false } +$shot = Invoke-Bridge $pipe 6 "report.snapshot.capture/v1" @{ pageId = "ReportSection1a2b3c"; scale = 2 } +[IO.File]::WriteAllBytes("sales.png", [Convert]::FromBase64String($shot.payload)) +``` + +On-disk PBIR (report) edits are picked up by `file.reload/v1`. On-disk TMDL (model) +edits are not; prefer live TOM `SaveChanges()` for model changes. + +## Locating the open PBIP from the bridge + +You do not need the file path in advance. Enumerate the pipe directory to auto-discover +the running Desktop PID, connect, and call `application.state.get/v1`; its +`currentFilePath` is the open `.pbip`/`.pbix` on disk. From it you have the project +folder and its `.Report` (PBIR) and `.SemanticModel` siblings, ready to drive with +`pbir` / the `pbir-format` skill. This is more reliable than the recent-file-history +method (section 10), which reads history rather than the live instance. + +```powershell +$procId = [System.IO.Directory]::GetFiles("\\.\pipe\") | + ForEach-Object { if ($_ -match 'pbi-desktop-bridge-(\d+)$') { $matches[1] } } | + Select-Object -First 1 +# connect to pbi-desktop-bridge-$procId (above), then: +$state = Invoke-Bridge $pipe 1 "application.state.get/v1" @{} +$pbip = $state.currentFilePath # e.g. C:\Reports\Sales\Sales.pbip +$reportDir = Join-Path (Split-Path $pbip) ((Split-Path $pbip -LeafBase) + ".Report") +``` + +## Fallback: the npm CLI + +If the raw pipe client misbehaves (framing, encoding, or a build that changed a param +shape), fall back to Microsoft's wrapper, which speaks the same methods over the same +pipe: `@microsoft/powerbi-desktop-bridge-cli` (the `powerbi-desktop` command, Node 20+; +install only with the user's go-ahead, and the same preview setting must be enabled). + +```bash +npm install -g @microsoft/powerbi-desktop-bridge-cli +powerbi-desktop status # list instances; pick the PID +powerbi-desktop reload --pid # wraps file.reload/v1 +powerbi-desktop screenshot --pid --output page.png # wraps report.snapshot.capture/v1 +``` + +Same endpoint, higher level. The `pbir-format` skill documents this wrapper path in full. + +## Notes + +- macOS: run inside the Parallels VM, the same path this skill uses for PowerShell; + see [parallels-macos.md](./parallels-macos.md). +- To CHANGE visuals, pages, or formatting, route to the `pbir-cli` skill; the bridge + only reloads and snapshots, it never edits the report. +- This is the named-pipe API distinct from the Analysis Services XMLA port this skill + connects to for the model; the two are independent. diff --git a/plugins/pbi-desktop/skills/connect-pbid/references/export-model.md b/plugins/pbi-desktop/skills/connect-pbid/references/export-model.md index 3b449c62..ab9306ba 100644 --- a/plugins/pbi-desktop/skills/connect-pbid/references/export-model.md +++ b/plugins/pbi-desktop/skills/connect-pbid/references/export-model.md @@ -46,17 +46,11 @@ Write-Output "Exported to $tmdlFolder" ## Via TOM — BIM (TMSL JSON) -TOM doesn't have a built-in BIM serializer. The closest approach is scripting via XMLA: +TOM has a built-in BIM serializer: `Microsoft.AnalysisServices.Tabular.JsonSerializer` (`scripts/load-tmdl.ps1` uses it for both load and save): ```powershell -# Use ScriptCreateOrReplace to generate TMSL, then save to file -# Simpler: just call Tabular Editor CLI against the port (see above) -# If you must use PowerShell only: -$json = [Newtonsoft.Json.JsonConvert]::SerializeObject( - $db.Model, - [Newtonsoft.Json.Formatting]::Indented -) -Set-Content -Path "C:\export\model.bim" -Value $json +$json = [Microsoft.AnalysisServices.Tabular.JsonSerializer]::SerializeDatabase($db) +[System.IO.File]::WriteAllText("C:\export\model.bim", $json) ``` -> This produces a partial JSON dump, not a fully valid BIM. Use TE CLI for reliable BIM output. +> Requires `Microsoft.AnalysisServices.Tabular.Json.dll` loaded alongside Core and Tabular (see SKILL.md Section 3). Deserialize with `[Microsoft.AnalysisServices.Tabular.JsonSerializer]::DeserializeDatabase($json)`. diff --git a/plugins/pbi-desktop/skills/connect-pbid/references/load-tmdl-files.md b/plugins/pbi-desktop/skills/connect-pbid/references/load-tmdl-files.md index ae8421eb..b3f76a98 100644 --- a/plugins/pbi-desktop/skills/connect-pbid/references/load-tmdl-files.md +++ b/plugins/pbi-desktop/skills/connect-pbid/references/load-tmdl-files.md @@ -140,15 +140,11 @@ $json = [Microsoft.AnalysisServices.Tabular.JsonSerializer]::SerializeDatabase($ ### Deploy the modified model -After modifying the in-memory model, deploy to a remote workspace: +After modifying the in-memory model, serialize it back to TMDL, then deploy the folder to a remote workspace via the fab CLI: ```powershell -# Option 1: save to TMDL, then use fab import [Microsoft.AnalysisServices.Tabular.TmdlSerializer]::SerializeDatabaseToFolder($db, $tmdlPath) fab import "WorkspaceName.Workspace/ModelName.SemanticModel" -i $tmdlPath -f - -# Option 2: deploy via fab CLI -fab import "WorkspaceName.Workspace/ModelName.SemanticModel" -i $tmdlPath -f ``` ## Key Differences: Live Connection vs Local Files diff --git a/plugins/pbi-desktop/skills/connect-pbid/scripts/connect-and-enumerate.ps1 b/plugins/pbi-desktop/skills/connect-pbid/scripts/connect-and-enumerate.ps1 index e6d2f38c..000c6613 100644 --- a/plugins/pbi-desktop/skills/connect-pbid/scripts/connect-and-enumerate.ps1 +++ b/plugins/pbi-desktop/skills/connect-pbid/scripts/connect-and-enumerate.ps1 @@ -42,8 +42,9 @@ if ($Port -gt 0) { $netstat = netstat -ano | Select-String "LISTENING" foreach ($line in $netstat) { $parts = ($line -split "\s+") | Where-Object { $_ -ne "" } - $pid = $parts[-1] - if ($pids -contains [int]$pid) { + # $pid is a read-only automatic variable in PowerShell; use another name + $ownerPid = $parts[-1] + if ($pids -contains [int]$ownerPid) { $portNum = ($parts[1] -split ":")[-1] if ($ports -notcontains $portNum) { $ports += $portNum diff --git a/plugins/pbi-desktop/skills/connect-pbid/scripts/create-field-parameter.ps1 b/plugins/pbi-desktop/skills/connect-pbid/scripts/create-field-parameter.ps1 index bd44fe22..a5877dd8 100644 --- a/plugins/pbi-desktop/skills/connect-pbid/scripts/create-field-parameter.ps1 +++ b/plugins/pbi-desktop/skills/connect-pbid/scripts/create-field-parameter.ps1 @@ -39,6 +39,8 @@ param( [string]$TableGroup = "05. Parameters" ) +$ErrorActionPreference = "Stop" + #region Prerequisites @@ -110,92 +112,97 @@ Write-Output $daxExpr #endregion -#region Create table - -$fpTable = New-Object Microsoft.AnalysisServices.Tabular.Table -$fpTable.Name = $Name - -$partition = New-Object Microsoft.AnalysisServices.Tabular.Partition -$partition.Name = $Name -$partition.Source = New-Object Microsoft.AnalysisServices.Tabular.CalculatedPartitionSource -$partition.Source.Expression = $daxExpr -$fpTable.Partitions.Add($partition) +#region Create and configure table -$model.Tables.Add($fpTable) -$model.SaveChanges() +try { + $fpTable = New-Object Microsoft.AnalysisServices.Tabular.Table + $fpTable.Name = $Name -Write-Output "Table created. Refreshing to populate columns..." + $partition = New-Object Microsoft.AnalysisServices.Tabular.Partition + $partition.Name = $Name + $partition.Source = New-Object Microsoft.AnalysisServices.Tabular.CalculatedPartitionSource + $partition.Source.Expression = $daxExpr + $fpTable.Partitions.Add($partition) -$model.Tables[$Name].RequestRefresh([Microsoft.AnalysisServices.Tabular.RefreshType]::Calculate) -$model.SaveChanges() + $model.Tables.Add($fpTable) + $model.SaveChanges() | Out-Null -#endregion + Write-Output "Table created. Refreshing to populate columns..." + $model.Tables[$Name].RequestRefresh([Microsoft.AnalysisServices.Tabular.RefreshType]::Calculate) + $model.SaveChanges() | Out-Null -#region Configure columns + # Re-fetch the table after refresh (columns are now auto-inferred) + $fpTable = $model.Tables[$Name] -# Re-fetch the table after refresh (columns are now auto-inferred) -$fpTable = $model.Tables[$Name] + # Identify the three auto-generated CalculatedTableColumns by SourceColumn + $nameCol = $fpTable.Columns | Where-Object { $_.SourceColumn -eq "[Value1]" } + $fieldsCol = $fpTable.Columns | Where-Object { $_.SourceColumn -eq "[Value2]" } + $orderCol = $fpTable.Columns | Where-Object { $_.SourceColumn -eq "[Value3]" } -# Identify the three auto-generated CalculatedTableColumns by SourceColumn -$nameCol = $fpTable.Columns | Where-Object { $_.SourceColumn -eq "[Value1]" } -$fieldsCol = $fpTable.Columns | Where-Object { $_.SourceColumn -eq "[Value2]" } -$orderCol = $fpTable.Columns | Where-Object { $_.SourceColumn -eq "[Value3]" } + if (-not $nameCol -or -not $fieldsCol -or -not $orderCol) { + throw "Could not identify auto-generated columns. Expected [Value1], [Value2], [Value3]." + } -if (-not $nameCol -or -not $fieldsCol -or -not $orderCol) { - Write-Error "Could not identify auto-generated columns. Expected [Value1], [Value2], [Value3]." + # Rename + $nameCol.Name = $Name + $fieldsCol.Name = "$Name Fields" + $orderCol.Name = "$Name Order" + + # Visibility + $fieldsCol.IsHidden = $true + $orderCol.IsHidden = $true + + # Sort name column by order column + $nameCol.SortByColumn = $orderCol + + # Sort fields column by order column + $fieldsCol.SortByColumn = $orderCol + + # Set groupBy relationship (name column grouped by fields column) + $gbc = New-Object Microsoft.AnalysisServices.Tabular.GroupByColumn + $gbc.GroupingColumn = $fieldsCol + $rcd = New-Object Microsoft.AnalysisServices.Tabular.RelatedColumnDetails + $rcd.GroupByColumns.Add($gbc) + $nameCol.RelatedColumnDetails = $rcd + + # Set ParameterMetadata extended property on fields column + $ep = New-Object Microsoft.AnalysisServices.Tabular.JsonExtendedProperty + $ep.Name = "ParameterMetadata" + $ep.Value = '{"version":3,"kind":2}' + $fieldsCol.ExtendedProperties.Add($ep) + + # Format order column + $orderCol.FormatString = "0" + + # Tabular Editor table group annotation + $fpTable.Annotations.Add( + (New-Object Microsoft.AnalysisServices.Tabular.Annotation -Property @{ + Name = "TabularEditor_TableGroup" + Value = $TableGroup + }) + ) + + $model.SaveChanges() | Out-Null +} +catch { + Write-Output "ERROR: $($_.Exception.Message)" + # Remove the half-created table so the model is left as found + $partial = $model.Tables[$Name] + if ($partial) { + $model.Tables.Remove($partial) + $model.SaveChanges() | Out-Null + Write-Output "Cleanup: removed partially created table [$Name]." + } $server.Disconnect() + Write-Output "Field parameter creation FAILED; no changes were kept." exit 1 } -# Rename -$nameCol.Name = $Name -$fieldsCol.Name = "$Name Fields" -$orderCol.Name = "$Name Order" - -# Visibility -$fieldsCol.IsHidden = $true -$orderCol.IsHidden = $true - -# Sort name column by order column -$nameCol.SortByColumn = $orderCol - -# Sort fields column by order column -$fieldsCol.SortByColumn = $orderCol - -# Set groupBy relationship (name column grouped by fields column) -$nameCol.RelatedColumnDetails = New-Object Microsoft.AnalysisServices.Tabular.RelatedColumnDetails -$nameCol.RelatedColumnDetails.GroupByColumn = $fieldsCol - -# Set ParameterMetadata extended property on fields column -$fieldsCol.SetExtendedProperty( - "ParameterMetadata", - '{"version":3,"kind":2}', - [Microsoft.AnalysisServices.Tabular.ExtendedPropertyType]::Json -) - -# Format order column -$orderCol.FormatString = "0" - #endregion -#region Annotations - -# Tabular Editor table group -$fpTable.Annotations.Add( - (New-Object Microsoft.AnalysisServices.Tabular.Annotation -Property @{ - Name = "TabularEditor_TableGroup" - Value = $TableGroup - }) -) - -#endregion - - -#region Save and report - -$model.SaveChanges() +#region Report Write-Output "" Write-Output "Field parameter [$Name] created with $($measureList.Count) measures:" diff --git a/plugins/pbi-desktop/skills/connect-pbid/scripts/debug-dax.ps1 b/plugins/pbi-desktop/skills/connect-pbid/scripts/debug-dax.ps1 index 1a2a6321..c0d77650 100644 --- a/plugins/pbi-desktop/skills/connect-pbid/scripts/debug-dax.ps1 +++ b/plugins/pbi-desktop/skills/connect-pbid/scripts/debug-dax.ps1 @@ -1,7 +1,8 @@ # Debug DAX expressions using EVALUATEANDLOG and the TOM Trace API. # Captures intermediate results and performance timings programmatically. # -# Usage: Modify $port and $queries, then run. +# Usage: Run with -Port (auto-detected if omitted). Add Invoke-DebugQuery +# calls in the "Run Queries" region, then rerun. # Requires TOM + ADOMD.NET NuGet packages (see connect-pbid skill Section 1). param( @@ -48,8 +49,8 @@ Write-Output "Connected to: $($db.Name) on port $Port" # DAX Evaluation Log trace $evalTrace = $server.Traces.Add("EvalDbg_" + [guid]::NewGuid().ToString("N").Substring(0,8)) $te = $evalTrace.Events.Add([Microsoft.AnalysisServices.TraceEventClass]::DAXEvaluationLog) -$te.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::TextData) -$te.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::EventClass) +$te.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::TextData) | Out-Null +$te.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::EventClass) | Out-Null $evalTrace.Update() $evalEvents = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) @@ -63,14 +64,14 @@ $perfTrace = $server.Traces.Add("PerfDbg_" + [guid]::NewGuid().ToString("N").Sub foreach ($ec in @([Microsoft.AnalysisServices.TraceEventClass]::QueryEnd, [Microsoft.AnalysisServices.TraceEventClass]::VertiPaqSEQueryEnd)) { $pte = $perfTrace.Events.Add($ec) - $pte.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::TextData) - $pte.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::EventClass) - $pte.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::Duration) - $pte.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::CpuTime) + $pte.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::TextData) | Out-Null + $pte.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::EventClass) | Out-Null + $pte.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::Duration) | Out-Null + $pte.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::CpuTime) | Out-Null } $pte2 = $perfTrace.Events.Add([Microsoft.AnalysisServices.TraceEventClass]::VertiPaqSEQueryCacheMatch) -$pte2.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::TextData) -$pte2.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::EventClass) +$pte2.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::TextData) | Out-Null +$pte2.Columns.Add([Microsoft.AnalysisServices.TraceColumn]::EventClass) | Out-Null $perfTrace.Update() $perfEvents = [System.Collections.ArrayList]::Synchronized((New-Object System.Collections.ArrayList)) @@ -126,7 +127,19 @@ function Invoke-DebugQuery { $sw.Stop() $conn.Close() - Start-Sleep -Milliseconds 800 + # Trace events arrive asynchronously; DAXEvaluationLog typically lags the + # query by 2-3.5s. Poll up to 10s instead of a fixed sleep. + $expectEval = $Query -match "EVALUATEANDLOG" + $deadline = [DateTime]::UtcNow.AddSeconds(10) + while ([DateTime]::UtcNow -lt $deadline) { + $evalReady = (-not $expectEval) -or ($evalEvents.Count -gt $esi) + $perfReady = $perfEvents.Count -gt $psi + if ($evalReady -and $perfReady) { + Start-Sleep -Milliseconds 300 # let trailing events in the same batch land + break + } + Start-Sleep -Milliseconds 500 + } # Performance $qe = @($perfEvents[$psi..($perfEvents.Count-1)] | Where-Object { $_.EventClass -eq "QueryEnd" }) diff --git a/plugins/pbi-desktop/skills/connect-pbid/scripts/load-tmdl.ps1 b/plugins/pbi-desktop/skills/connect-pbid/scripts/load-tmdl.ps1 index ef29f50c..10a3abe4 100644 --- a/plugins/pbi-desktop/skills/connect-pbid/scripts/load-tmdl.ps1 +++ b/plugins/pbi-desktop/skills/connect-pbid/scripts/load-tmdl.ps1 @@ -6,6 +6,7 @@ # powershell.exe -NoProfile -ExecutionPolicy Bypass -File load-tmdl.ps1 -Path "C:\MyModel\definition" # powershell.exe -NoProfile -ExecutionPolicy Bypass -File load-tmdl.ps1 -Path "C:\model.bim" # powershell.exe -NoProfile -ExecutionPolicy Bypass -File load-tmdl.ps1 -Path "C:\MyModel\definition" -AddMeasure "Sales:Test Measure:SUM(Sales[Amount])" +# powershell.exe -NoProfile -ExecutionPolicy Bypass -File load-tmdl.ps1 -Path "C:\MyModel\definition" -AddMeasure "Sales:Test Measure:SUM(Sales[Amount])" -Save param( [Parameter(Mandatory=$true)] @@ -171,5 +172,9 @@ if ($savePath -ne "") { Write-Output "Saved TMDL to: $savePath" } } +elseif ($AddMeasure -ne "") { + Write-Output "" + Write-Output "WARNING: -AddMeasure changed the in-memory model only; pass -Save or -SaveTo to persist." +} #endregion diff --git a/plugins/pbi-desktop/skills/connect-pbid/scripts/query-dax.ps1 b/plugins/pbi-desktop/skills/connect-pbid/scripts/query-dax.ps1 index 00d22073..35933cb9 100644 --- a/plugins/pbi-desktop/skills/connect-pbid/scripts/query-dax.ps1 +++ b/plugins/pbi-desktop/skills/connect-pbid/scripts/query-dax.ps1 @@ -64,8 +64,9 @@ try { $values = @() for ($i = 0; $i -lt $reader.FieldCount; $i++) { $val = $reader.GetValue($i) - if ($val -is [System.DBNull]) { $val = "(null)" } - $values += $val.ToString() + # Blank DAX results come back as CLR null (not just DBNull) + if ($null -eq $val -or $val -is [System.DBNull]) { $val = "(null)" } + $values += "$val" } Write-Output ($values -join "`t") $rowCount++ diff --git a/plugins/pbip/.claude-plugin/plugin.json b/plugins/pbip/.claude-plugin/plugin.json index d2059e76..62cc4a21 100644 --- a/plugins/pbip/.claude-plugin/plugin.json +++ b/plugins/pbip/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "pbip", - "version": "26.20", + "version": "26.24", "description": "Get this plugin if you'll work with pbip files, tmdl, or pbir. It helps with file structure and direct modifications.", "author": { "name": "Kurt Buhler", diff --git a/plugins/pbip/hooks/config.yaml b/plugins/pbip/hooks/config.yaml index 19a33f1e..27776725 100644 --- a/plugins/pbip/hooks/config.yaml +++ b/plugins/pbip/hooks/config.yaml @@ -22,6 +22,9 @@ required_fields: true # $schema URL format validation schema_url: true +# Enumerated property values (e.g. page displayOption) must match the schema enum +enum_values: true + # Visual/page name format (word chars and hyphens only) name_format: true diff --git a/plugins/pbip/hooks/hooks.json b/plugins/pbip/hooks/hooks.json index 316affb8..7d55eb6b 100644 --- a/plugins/pbip/hooks/hooks.json +++ b/plugins/pbip/hooks/hooks.json @@ -9,7 +9,7 @@ "type": "command", "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/validate-pbir.sh\"", "timeout": 10, - "if": "Edit(**.Report/**)" + "if": "Edit(**/*.Report/**)" }, { "type": "command", @@ -32,7 +32,7 @@ "type": "command", "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/validate-pbir.sh\"", "timeout": 10, - "if": "Write(**.Report/**)" + "if": "Write(**/*.Report/**)" }, { "type": "command", diff --git a/plugins/pbip/hooks/validate-pbir.sh b/plugins/pbip/hooks/validate-pbir.sh index 50dc9e03..365fe69b 100644 --- a/plugins/pbip/hooks/validate-pbir.sh +++ b/plugins/pbip/hooks/validate-pbir.sh @@ -169,7 +169,8 @@ validate_file() { (has("name") | tostring), (has("displayName") | tostring), (has("displayOption") | tostring), - (.name // "") + (.name // ""), + (.displayOption // "") ' "$FILE_PATH" 2>/dev/null) || return 0 VALS=() @@ -177,6 +178,7 @@ validate_file() { SCHEMA="${VALS[0]:-}" local HAS_NAME="${VALS[1]:-}" HAS_DISPLAY_NAME="${VALS[2]:-}" local HAS_DISPLAY_OPTION="${VALS[3]:-}" NAME="${VALS[4]:-}" + local DISPLAY_OPTION="${VALS[5]:-}" if check_enabled required_fields; then MISSING=() @@ -194,6 +196,23 @@ validate_file() { fi fi + # Valid displayOption values per the Microsoft PageDisplayOption schema. + if check_enabled enum_values && [[ -n "$DISPLAY_OPTION" ]]; then + case "$DISPLAY_OPTION" in + FitToPage|FitToWidth|ActualSize|DeprecatedDynamic|ActualSizeTopLeft) ;; + *) + echo "PBIR validation failed: $FILE_PATH" >&2 + echo "" >&2 + echo "Invalid displayOption value: '$DISPLAY_OPTION'" >&2 + echo "Valid: FitToPage, FitToWidth, ActualSize (DeprecatedDynamic and ActualSizeTopLeft are deprecated)." >&2 + echo "A 16:9 page ratio comes from height/width (e.g. 1080/1920), not displayOption." >&2 + echo "" >&2 + echo "$SKILL_TIP" >&2 + return 2 + ;; + esac + fi + if check_enabled schema_url && [[ -n "$SCHEMA" ]]; then if [[ ! "$SCHEMA" =~ ^https://developer\.microsoft\.com/json-schemas/fabric/item/report/definition/ ]]; then echo "PBIR validation failed: $FILE_PATH" >&2 diff --git a/plugins/pbip/skills/pbip/SKILL.md b/plugins/pbip/skills/pbip/SKILL.md index 7c0fda0c..a19c316f 100644 --- a/plugins/pbip/skills/pbip/SKILL.md +++ b/plugins/pbip/skills/pbip/SKILL.md @@ -1,6 +1,6 @@ --- name: pbip -version: 26.20 +version: 26.24 description: Expert guidance for the Power BI Project (PBIP) file format; project structure, cross-cutting operations (renames, forking), and PBIX extraction/conversion. Automatically invoke when the user mentions PBIP, PBIX, .pbip/.pbism/.platform files, or asks about "PBIP project structure", "PBIP vs PBIX", "thin report vs thick report", "rename a table", "cascade rename", "fork a PBIP project", "convert pbix to pbip", "extract pbix", "what files are in a PBIP", "PBIP encoding", "definition.pbir", or discusses project-level file structure and post-rename verification. --- diff --git a/plugins/pbip/skills/pbir-format/SKILL.md b/plugins/pbip/skills/pbir-format/SKILL.md index 88d681a6..27cf90cb 100644 --- a/plugins/pbip/skills/pbir-format/SKILL.md +++ b/plugins/pbip/skills/pbir-format/SKILL.md @@ -1,6 +1,6 @@ --- name: pbir-format -version: 26.20 +version: 26.24 description: Format reference for Power BI Enhanced Report (PBIR) JSON schemas and patterns. Automatically invoke when the user asks about PBIR JSON structure, visual.json properties, PBIR expressions, objects vs visualContainerObjects, theme inheritance, conditional formatting patterns, extension measures, bookmarks, field references, filter formatting, query roles, PBIR page structure, report wallpaper, or any PBIR metadata format question. --- @@ -29,7 +29,7 @@ Report.Report/ +-- .pbi/localSettings.json # Local-only, gitignored +-- .platform # Fabric Git integration +-- definition.pbir # Semantic model connection (byPath or byConnection) can open this file in Power BI Desktop to open the report -+-- mobileState.json # Mobile layout (niche) ++-- mobileState.json # PBIR-Legacy artifact only; current PBIR stores phone layout per-visual in mobile.json +-- semanticModelDiagramLayout.json # Model diagrams +-- CustomVisuals/ # Private custom visuals only +-- definition/ @@ -127,6 +127,13 @@ For detailed report design guidance (layout, spacing, visual hierarchy, color, a | Fix broken field references after model changes | **`references/how-to/fix-broken-field-references.md`** -- diagnosis, repair workflow for renamed/moved/removed fields, slicer value pitfalls | | Convert legacy report.json to PBIR format | **`references/how-to/convert-legacy-to-pbir.md`** -- format differences, step-by-step conversion, projections-to-queryState mapping, validation | | Understand reportExtensions.json schema | **`references/report-extensions.md`** -- file schema structure, entities, visual calculations; see `references/measures.md` for DAX authoring patterns | +| Set dynamic (measure-driven) alt text | **`references/visual-container-formatting.md`** -- altText as a Measure expression under visualContainerObjects.general | +| Add a dynamic text run to a textbox | **`references/textbox.md`** -- measure-bound textRuns; round-trip from Desktop required | +| Understand drill-down cross-filter behavior | **`references/visual-json.md`** -- drillFilterOtherVisuals vs visualInteractions | +| Register a custom visual (AppSource, org-store, private .pbiviz) | **`references/report.md`** -- publicCustomVisuals, organizationCustomVisuals, resourcePackages | +| Understand schema version coupling | **`references/schemas.md`** -- parent/embedded schema sets, copying fragments between reports | +| Set up mobile (phone) layout | **`references/pbir-structure.md`** -- mobile.json per-visual, coordinate space, git hygiene | +| Git hygiene for a PBIR project | **`references/pbir-structure.md`** -- what to track, ignore, and leave to Desktop | ## definition.pbir @@ -149,9 +156,10 @@ A report must be connected to a semantic model. There are two ways to do this: - **`examples/visuals/`** -- 54 standalone visual.json examples; see `examples/visuals/__index.md` for a catalog. Split into `default/` (minimal, theme-only) and `formatted/` (bespoke formatting, conditional formatting, gradients, filters) **Core references:** -- **`references/visual-json.md`** -- visual.json: expressions, field refs, query roles, position, objects vs vCO, selectors, sorting, filters -- **`references/pbir-structure.md`** -- PBIR folder structure details -- **`references/schemas.md`** -- Schema versions and URLs +- **`references/visual-json.md`** -- visual.json: expressions, field refs, query roles, position, objects vs vCO, selectors, sorting, filters, drill-down propagation +- **`references/desktop-bridge.md`** -- Verifying PBIR edits on the canvas via the Desktop Bridge CLI (`powerbi-desktop` reload + screenshot); preview setting; locating the open PBIP +- **`references/pbir-structure.md`** -- PBIR folder structure, mobile.json storage mechanics, git hygiene +- **`references/schemas.md`** -- Schema versions, URLs, and embedded schema coupling - **`references/enumerations.md`** -- Valid property enumerations - **`references/version-json.md`** -- version.json format (concise) - **`references/platform.md`** -- .platform file format (concise) @@ -160,14 +168,14 @@ A report must be connected to a semantic model. There are two ways to do this: **Formatting & expressions:** - **`references/theme.md`** -- Theme wildcards, inheritance, color system, filter pane styling, visual-type overrides. Includes jq patterns for inspecting and modifying theme JSON directly - **`references/schema-patterns/`** -- Expressions, selectors, conditional formatting, visual calculations -- **`references/visual-container-formatting.md`** -- objects vs visualContainerObjects deep-dive +- **`references/visual-container-formatting.md`** -- objects vs visualContainerObjects deep-dive; dynamic (measure-driven) altText - **`references/measures-vs-literals.md`** -- When to use measure expressions vs literal values - **`references/measures.md`** -- Extension measure patterns **Visual & page configuration:** -- **`references/textbox.md`** -- Textbox visual format +- **`references/textbox.md`** -- Textbox visual format; dynamic (measure-bound) text runs - **`references/page.md`** -- Page configuration and backgrounds -- **`references/report.md`** -- Report-level settings +- **`references/report.md`** -- Report-level settings; custom visual registration (AppSource, org-store, private .pbiviz) - **`references/wallpaper.md`** -- Report wallpaper/canvas background - **`references/filter-pane.md`** -- Filter pane formatting - **`references/sort-visuals.md`** -- Visual sort configuration diff --git a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/54698b9cd0a0c57906b7.bookmark.json b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/54698b9cd0a0c57906b7.bookmark.json index c8fe1e67..ec06cce2 100644 --- a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/54698b9cd0a0c57906b7.bookmark.json +++ b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/54698b9cd0a0c57906b7.bookmark.json @@ -7,7 +7,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/71abeb475381792b035d.bookmark.json b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/71abeb475381792b035d.bookmark.json index c9a2c7eb..2fd1682e 100644 --- a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/71abeb475381792b035d.bookmark.json +++ b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/71abeb475381792b035d.bookmark.json @@ -10,7 +10,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/82ce9a7f49847d03190a.bookmark.json b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/82ce9a7f49847d03190a.bookmark.json index 77d17a35..16a2b8a1 100644 --- a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/82ce9a7f49847d03190a.bookmark.json +++ b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/82ce9a7f49847d03190a.bookmark.json @@ -10,7 +10,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/8f5a4883bde0d3bd075d.bookmark.json b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/8f5a4883bde0d3bd075d.bookmark.json index b8e5edba..fc054158 100644 --- a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/8f5a4883bde0d3bd075d.bookmark.json +++ b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/8f5a4883bde0d3bd075d.bookmark.json @@ -10,7 +10,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/958f29ad733c047ee0b8.bookmark.json b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/958f29ad733c047ee0b8.bookmark.json index d63ba894..7dedcee6 100644 --- a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/958f29ad733c047ee0b8.bookmark.json +++ b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/958f29ad733c047ee0b8.bookmark.json @@ -7,7 +7,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/e5b8aa0b8e0565be9ce0.bookmark.json b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/e5b8aa0b8e0565be9ce0.bookmark.json index 032c5731..2e3c6abf 100644 --- a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/e5b8aa0b8e0565be9ce0.bookmark.json +++ b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/bookmarks/e5b8aa0b8e0565be9ce0.bookmark.json @@ -10,7 +10,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/pages/AfterReport.Page/visuals/LineChart_TopLeft.Visual/visual.json b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/pages/AfterReport.Page/visuals/LineChart_TopLeft.Visual/visual.json index 141df89f..594dafe3 100644 --- a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/pages/AfterReport.Page/visuals/LineChart_TopLeft.Visual/visual.json +++ b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/pages/AfterReport.Page/visuals/LineChart_TopLeft.Visual/visual.json @@ -61,7 +61,7 @@ "Property": "OTD % (Value; PY REPT)" } }, - "queryRef": "On-Time Delivery.OTD % (Value; PY)", + "queryRef": "On-Time Delivery.OTD % (Value; PY REPT)", "nativeQueryRef": "PY", "displayName": "PY" } @@ -107,7 +107,7 @@ } }, "selector": { - "metadata": "On-Time Delivery.OTD % (Value; PY)" + "metadata": "On-Time Delivery.OTD % (Value; PY REPT)" } } ] diff --git a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/pages/pages.json b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/pages/pages.json index 335c73f3..93e3adfa 100644 --- a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/pages/pages.json +++ b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/pages/pages.json @@ -1,18 +1,8 @@ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/report/definition/pagesMetadata/1.0.0/schema.json", "pageOrder": [ - "5d82d19c82c0d41db580", - "62e0e50a60b0575015eb", - "da2e63ebeb2179a994f1", - "0c32c81bce347402001e", - "1ce692e0ba0d1200c2ab", - "f11d192709500600d380", - "f1bf8244c8632606ac7d", - "57c5400a36b337093301", - "2dfd4022797c06129390", - "b1eca2da055007b1e7e5", - "b05b633226b8b89ed938", - "bf728418368097c42629" + "56bda0fdb3f32a7d1fe9", + "5b9de88f152744001bbe" ], - "activePageName": "da2e63ebeb2179a994f1" + "activePageName": "56bda0fdb3f32a7d1fe9" } diff --git a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/report.json b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/report.json index 854c1bc7..8c7743fd 100644 --- a/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/report.json +++ b/plugins/pbip/skills/pbir-format/examples/K201-MonthSlicer.Report/definition/report.json @@ -11,7 +11,7 @@ "type": "SharedResources" }, "customTheme": { - "name": "SqlbiDataGoblinTheme0699179381108107.json", + "name": "SqlbiDataGoblinTheme.json", "reportVersionAtImport": { "visual": "2.1.0", "report": "2.1.0", @@ -557,8 +557,8 @@ "type": "RegisteredResources", "items": [ { - "name": "SqlbiDataGoblinTheme0699179381108107.json", - "path": "SqlbiDataGoblinTheme0699179381108107.json", + "name": "SqlbiDataGoblinTheme.json", + "path": "SqlbiDataGoblinTheme.json", "type": "CustomTheme" } ] diff --git a/plugins/pbip/skills/pbir-format/references/desktop-bridge.md b/plugins/pbip/skills/pbir-format/references/desktop-bridge.md new file mode 100644 index 00000000..746dae15 --- /dev/null +++ b/plugins/pbip/skills/pbir-format/references/desktop-bridge.md @@ -0,0 +1,50 @@ +# Verifying PBIR edits with the Desktop Bridge + +PBIR JSON can be schema-valid yet render wrong in Power BI Desktop. To see on-disk PBIR +edits on the canvas without reopening the file, drive an open Desktop instance with +Microsoft's Desktop Bridge CLI; it reloads the report and screenshots pages. + +Two npm packages (Node 20+; install only with the user's go-ahead): + +```yaml +"@microsoft/powerbi-desktop-bridge-cli": provides `powerbi-desktop` (open, status, reload, screenshot) +"@microsoft/powerbi-report-authoring-cli": provides `powerbi-report-author` (PBIR validate/edit); optional +``` + +The bridge needs a **preview setting** enabled: in Power BI Desktop, File > Options and +settings > Options > Preview features, turn on the developer-mode / report-bridge +feature, then restart Desktop. + +## The loop + +```bash +npm install -g @microsoft/powerbi-desktop-bridge-cli +powerbi-desktop open "Sales.pbip" # start Desktop on the project (or it is already open) +powerbi-desktop status # list instances; pick the target PID +powerbi-desktop reload --pid # re-read the on-disk PBIR into the live canvas +powerbi-desktop screenshot --pid --output shots/page.png +``` + +Edit PBIR -> `reload --pid` -> `screenshot --pid` -> review the PNG -> fix and repeat. + +- Select by PID from `status`, never by report path; the same project can be open in + several Desktop processes. +- `` is the PBIR section id (for example `ReportSection1a2b3c`), not the page + display name. Read it from the page folder name. +- Screenshots default to scale 2; pass `--scale 1` for smaller, `--scale 3` for detail. + +## Useful side effects + +- `status` reports each instance's `currentFilePath`, so the bridge can tell you where + the open PBIP folder is on disk (auto-discovered PID -> file path), handy when you do + not already know the project location. +- `reload` covers report-definition (PBIR) changes. For semantic-model / TMDL changes, + reopen the PBIP if the model change is not reflected. + +## Notes + +- It drives the Windows Desktop app, so on macOS run it through Parallels. +- For the raw named-pipe JSON-RPC behind this CLI (PowerShell, no npm wrapper), see the + `connect-pbid` skill's Desktop Bridge reference. +- To CHANGE visuals or pages, use the `pbir-cli` skill; the bridge only reloads and + screenshots. diff --git a/plugins/pbip/skills/pbir-format/references/pbir-structure.md b/plugins/pbip/skills/pbir-format/references/pbir-structure.md index cb9b3db5..3cb68780 100644 --- a/plugins/pbip/skills/pbir-format/references/pbir-structure.md +++ b/plugins/pbip/skills/pbir-format/references/pbir-structure.md @@ -19,7 +19,7 @@ Report.Report/ | +-- localSettings.json # Local editor settings (gitignored) +-- .platform # Fabric Git integration metadata +-- definition.pbir # Semantic model connection (byPath or byConnection) -+-- mobileState.json # OPTIONAL -- generated by PBI Desktop for mobile layout; no external editing ++-- mobileState.json # LEGACY PBIR-Legacy name only; current PBIR stores phone layout per-visual (see mobile.json below) +-- semanticModelDiagramLayout.json # OPTIONAL -- generated by PBI Desktop for model diagram positions; no external editing +-- CustomVisuals/ # Private custom visuals (optional) +-- definition/ @@ -33,7 +33,7 @@ Report.Report/ | | +-- visuals/ | | +-- [VisualName]/ # Recommend letters, digits, underscores, hyphens only | | +-- visual.json # Visual definition -| | +-- mobile.json # Mobile layout (optional) +| | +-- mobile.json # Phone layout for this visual (optional; validated by visualContainerMobileState schema) | +-- bookmarks/ # OPTIONAL | +-- bookmarks.json # Bookmark order and groups | +-- [id].bookmark.json # Individual bookmark state @@ -259,12 +259,59 @@ definition/pages/ ``` **Rules:** -- Folder names can be changed freely; spaces are allowed (Power BI Desktop creates them), but prefer letters, digits, underscores, hyphens for human-authored names +- Folder names can be changed freely; spaces are valid PBIR (Power BI Desktop creates them), but the pbir CLI's path syntax handles them awkwardly and `pbir validate` emits an INVALID_PAGE_FOLDER warning; rename to letters, digits, underscores, hyphens when practical - JSON filenames (`visual.json`, `page.json`) must remain unchanged - The `name` property inside JSON files must remain unchanged - Power BI Desktop preserves renamed folders on save - Restart Power BI Desktop after renaming +## Mobile Layout Storage + +The phone layout is stored per-visual, not per-page. Each visual folder may contain a `mobile.json` beside its `visual.json`, validated by the `visualContainerMobileState` schema (current 2.4.0). "The page has a phone layout" is emergent: it is true if and only if at least one visual has a `mobile.json`. + +**Correction:** `mobileState.json` at the report root was the old PBIR-Legacy name; current PBIR has no page-level mobile file. The per-visual `mobile.json` is a normal, hand-editable, diffable JSON file. + +Required keys: `$schema` and `position`. `position` reuses the same field names as desktop (`x, y, z, height, width, tabOrder, angle`) but in the phone-canvas coordinate space, independent of the desktop position. Optional `objects`/`visualContainerObjects` allow mobile-only formatting overrides (deltas only; omit properties that should inherit from the desktop visual). + +```json +{ + "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/report/definition/visualContainerMobileState/2.4.0/schema.json", + "position": {"x": 0, "y": 0, "z": 0, "width": 360, "height": 200, "tabOrder": 0} +} +``` + +Pitfalls: +- Editing `visual.json` position has no effect on the phone layout and vice versa; they are independent records +- Deleting a visual's folder must also drop its `mobile.json` +- Copy the `$schema` URL from an existing `mobile.json` in the same report; a stale schema version is the most common validation failure +- `mobile.json` is in scope for the `validate-pbir.sh` hook + +## Git Hygiene for a PBIR Project + +Initialize the repo at the PBIP-folder root (the directory holding the `.pbip` file) so `.pbip`, `.Report/`, and `.SemanticModel/` are tracked together. + +`.gitignore` essentials: +```gitignore +.pbi/localSettings.json # per-user, per-machine state +.cache/ +``` + +Files that are plain text but should be left for Desktop to manage (do not hand-edit): +- `semanticModelDiagramLayout.json` +- Registration entries in `CustomVisuals/` for private visuals +- (legacy only) `mobileState.json` + +Files that diff cleanly and are the substance of review: +- Each `page.json` and `visual.json` +- `report.json`, `version.json`, `reportExtensions.json` +- Bookmark files, `StaticResources/` themes +- `mobile.json` per visual + +Pitfalls: +- A binary `.pbix` in the same folder defeats source control; keep the project as PBIP/PBIR +- `.platform` is a Fabric Git-integration system file; leave it tracked, do not edit it +- Verify the `.pbi/` ignore pattern actually matches the path (easy to mistype as `pbi/` without the leading dot) + ## Best Practices 1. **Version Control** -- commit PBIR directories to Git; use meaningful visual/page folder names diff --git a/plugins/pbip/skills/pbir-format/references/report.md b/plugins/pbip/skills/pbir-format/references/report.md index f0b148b0..98cb639a 100644 --- a/plugins/pbip/skills/pbir-format/references/report.md +++ b/plugins/pbip/skills/pbir-format/references/report.md @@ -72,6 +72,33 @@ Registers themes, images, and other static resources. Every custom theme and ima Item types: `"BaseTheme"`, `"CustomTheme"`, `"Image"`. See [images.md](./images.md) for image usage. +### Custom Visual Registration + +A `visualType` set to a custom visual's GUID is inert on its own. Rendering requires a matching registration elsewhere AND the visual being installed/approved in the consuming environment. + +Three registration paths in `report.json` (schema `report/3.2.0`): + +```yaml +publicCustomVisuals: + - array of GUID strings (AppSource visuals; code is NOT in the report; resolved at open time) +organizationCustomVisuals: + - array of {name, path, disabled?} (org-store-approved references; admin-managed) +resourcePackages: + - CustomVisualJavascript/Css/Screenshot items (private .pbiviz only; JS/CSS ship inside the report) +CustomVisuals/ folder: + - metadata for private .pbiviz only; absent for AppSource/org-store visuals +``` + +**AppSource visual (hand-adding):** Set `visualType` to the GUID and append the same GUID to `publicCustomVisuals`. Without the `publicCustomVisuals` entry, the visual renders blank with no validation error. + +**Private `.pbiviz`:** Self-contained; JS/CSS ship via `resourcePackages` + `CustomVisuals/`. You cannot synthesize a private visual by hand-writing `resourcePackages`. Import once in Desktop, let it serialize the payload, then copy and edit. + +Pitfalls: +- `visualType` GUID present but missing from `publicCustomVisuals` is the most common silent failure when copying a custom visual between reports +- Copying a private-`.pbiviz` visual requires copying its `resourcePackages`/`CustomVisuals` payload too +- `disabled: true` on an `organizationCustomVisuals` entry means the admin pulled the visual from the org store +- AppSource/org-store visuals are unavailable in Power BI Report Server; private `.pbiviz` must be used there instead + ### settings Report-wide behavioral settings. Values are bare (not wrapped in expr). diff --git a/plugins/pbip/skills/pbir-format/references/schema-patterns/conditional-formatting.md b/plugins/pbip/skills/pbir-format/references/schema-patterns/conditional-formatting.md index 8d3692f8..9ed3ff34 100644 --- a/plugins/pbip/skills/pbir-format/references/schema-patterns/conditional-formatting.md +++ b/plugins/pbip/skills/pbir-format/references/schema-patterns/conditional-formatting.md @@ -115,7 +115,7 @@ RETURN } ``` -**Theme color names:** `"bad"`, `"good"`, `"neutral"`, `"minColor"`, `"midColor"`, `"maxColor"` +**Theme color names:** `"bad"`, `"good"`, `"neutral"`, `"minColor"`, `"maxColor"` (note: `"midColor"` is NOT a valid token; use `"neutral"` for a middle state) ### Two-Color Gradient (linearGradient2) diff --git a/plugins/pbip/skills/pbir-format/references/schemas.md b/plugins/pbip/skills/pbir-format/references/schemas.md index df5d94f3..48eb4f75 100644 --- a/plugins/pbip/skills/pbir-format/references/schemas.md +++ b/plugins/pbip/skills/pbir-format/references/schemas.md @@ -23,9 +23,9 @@ https://developer.microsoft.com/json-schemas/fabric/item/report/{type}/{version} - PBIP project: `.../fabric/pbip/pbipProperties/1.0.0/schema.json` - Semantic model: `.../fabric/item/semanticModel/{type}/{version}/schema.json` -## K201 Example Schema Versions (late 2025) +## K201 Example Schema Versions -Versions below match the K201 example project bundled with this skill. As of early 2026, newer versions exist (e.g., `visualContainer/2.7.0`, `report/3.2.0`, `page/2.1.0`, `bookmark/2.1.0`). Microsoft updates schemas roughly monthly — **always use the `$schema` URL from your existing project files** rather than assuming these versions. +Versions below match the K201 example project bundled with this skill. As of mid 2026, newer versions exist (e.g., `visualContainer/2.7.0`, `report/3.3.0`, `page/2.1.0`, `bookmark/2.1.0`). Microsoft updates schemas roughly monthly — **always use the `$schema` URL from your existing project files** rather than assuming these versions. | Schema Type | Version | File | |-------------|---------|------| @@ -89,6 +89,26 @@ Common `expr` wrapper types (not exhaustive — the full schema defines 48+ type Selectors can be combined: `metadata` + `data` + `id` + `order` on the same selector object. +## Schema Coupling: Versions Move as a Set + +Top-level schemas embed references to sub-schemas at fixed versions, and those versions advance together. Copying a fragment from a newer report into an older one can pass a naive JSON check yet fail at load time or silently drop properties. + +Example couplings (verify against the CHANGELOG; these advance over time): +- `report/3.2.0` embeds `filterConfiguration/1.3.0` + `formattingObjectDefinitions/1.5.0` +- `report/2.1.0` embeds `filterConfiguration/1.2.0` + `formattingObjectDefinitions/1.4.0` +- `page/2.1.0` embeds `filterConfiguration/1.3.0` + `formattingObjectDefinitions/1.5.0` + `semanticQuery/1.4.0` + +Treat a report's schema versions as one matched set. When copying a `visual.json` or filter fragment between reports, confirm compatible parent versions or regenerate in the target project. Check the CHANGELOG files for exact coupling: +```bash +gh api repos/microsoft/json-schemas/contents/fabric/item/report/definition/report --jq '.[].name' +``` + +Upgrade the whole matched set in one commit, then validate. + +Pitfalls: +- `reportVersionAtImport` in theme metadata is engine-managed; it is not the report's schema version and must not be edited to "match" +- Schemas advance roughly monthly; pin tooling to the project's own `$schema` URLs, not a remembered version number + ## Schema Exploration ```bash diff --git a/plugins/pbip/skills/pbir-format/references/textbox.md b/plugins/pbip/skills/pbir-format/references/textbox.md index 6d67a56a..56af58c5 100644 --- a/plugins/pbip/skills/pbir-format/references/textbox.md +++ b/plugins/pbip/skills/pbir-format/references/textbox.md @@ -358,6 +358,48 @@ For page titles, typical positioning: } ``` +## Dynamic Text Runs (Measure-Bound Values) + +A text run can bind to a measure instead of a static literal, so prose like "Revenue is 12.3M, up 8% on plan" updates with filter context. This is the UI's `fx` (Values) button on a textbox. + +The existing `textRuns[].value` string is replaced with an expression binding. The `textbox` content object is NOT fully defined in the published `visualContainer` schema (no `textRun`/`paragraph` definition), so `pbir validate` passes a malformed dynamic run without complaint. Correctness is on you. + +**Critical workflow:** Because no Microsoft schema or official sample defines the exact JSON, author one dynamic value in Desktop, save, then read the resulting `visual.json` for the precise shape, and use that as your template. Do not invent the structure. + +A paragraph with a mixed static + dynamic run looks like: +```json +"paragraphs": [ + { + "textRuns": [ + { + "value": "Revenue is ", + "textStyle": {"fontFamily": "Segoe UI", "fontSize": "14pt"} + }, + { + "textStyle": {"fontFamily": "Segoe UI", "fontSize": "14pt", "fontWeight": "bold"}, + "valueExpr": { + "expr": { + "Measure": { + "Expression": {"SourceRef": {"Entity": "Sales"}}, + "Property": "Revenue Formatted" + } + } + } + } + ] + } +] +``` + +Note: the exact key name and shape for the bound run may differ from the static `"value"` key; capture from Desktop, do not guess. + +Pitfalls: +- Each run is a separate entry in `textRuns`; mixing literal and bound runs in one paragraph is the expected pattern +- A bound run respects page/visual filters; an "all-data" baseline must use `CALCULATE(..., ALL(...))` inside the measure +- Format inheritance: the run inherits the measure's dynamic format string; push currency/unit/decimal formatting into the measure rather than the run's display properties +- `pbir validate` does not catch a malformed dynamic run; round-trip through Desktop to confirm +- For conditionally-styled clauses inside one sentence (e.g., a clause that turns red on a miss), an SVG narrative measure is the right tool; the Smart Narrative visual is non-deterministic and cannot be diffed + ## Drill Filtering Textboxes should not filter other visuals: diff --git a/plugins/pbip/skills/pbir-format/references/visual-container-formatting.md b/plugins/pbip/skills/pbir-format/references/visual-container-formatting.md index 5c858fc0..c74d7902 100644 --- a/plugins/pbip/skills/pbir-format/references/visual-container-formatting.md +++ b/plugins/pbip/skills/pbir-format/references/visual-container-formatting.md @@ -151,16 +151,43 @@ Both are found in real reports. When editing an older report, container properti ### Accessible visual (with altText) +`altText` lives at `visualContainerObjects.general[].properties.altText`, NOT inside `objects`. The value is an `expr`, so it can be either a static literal or a dynamic measure reference. + +**Static (literal):** ```json "visualContainerObjects": { "general": [{ "properties": { - "altText": {"expr": {"Literal": {"Value": "'Bar chart showing revenue by region, Q4 2024'"}}} + "altText": {"expr": {"Literal": {"Value": "'Revenue by region, current fiscal year'"}}} } }] } ``` +**Dynamic (preferred for filtered visuals):** Author a `_Report` extension measure returning a readable sentence, then bind via a `Measure` expression. The measure re-reads when filter context changes, so the description stays accurate. +```json +"visualContainerObjects": { + "general": [{ + "properties": { + "altText": { + "expr": { + "Measure": { + "Expression": {"SourceRef": {"Schema": "extension", "Entity": "_Report"}}, + "Property": "Alt revenue by region" + } + } + } + } + }] +} +``` + +Pitfalls: +- Placing `altText` under `objects.general` instead of `visualContainerObjects.general` is valid JSON but the screen reader will not pick it up +- A measure that can return BLANK must guard: `IF(COUNTROWS(...) > 0, ..., "No data for current selection.")` +- Do not duplicate the title text; the screen reader already speaks title + visual type before alt text +- Decorative shapes/images should have no alt text and should be removed from tab order (`tabOrder: -1`) + ## Theme Interaction Container formatting is heavily influenced by the theme. The theme's `visualStyles["*"]["*"]` section sets defaults for all container properties across all visuals. Visual-type exceptions (like `visualStyles["textbox"]["*"]`) override those defaults for specific types. diff --git a/plugins/pbip/skills/pbir-format/references/visual-json.md b/plugins/pbip/skills/pbir-format/references/visual-json.md index f7d23df8..0419e17b 100644 --- a/plugins/pbip/skills/pbir-format/references/visual-json.md +++ b/plugins/pbip/skills/pbir-format/references/visual-json.md @@ -262,6 +262,27 @@ Types: `"NoFilter"` (disable cross-filter), `"Filter"` (cross-filter), `"Highlig Only interactions that deviate from the default need to be listed. By default, all visuals cross-filter each other. +## Drill-Down Propagation + +`drillFilterOtherVisuals` is a boolean on the visual's `visual` object (sibling to `visualType`). It controls whether drilling into a hierarchy re-filters other visuals on the page. + +```json +"visual": { + "visualType": "barChart", + "drillFilterOtherVisuals": true, + ... +} +``` + +Two behaviors that are easy to conflate: + +- `drillFilterOtherVisuals: true` -- drilling a hierarchy level re-filters the rest of the page, behaving like a data-point click; `false` isolates the drill to that visual. Desktop writes this flag explicitly per visual, so read the value on the visual you are editing rather than assuming a global default +- `visualInteractions` (page.json) -- controls on-click cross-filter mode (NoFilter/Filter/Highlight). Both settings must align; a `true` drill flag still respects any `NoFilter` interaction pairs for that visual + +Do not confuse `drillFilterOtherVisuals` (same-page hierarchy walk) with drillthrough (navigates to a separate page via `visualLink.type: "Drillthrough"`). + +Cross-filter also carries the source visual's `filterConfig` to target visuals for the duration of the selection. If an unwanted filter travels during cross-filter, the fix is either a `NoFilter` pair in `visualInteractions` or moving the filter to page level. + ## Table/Matrix Column Widths Column widths in tables and matrices are set via the `columnWidth` object with a `metadata` selector targeting the specific column: @@ -572,14 +593,14 @@ Can also include `dataBars` and `fontColor` per column. Each entry targets one c ## Small Multiples -Many chart types support small multiples -- a grid of the same chart broken out by a dimension. Configured via the `Series` query role and `smallMultiplesLayout` in `objects`: +Many chart types support small multiples -- a grid of the same chart broken out by a dimension. True small multiples use the dedicated `SmallMultiples` query role (not `Series`). `Series`/`Legend` overlays series in a single frame; `SmallMultiples` partitions into a grid. The `smallMultiplesLayout` object in `objects` only takes effect when the `SmallMultiples` role is populated. ```json "query": { "queryState": { "Category": {"projections": [...]}, "Y": {"projections": [...]}, - "Series": {"projections": [{"field": {"Column": {"Expression": {"SourceRef": {"Entity": "Products"}}, "Property": "Category"}}}]} + "SmallMultiples": {"projections": [{"field": {"Column": {"Expression": {"SourceRef": {"Entity": "Products"}}, "Property": "Category"}}}]} } } ``` @@ -595,6 +616,8 @@ Many chart types support small multiples -- a grid of the same chart broken out Supported on: lineChart, areaChart, barChart, columnChart, comboChart, and their stacked/100% variants. +Features that are inert once a visual is trellised (do not add analytics overlays expecting them to work): total labels for stacked charts, trend lines, forecasting, zoom sliders, line high-density sampling, concatenate axis labels, hierarchical axis, scroll-to-load-more. + ## Related - [visual-container-formatting.md](./visual-container-formatting.md) -- objects vs visualContainerObjects diff --git a/plugins/pbip/skills/tmdl/SKILL.md b/plugins/pbip/skills/tmdl/SKILL.md index c1b82489..aa6855d7 100644 --- a/plugins/pbip/skills/tmdl/SKILL.md +++ b/plugins/pbip/skills/tmdl/SKILL.md @@ -1,6 +1,6 @@ --- name: tmdl -version: 26.20 +version: 26.24 description: Direct TMDL file authoring and BIM-to-TMDL conversion for semantic models in PBIP projects. Automatically invoke when the user asks to "edit TMDL", "add a measure in TMDL", "TMDL syntax", "fix formatString", "fix summarizeBy", "TMDL indentation", "convert BIM to TMDL", "add a column description", "create a calculated column in TMDL", or mentions .tmdl file editing or BIM-to-TMDL migration. --- diff --git a/plugins/reports/.claude-plugin/plugin.json b/plugins/reports/.claude-plugin/plugin.json index 2d608c18..e564c02f 100644 --- a/plugins/reports/.claude-plugin/plugin.json +++ b/plugins/reports/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "reports", - "version": "26.20", + "version": "26.24", "description": "Get this plugin for agentic development and management of reports.", "author": { "name": "Kurt Buhler", diff --git a/plugins/reports/skills/create-pbi-report/SKILL.md b/plugins/reports/skills/create-pbi-report/SKILL.md index 18810aa5..ec8091ee 100644 --- a/plugins/reports/skills/create-pbi-report/SKILL.md +++ b/plugins/reports/skills/create-pbi-report/SKILL.md @@ -1,6 +1,6 @@ --- name: create-pbi-report -version: 26.20 +version: 26.24 description: Step-by-step workflow for creating complete Power BI reports from scratch using pbir CLI. Covers model discovery, report creation, page layout, theme setup, visual placement, field binding, filtering, formatting, validation, and publishing. Automatically invoke when the user asks to "create a new report", "build a report from scratch", "make a dashboard", "set up a report with KPIs", "create an executive dashboard", "add pages and visuals to a new report". --- @@ -41,10 +41,13 @@ When the user's request lacks specific measures, audience context, structural pr 8. Query field values for filters or formatting: `pbir model "Name.Report" -q "EVALUATE VALUES('Table'[Column])"` 9. Inspect field data types: `pbir model "Name.Report" -d -t Table` 10. Add visuals (the page already has a textbox for the title): `pbir add visual kpi "Name.Report/Overview.Page" --title "Revenue"` -11. Validate: `pbir validate "Name.Report"` -12. Publish: `pbir publish "Name.Report" "Workspace.Workspace/Name.Report"` -13. Open in Fabric after publish: `pbir publish "Name.Report" "Workspace.Workspace/Name.Report" -o` -14. Or open locally in Power BI Desktop: `pbir open "Name.Report"` +11. Configure query reduction (slicers + heavy visuals): see `references/interactivity.md` +12. Wire cross-filter overrides and page navigation: see `references/interactivity.md` +13. Add filters and slicers; configure slicer sync and reset: see `references/interactivity.md` +14. Validate: `pbir validate "Name.Report"` +15. Publish: `pbir publish "Name.Report" "Workspace.Workspace/Name.Report"` +16. Open in Fabric after publish: `pbir publish "Name.Report" "Workspace.Workspace/Name.Report" -o` +17. Or open locally in Power BI Desktop: `pbir open "Name.Report"` ## Step-by-Step Process @@ -142,7 +145,27 @@ Key principles: - **No redundant titles**: Page title = subject ("Order Lines"), visual titles = differentiator ("by Key Account", "Monthly Trend"). Hide subtitles: `pbir visuals subtitle "path" --no-show` - **Sorting**: Charts auto-sort descending by first measure. After `pbir visuals bind`, set sort explicitly: `pbir visuals sort "path" -f "Table.Measure" -d Descending` -### Step 7: Add Filters +For reports where users pick which measure or dimension to view, use field parameters to collapse N near-duplicate visuals into one. See **`references/interactivity.md`** (Field Parameters section). + +### Step 7: Configure Query Reduction + +For any report with slicers or a busy page (matrices, maps, high-cardinality tables), apply query reduction at creation -- not after. Retrofitting leaves stale interaction pairs and the page may already have unintended cross-filtering wired in. + +The three components (see **`references/interactivity.md`** for full detail): + +1. Add `NoFilter` `visualInteractions` pairs from each slicer to heavy visuals +2. Enable the filter-pane Apply button: `pbir filters pane-set` +3. Enable per-slicer Apply buttons: `pbir visuals format` on each slicer + +For Import-mode reports with cheap queries, evaluate whether disabling cross-highlight is worth the loss of interactivity before applying. + +### Step 8: Wire Interactions and Navigation + +After placing visuals, set cross-filter overrides and build navigation. Default is everything cross-filters everything, so author only the exceptions. For multi-page reports, add a `pageNavigator` visual rather than individual buttons. + +See **`references/interactivity.md`** (Wiring Interactions and Navigation section) for the step-by-step and pitfalls. + +### Step 9: Add Filters and Slicers ```bash pbir add filter Date Year -r "Sales.Report" --values 2025 @@ -150,7 +173,9 @@ pbir add filter Geography Region -r "Sales.Report" pbir add filter Products Category -p "Sales.Report/Detail.Page" ``` -### Step 8: Format Visuals +For slicer type selection, sync groups, and reset/persist filter patterns, see **`references/interactivity.md`** (Slicer Type, Sync, and Reset section). + +### Step 10: Format Visuals Most formatting should come from the theme (Step 5). Apply bespoke formatting only for genuinely one-off cases. @@ -164,14 +189,32 @@ pbir visuals title "Sales.Report/Overview.Page/Revenue.Visual" --fontSize 14 --b pbir visuals background "Sales.Report/Overview.Page/Revenue.Visual" --color "#F8F9FA" ``` -### Step 9: Validate +For titles that react to slicer selections or change color with status, use a `_Report` extension measure: + +```bash +# Author a selection-aware title measure +pbir dax measures add \ + -n "Title_Sales" \ + -e 'IF(ISFILTERED(Region[Region]), "Sales: " & SELECTEDVALUE(Region[Region], "multiple regions"), "Sales: all regions")' \ + -t _Report + +# Bind via title CF, or drive a textbox text run with the measure +``` + +Key rules for dynamic titles: +- `SELECTEDVALUE(..., "")` is mandatory; without it the title disappears under multi-select +- Keep the text measure and color measure separate (one returns the string, one returns a theme token like "good"/"bad"); each is independently testable +- A measure-driven title and a literal title cannot coexist; clear the literal first +- Test across no/single/multi selection by reasoning about `ISFILTERED`/`HASONEVALUE`, then confirm via render + +### Step 11: Validate ```bash pbir validate "Sales.Report" pbir tree "Sales.Report" -v ``` -### Step 10: Publish or Open +### Step 12: Publish or Open ```bash pbir publish "Sales.Report" "MyWorkspace.Workspace/Sales.Report" # Publish @@ -205,4 +248,6 @@ pbir open "Sales.Report" # Open in ## Reference Files - **`references/vague-prompts.md`** -- Handling underspecified prompts: targeted questions, sensible defaults, propose-before-building workflow -- **`references/layout-example.md`** -- Complete layout with coordinates, spacing verification, time granularity table +- **`references/layout-example.md`** -- Complete layout with coordinates, spacing verification, time granularity guidance +- **`references/interactivity.md`** -- Query reduction, cross-filter wiring, navigation, slicer type/sync/reset, field parameters +- **`references/limitations.md`** -- Agent limitations to communicate to users diff --git a/plugins/reports/skills/create-pbi-report/references/interactivity.md b/plugins/reports/skills/create-pbi-report/references/interactivity.md new file mode 100644 index 00000000..0a252d22 --- /dev/null +++ b/plugins/reports/skills/create-pbi-report/references/interactivity.md @@ -0,0 +1,134 @@ +# Interactivity: Slicers, Interactions, Field Parameters, and Filter Persistence + +## Query Reduction at Creation + +The "Query reduction" preset bundles three settings that collapse unnecessary visual queries. Apply at report creation; retrofitting leaves stale interaction pairs. + +The three components set directly in PBIR: + +- **Cross-interaction default**: add `NoFilter` pairs in `page.json` `visualInteractions[]` from each slicer to heavy visuals (matrices, maps, high-cardinality tables). There is no single report-wide off key; write per-pair overrides. +- **Filter-pane Apply button**: set via `pbir filters pane-set`; validate with `--qa` after. +- **Slicer Apply button**: set per-slicer via `pbir visuals format`. The `slicer` and `advancedSlicerVisual` object models differ; confirm the container key with `pbir schema describe` first. + +Microsoft's optimization guidance shows these three changes cutting a page's visual queries by a large factor, which is why the "fewer visuals / apply buttons" audit rules exist. + +Pitfalls: +- A hard-coded `Highlight` pair in `visualInteractions[]` survives the preset; audit per page after applying +- Apply buttons are per-slicer/per-pane; five slicers need five configurations unless you also add page-level Apply-all/Clear-all button visuals +- Disabling cross-highlight is a visible design tradeoff (selecting a bar no longer dims others); flag this on Import-mode reports where query cost is already cheap + + +## Wiring Interactions and Navigation + +After placing and formatting visuals, set interaction overrides and build navigation before final validation. A page of correctly bound visuals still ships poorly if every selection cross-filters everything or if a multi-page report has no way to move between pages. + +1. Decide the cross-filter graph per page before touching JSON. Default is everything cross-filters everything, so author only the exceptions: KPI cards that should stay stable when a detail chart is clicked; charts that should not filter a slicer back. +2. Write overrides as page-level `visualInteractions` entries (`{source, target, type}`) using the visual `name`, not the title. Use `Highlight`/`Filter` only when overriding a default (charts default to Highlight, line/scatter/map to Filter). +3. For multi-page reports, prefer a native `pageNavigator` visual over hand-built buttons. One navigator auto-syncs to the page list; N buttons are N blobs to re-point on every page add or rename. +4. Validate, then reload+screenshot to confirm a slicer click filters the intended visuals and leaves the cards alone. + +```bash +# Example: set KPI card to NoFilter from bar chart clicks +pbir visuals interactions "Sales.Report/Overview.Page" \ + --source "Bar Chart.Visual" --target "Revenue KPI.Visual" --type NoFilter + +# Add a page navigator +pbir add visual pageNavigator "Sales.Report/Overview.Page" \ + --x 24 --y 680 --width 400 --height 32 +``` + +Pitfalls: +- `visualInteractions` pairs are directional (`source` filters `target`, not the reverse) +- Stale pairs referencing a renamed or regenerated visual `name` silently no-op; they do not error during validate +- Override exceptions, keep the default elsewhere; do not over-suppress interactivity + + +## Slicer Type, Sync, and Reset + +### Type and single-select + +Map slicer intent to type and mode: + +``` +low cardinality, single pick -> slicer (data.mode=Dropdown) or advancedSlicerVisual with singleSelect +measure beside each option -> listSlicer (Values: Column/Hierarchy; Tooltips: Measure) +date window -> slicer (data.mode=Between) or relative-date mode +``` + +Force single-select where the analysis assumes one value: + +```bash +pbir set / 'visual.objects.selection[0].properties.singleSelect' true +pbir set / 'visual.objects.selection[0].properties.strictSingleSelect' true # removes clear-to-all +``` + +Default to `strictSingleSelect` for metric-swap slicers (field parameter pickers) and single-entity slicers. Multi-select on those is the most common silent break. + +### Sync groups + +Sync is not a slicer property; it lives in `report.json` as a sync group keyed by name. A slicer joins a group by sharing the name. Two independent toggles: +- sync filter state: selection follows the reader page to page +- sync visibility: whether the slicer is drawn on each page + +The common pattern is sync-state everywhere, show-on-one-page. After hand-editing sync groups, validate and reload (a silent group-name typo de-syncs with no error). Sync supports one field per slicer only; a two-field slicer opts out. + +### Reset and persist filters + +Power BI has no native reset button; build it from a bookmark: + +1. Set every slicer/filter to its intended default state +2. Capture a **Data-scoped** (not Display) bookmark: `pbir bookmark add "Default View"` +3. Bind a button to it via `pbir visuals action` + +Use the same bookmark as the page launch bookmark so the report opens at the baseline regardless of persisted state. + +The "Don't allow end users to save filters" report setting turns off per-reader persistence. Select it for shared/kiosk reports that must open clean; leave persistence on for personal analytical reports. Persistence-on plus a launch bookmark conflict; decide one model per report. + +Pitfalls: +- A Display-scoped reset bookmark moves/shows/hides visuals unexpectedly; always use Data scope for filter resets +- The reset button only restores what the bookmark captured; re-capture when slicers change +- Sync group names must match exactly; a trailing space de-syncs silently + + +## Field Parameters + +A field parameter is a calculated table in the model whose rows each name a column or measure (`NAMEOF`) plus a sort order. Bind the table's value column to a visual drop zone and to a single-select slicer; readers pick which measure or dimension renders without bookmarks or visual stacking. It collapses N near-duplicate visuals into one. + +The table is model work (define exactly three columns per row, all values unique, or the parameter produces unexpected results). In PBIR the parameter participates like any column: + +```bash +# Add the chart bound to the parameter +pbir add visual barChart "Sales.Report/Overview.Page" \ + --title "by Metric" \ + -d "Category:Product.Category" \ + -d "Y:Metric.Metric Fields" \ + --x 24 --y 120 --width 600 --height 360 + +# Add the slicer to pick the metric +pbir add visual slicer "Sales.Report/Overview.Page" \ + -d "Values:Metric.Metric" \ + --x 24 --y 72 --width 200 --height 40 + +# Force single-select so the report never opens showing every measure at once +pbir set "Sales.Report/Overview.Page/.Visual" \ + 'visual.objects.selection[0].properties.singleSelect' true + +# Validate that the parameter field binding resolves +pbir validate "Sales.Report" --fields +``` + +### PBIR-side pitfalls + +These are the parts that break after the model is correct: + +- **Blank/"none" selection means ALL, not nothing**: there is no empty state; use strict single-select plus a default selection so the report never opens showing every measure collapsed together +- **Top N filters break on the parameter**: they rank alphabetically by display name, not by the measure value. Fix: author a helper measure reading the active row via `SELECTEDVALUE` over the parameter's hidden column + `SWITCH`, then apply Top N on that helper +- **CF does not follow the swap**: conditional formatting binds to a concrete field. Drive CF from the same `SELECTEDVALUE` + `SWITCH` helper measure +- **Implicit measures**: an explicit measure must exist first; no implicit aggregation is created for the parameter column +- **Drillthrough/tooltip link fields**: not usable as drillthrough or tooltip link fields; link the underlying dimensions instead +- **Live connection models**: field parameters require a local model; a pure live connection cannot host one +- **Matrix "Persist hierarchy level"**: turn this report setting off; otherwise the hierarchy level collapses on every parameter switch + +### Validate and render + +Always validate with `--fields` after binding a field parameter; mismatched Column vs Measure types pass JSON validation but fail at query time. diff --git a/plugins/reports/skills/create-pbi-report/references/layout-example.md b/plugins/reports/skills/create-pbi-report/references/layout-example.md index 54d8f7b1..c9a041ec 100644 --- a/plugins/reports/skills/create-pbi-report/references/layout-example.md +++ b/plugins/reports/skills/create-pbi-report/references/layout-example.md @@ -70,12 +70,12 @@ If no clear target exists, ask the user via `AskUserQuestion`. When adding trend visuals, infer the appropriate time axis from active filters: -| Active Filter | Trend Granularity | Date Column | -|---|---|---| -| Year (e.g., 2021) | Monthly | `Date.Calendar Month (ie Jan)` or `Date.Calendar Month Year (ie Jan 21)` | -| Quarter | Monthly or Weekly | `Date.Calendar Month (ie Jan)` or `Date.Calendar Week EU (ie WK25)` | -| Month | Daily or Weekly | `Date.Date` or `Date.Calendar Week EU (ie WK25)` | -| No date filter | Monthly or Quarterly | `Date.Calendar Month Year (ie Jan 21)` or `Date.Calendar Quarter Year (ie Q1 2021)` | +```yaml +Year (e.g. 2021): Monthly # Date.Calendar Month (ie Jan) or Date.Calendar Month Year (ie Jan 21) +Quarter: Monthly or Weekly # Date.Calendar Month or Date.Calendar Week EU (ie WK25) +Month: Daily or Weekly # Date.Date or Date.Calendar Week EU (ie WK25) +No date filter: Monthly or Quarterly # Date.Calendar Month Year or Date.Calendar Quarter Year +``` If unsure, default to monthly granularity -- it works well for most business reporting contexts. diff --git a/plugins/reports/skills/create-pbi-report/references/vague-prompts.md b/plugins/reports/skills/create-pbi-report/references/vague-prompts.md index 1ef8592e..32a2f4fa 100644 --- a/plugins/reports/skills/create-pbi-report/references/vague-prompts.md +++ b/plugins/reports/skills/create-pbi-report/references/vague-prompts.md @@ -39,14 +39,26 @@ If the user still deflects ("just make it look good"), proceed with sensible def When specifics are missing, fall back to these rather than guessing: -| Decision | Default | Rationale | -|----------|---------|-----------| -| Theme | Check if a theme is applied; if not, apply the **sqlbi** theme | Professional colors and typography out of the box | -| Layout | Executive dashboard pattern (KPI row, trend chart, breakdown, detail table) | Most broadly useful; follows 3-30-300 | -| Page size | 1280x720 | Standard 16:9 | -| KPI selection | Top measures by business importance from the model | Explore with `pbir model -d`; propose before building | -| Time granularity | Monthly if yearly filter context; weekly/daily if monthly | Match the grain to the decision cadence | -| Conditional formatting | Gap/variance columns only; theme sentiment colors ("good"/"bad") | Formatting everything means formatting nothing | +```yaml +Theme: + default: sqlbi (already applied to new reports) + reason: professional colors and typography out of the box +Layout: + default: executive dashboard pattern (KPI row, trend chart, breakdown, detail table) + reason: most broadly useful; follows 3-30-300 +Page size: + default: 1280x720 + reason: standard 16:9 +KPI selection: + default: top measures by business importance from the model + reason: explore with `pbir model -d`; propose before building +Time granularity: + default: monthly if yearly filter context; weekly/daily if monthly + reason: match the grain to the decision cadence +Conditional formatting: + default: gap/variance columns only; theme sentiment colors ("good"/"bad") + reason: formatting everything means formatting nothing +``` ### 4. Propose before building diff --git a/plugins/reports/skills/deneb-visuals/SKILL.md b/plugins/reports/skills/deneb-visuals/SKILL.md index 79016710..79e8dcbb 100644 --- a/plugins/reports/skills/deneb-visuals/SKILL.md +++ b/plugins/reports/skills/deneb-visuals/SKILL.md @@ -1,6 +1,6 @@ --- name: deneb-visuals -version: 26.20 +version: 26.24 description: Deneb visual creation, Vega/Vega-Lite spec authoring, and Deneb best practices for PBIR reports. Automatically invoke whenever the user mentions "Deneb" in any context, or asks about Vega/Vega-Lite specs in Power BI, Deneb cross-filtering, Deneb interactivity, pbiColor theme integration, Deneb field name escaping, or Deneb rendering issues. --- @@ -164,7 +164,7 @@ Enable interactivity via the `vega` objects in visual.json: ### Cross-Filtering -When `enableSelection` is true, handle `__selected__` (`"on"`, `"off"`, `"neutral"`) in encode blocks. Selection modes: `simple` (auto-resolves) or `advanced` (requires event definitions, Vega only). See `references/vega-patterns.md` for the full pattern. +When `enableSelection` is true, handle `__selected__` (`"on"`, `"off"`, `"neutral"`) in encode blocks. Selection modes: `simple` (auto-resolves, up to 250 data points) or `advanced` (Vega only; required for brush/lasso/region selection, supports up to 2500 via `options.limit`, exposes `pbiCrossFilterApply` and `pbiCrossFilterClear` signals). See `references/vega-patterns.md` for the simple pattern and `references/advanced-patterns.md` for the advanced signal API. ### Cross-Highlighting @@ -186,7 +186,7 @@ Key fields: `__row__` (zero-based row index, replaces removed `__identity__`), ` 4. **Use theme colors** (`pbiColor`, `pbiColorNominal`) instead of hex values 5. **Use `enter`/`update`/`hover`** encode blocks for clean state management (Vega only) 6. **Enable tooltips** with `"tooltip": {"signal": "datum"}` on marks -7. **Mind row limits** -- 10K default; set `dataLimit.override` and use `renderMode: canvas` for large datasets +7. **Performance** -- aggregate in DAX first, prefer `renderMode: canvas` for many marks, and only then raise `dataLimit.override`. See `references/advanced-patterns.md` for the full lever order 8. **Test field names** -- verify `nativeQueryRef` matches spec field references 9. **Avoid external data** -- AppSource certification prevents loading external URLs 10. **Escaping depends on context** -- double quotes in standalone specs, doubled single quotes in PBIR visual.json (see escaping rules above) @@ -207,7 +207,8 @@ Deneb is the preferred choice for **advanced custom visuals** that need interact - **`references/vega-patterns.md`** -- Vega chart patterns (bar, line, scatter, donut, stacked, heatmap, area, lollipop, bullet, KPI card), standard config, transforms and scales reference - **`references/vega-lite-patterns.md`** -- Vega-Lite chart patterns (for editing existing Vega-Lite visuals only) - **`references/pbir-structure.md`** -- PBIR JSON structure (literal encoding, query state, interactivity example) -- **`references/capabilities.md`** -- Full Deneb object properties reference and template format +- **`references/capabilities.md`** -- Full Deneb object properties reference and template format (`usermeta` schema) +- **`references/advanced-patterns.md`** -- Advanced cross-filtering signals (Vega `pbiCrossFilterApply`/`pbiCrossFilterClear`), performance engineering lever order, and community template round-trip from the terminal - **`examples/visual/bullet-chart.json`** -- PBIR visual.json: faceted bullet chart with conditional indicators and cross-filtering (Vega-Lite) - **`examples/visual/kpi-card.json`** -- PBIR visual.json: KPI card with layered text and conditional % change coloring (Vega-Lite) - **`examples/visual/trend-line.json`** -- PBIR visual.json: dual-series line chart with fold transform and color/legend mapping (Vega-Lite) diff --git a/plugins/reports/skills/deneb-visuals/references/advanced-patterns.md b/plugins/reports/skills/deneb-visuals/references/advanced-patterns.md new file mode 100644 index 00000000..addba41d --- /dev/null +++ b/plugins/reports/skills/deneb-visuals/references/advanced-patterns.md @@ -0,0 +1,96 @@ +# Advanced Deneb Patterns + +## Advanced Cross-Filtering (Vega only) + +Simple selection mode (`selectionMode: simple`) auto-resolves `__selected__` per mark. Advanced mode is required for brush-select, lasso, region-click, or any selection that is not one-mark = one-row. It is **Vega only**; Vega-Lite cannot use it, so plan the provider from the start. + +Two signal functions exposed in advanced mode: + +- `pbiCrossFilterApply(event, filter?, options?)` -- filter the original dataset by the optional predicate and cross-filter on the result; `event` is required as the first arg +- `pbiCrossFilterClear()` -- clear, no args + +Basic point-click pattern: + +```json +"signals": [{ + "name": "pbiCrossFilterSelection", + "value": [], + "on": [ + { + "events": {"source": "scope", "type": "mouseup", "markname": "data-point"}, + "update": "pbiCrossFilterApply(event)" + }, + { + "events": { + "source": "view", "type": "mouseup", + "filter": ["!event.item || event.item.mark.name != 'data-point'"] + }, + "update": "pbiCrossFilterClear()" + } + ] +}] +``` + +For a brush, bind a Vega `interval` selection and pass its extent into the `filter` argument. The `options.limit` key (1-2500, default 50) overrides the simple-mode 250 cap because you control row resolution. + +Enable via: + +```bash +pbir set "" objects.vega[0].properties.selectionMode '"advanced"' +``` + +Pitfalls: + +- Switching a working simple-mode visual to advanced silently kills cross-filtering until the signals are authored +- After aggregated marks the resolved data is at a different granularity than the base `dataset`; aliased columns must match original names or rows resolve empty +- A community-reported failure pattern: the visual filters others but never reflects an external filter back into `__selected__`; flag in review, it is a known Deneb limitation in some versions + +## Performance Engineering + +A Deneb visual re-runs the whole Vega dataflow on every interaction. Cost scales row count x mark count x per-row field count, and Power BI widens every row with runtime fields. Apply levers cheapest-first: + +1. **Aggregate in DAX, not Vega** -- bind a monthly measure so the model sends ~12 rows; a `transform aggregate` over 50K rows recomputes on every hover. Push grouping to the model or a visual-level Top N filter +2. **Prefer `renderMode: canvas` for many marks** -- SVG creates a DOM node per mark; canvas draws to a bitmap. Keep SVG only when marks are few or selectable text is needed +3. **Trim marks before raising limits** -- collapse decorative layers first +4. **Drop unused runtime field bloat** -- every bound measure ships `__formatted` and `__format`; bind only what the spec uses +5. **`dataLimit.override` as a last resort** -- paired with canvas; removes the soft 10K guard but the model's own row cap still applies and can truncate silently + +```bash +pbir set "" objects.vega[0].properties.renderMode '"canvas"' +pbir set "" objects.dataLimit[0].properties.override true +``` + +Keep the cross-filter `limit` as low as the interaction allows (simple mode caps at 250 for this reason). + +Pitfalls: + +- `dataLimit.override` does not remove the model-side cap; verify the expected row count actually arrives +- Canvas mode breaks SVG-dependent tooltips and makes text non-selectable; confirm tooltips still fire after switching +- A spec fine in Desktop can stall in a specific browser (Edge has had known rendering issues with large canvas specs); check the audience's browser + +## Template Round-Trip (Terminal Import) + +The Deneb GUI prompts for placeholder mapping on import; from a terminal you substitute manually then inject. A template is a Vega/Vega-Lite spec whose field references are placeholders matching `^__[a-zA-Z0-9]+__$` (e.g. `__category__`), described in `usermeta.dataset`. + +Steps: + +1. Read `usermeta.dataset` for each `key` (placeholder), `kind` (`column` vs `measure`), `type`. Kind is load-bearing: wrong kind passes `pbir validate` but fails at runtime +2. Strip `usermeta` and `$schema` from the spec body (Deneb's object does not consume `usermeta`) +3. Replace each `__placeholder__` token in the spec text with the real field name, applying PBIR escaping (doubled single quotes inside `jsonSpec` for names with spaces/apostrophes); do this before injection +4. Add the visual, inject the rewritten spec into `jsonSpec`, create the `dataset` role bindings: + +```bash +pbir visuals bind "" -r dataset -d "Sales.OrderDate" -t Column +pbir visuals bind "" -r dataset -d "Sales.Total" -t Measure +``` + +5. Set `provider` to match (`vega` / `vegaLite`); if the template shipped a `config` block, inject it into `jsonConfig`, not `jsonSpec`; validate + +Pitfalls: + +- `usermeta.deneb.providerVersion` can lag the installed Deneb build; the most common break is a leftover `datum.__identity__` (removed in Deneb 1.9); rewrite to `datum.__row__` +- Leaving `usermeta` in `jsonSpec`, or leaving `"data": {"values": [...]}` sample rows, ships stale embedded data; the spec must read `{"name": "dataset"}` +- External `data.url` templates fail AppSource certification and break offline; replace with the `dataset` binding +- A wrong-`kind` binding (column where measure expected) passes validate but renders blank + +For the `usermeta.dataset` schema see `references/capabilities.md`. diff --git a/plugins/reports/skills/deneb-visuals/references/community-examples.md b/plugins/reports/skills/deneb-visuals/references/community-examples.md index ec7aaf76..adfdc88f 100644 --- a/plugins/reports/skills/deneb-visuals/references/community-examples.md +++ b/plugins/reports/skills/deneb-visuals/references/community-examples.md @@ -70,6 +70,8 @@ Key elements: 3. Stringify the spec and wrap in single quotes for `jsonSpec` literal value (see escaping rules in SKILL.md Step 3) 4. Extract `config` separately for `jsonConfig` +For a full terminal workflow (CLI bind commands, `kind` validation, breaking changes, pitfalls) see `references/advanced-patterns.md`. + **To create a new template from an existing spec:** 1. Replace hardcoded field names with placeholder keys (`__0__`, `__1__`, etc.) diff --git a/plugins/reports/skills/modifying-theme-json/SKILL.md b/plugins/reports/skills/modifying-theme-json/SKILL.md index 901e6056..809eebd9 100644 --- a/plugins/reports/skills/modifying-theme-json/SKILL.md +++ b/plugins/reports/skills/modifying-theme-json/SKILL.md @@ -1,6 +1,6 @@ --- name: modifying-theme-json -version: 26.20 +version: 26.24 description: Design, enforce, audit, and validate Power BI report themes. This skill MUST be invoked when a report uses the default or built-in theme, has a minimal custom theme (few or no visualStyles), or has accumulated many visual-level formatting overrides (objects/visualContainerObjects in visual.json); these are signs the theme needs attention. Also automatically invoke when the user asks to "create a theme", "design a theme", "enforce theme compliance", "audit theme adherence", "push formatting to theme", "clear visual overrides", "standardize report formatting", "update theme colors", "change theme typography", "set theme text classes", "validate a theme", "add visual-type overrides to the theme", "copy a theme", "download a theme", "apply a template", or mentions theme design, enforcement, compliance, or visual formatting inconsistency. --- @@ -241,43 +241,72 @@ After validation, deploy and visually verify: | Resource | URL | |----------|-----| | Official report theme JSON schema (versioned, Draft 7) | [microsoft/powerbi-desktop-samples — Report Theme JSON Schema](https://github.com/microsoft/powerbi-desktop-samples/tree/main/Report%20Theme%20JSON%20Schema) | -| Latest schema (v2.152, March 2026, exploration v5.71) — **check repo for newer** | [reportThemeSchema-2.152.json](https://github.com/microsoft/powerbi-desktop-samples/blob/main/Report%20Theme%20JSON%20Schema/reportThemeSchema-2.152.json) | -| Raw schema URL (for `$schema` IDE integration) — **update version as needed** | `https://raw.githubusercontent.com/microsoft/powerbi-desktop-samples/main/Report%20Theme%20JSON%20Schema/reportThemeSchema-2.152.json` | +| Latest schema (resolve the newest version at author time, see below) | [Report Theme JSON Schema folder](https://github.com/microsoft/powerbi-desktop-samples/tree/main/Report%20Theme%20JSON%20Schema) | +| Raw schema URL (for `$schema` IDE integration) — update version to match consumers' Desktop | `https://raw.githubusercontent.com/microsoft/powerbi-desktop-samples/main/Report%20Theme%20JSON%20Schema/reportThemeSchema-2.154.json` | | Microsoft Learn — Use report themes in Power BI Desktop | https://learn.microsoft.com/en-us/power-bi/create-reports/desktop-report-themes | | Microsoft Learn — Report theme JSON file format | https://learn.microsoft.com/en-us/power-bi/create-reports/desktop-report-themes#report-theme-json-file-format | | Community theme templates | [deldersveld/PowerBI-ThemeTemplates](https://github.com/deldersveld/PowerBI-ThemeTemplates) | | PBIR item schemas | [microsoft/powerbi-desktop-samples — item-schemas](https://github.com/microsoft/powerbi-desktop-samples/tree/main/item-schemas) | +Resolve the current schema version at author time rather than hardcoding a number: + +```bash +gh api repos/microsoft/powerbi-desktop-samples/contents/"Report Theme JSON Schema" \ + --jq '.[].name' | sort | tail -1 +``` + ### IDE Integration (`$schema`) -Add a `$schema` property to the theme JSON to enable autocomplete and validation in VS Code (or any JSON Schema-aware editor): +Add a `$schema` property to the theme JSON to enable autocomplete and validation in VS Code. Use the versioned raw GitHub URL, not the generic `powerbi.com` marker: ```json { - "$schema": "https://raw.githubusercontent.com/microsoft/powerbi-desktop-samples/main/Report%20Theme%20JSON%20Schema/reportThemeSchema-2.152.json", + "$schema": "https://raw.githubusercontent.com/microsoft/powerbi-desktop-samples/main/Report%20Theme%20JSON%20Schema/reportThemeSchema-2.154.json", "name": "MyTheme", - "dataColors": ["#1971c2", ...] + "dataColors": ["#1971c2", "..."] } ``` -The schema is used verbatim by Power BI Desktop to validate themes on import — if the JSON fails schema validation, Power BI Desktop will reject the theme. Always target the schema version that matches the Power BI Desktop version in use. Schemas follow the pattern `reportThemeSchema-2.{version}.json` where the version matches the monthly Desktop release. +Power BI validates an imported theme against the schema baked into the Desktop build. Validation is reject-unknown-fields: one misspelled key refuses the entire theme. A theme passing `jq` validation can still fail Desktop import on a typo. See `references/theme-authoring.md` for the full schema version guidance. ## Theme Top-Level Keys -| Key | Type | Purpose | -|-----|------|---------| -| `name` | string | Display name shown in Power BI UI | -| `dataColors` | string[] | Ordered hex palette for data series | -| `good` / `bad` / `neutral` | string | Flat hex keys for CF measure semantic colors | -| `maximum` / `center` / `minimum` | string | Gradient color extremes (flat hex keys) | -| `foreground` variants | string | `foreground`, `foregroundLight`, `foregroundDark`, `foregroundNeutralSecondary`, etc. | -| `background` variants | string | `background`, `backgroundLight`, `backgroundNeutral`, `backgroundDark` | -| `textClasses` | object | Typography per semantic role (`title`, `label`, `callout`, `header`, `boldLabel`, etc.) | -| `visualStyles` | object | `[visualType][state]` formatting cascade | +```yaml +name: string # display name shown in Power BI UI +dataColors: string[] # ordered hex palette for data series + +# Semantic CF colors (flat hex keys; CF measures return these names as strings) +good / bad / neutral: string + +# Gradient CF colors (flat hex keys; "null" is the key name, not JSON null) +maximum / center / minimum / null: string + +# Structural colors (non-data chrome: gridlines, axis labels, filter-card bg, etc.) +# Use level-N names for new themes; CLI alias equivalents shown in parentheses +firstLevelElements: string # (foreground) +secondLevelElements: string # (foregroundNeutralSecondary) +thirdLevelElements: string # (backgroundLight) +fourthLevelElements: string # (foregroundNeutralTertiary) +background: string +secondaryBackground: string # (backgroundNeutral) +tableAccent: string + +# Extended surface/foreground palette +foregroundLight / foregroundDark: string +backgroundLight / backgroundNeutral / backgroundDark: string + +textClasses: object # typography per semantic role (title, label, callout, header, boldLabel, etc.) +visualStyles: object # [visualType][state] formatting cascade +``` + +For `textClasses`: 4 primary classes you set; 8 secondary classes that derive automatically. See `references/theme-authoring.md`. + +For `visualStyles`: named style presets are a second key alongside `"*"` inside a visual-type section; they surface in the Format pane Style dropdown. See `references/advanced-theme-features.md`. ## References -- **`references/theme-authoring.md`** — Color system design, typography, wildcard minimum set, schema integration +- **`references/theme-authoring.md`** — Color system design (data colors, semantic, structural, null gradient), text class inheritance, `$id` filter-card states, wildcard minimum set, schema version guidance +- **`references/advanced-theme-features.md`** — Named style presets (Format-pane dropdown), base theme layering model, organizational theme distribution, mobile-only formatting overrides - **`references/serialize-build.md`** — Serialize/build workflow: splitting themes into editable files, editing, rebuilding, validation, temporary folder guidance - **`references/applying-themes.md`** — Applying templates, post-apply enforcement, clearing visual overrides, normalizing hardcoded colors - **`references/copying-themes.md`** — Copying themes between reports, extracting/downloading themes, comparing themes, consolidating across a portfolio diff --git a/plugins/reports/skills/modifying-theme-json/references/advanced-theme-features.md b/plugins/reports/skills/modifying-theme-json/references/advanced-theme-features.md new file mode 100644 index 00000000..2b09e9bf --- /dev/null +++ b/plugins/reports/skills/modifying-theme-json/references/advanced-theme-features.md @@ -0,0 +1,120 @@ +# Advanced Theme Features + +Covers named style presets, the base theme layering model, organizational theme distribution, and mobile-only formatting overrides. + +--- + +## Named Style Presets + +A native theme-file feature distinct from the `pbir visuals preset` CLI command (which stamps formatting onto a visual instance). Theme named presets surface a Style dropdown in the Format pane so report authors can pick a pre-built visual look per-visual; they are the file-based way to ship multiple branded table looks or chart variants. + +Structure: add a named key alongside `"*"` inside a visual-type section. The key name is what appears in the dropdown: + +```json +"columnChart": { + "*": { + "stylePreset": [{ "name": "Brand Bottom Legend" }], + "legend": [{ "position": "BottomCenter", "show": true }] + }, + "Brand Bottom Legend": { + "legend": [{ "position": "BottomCenter" }] + }, + "Brand Right Legend": { + "legend": [{ "position": "Right" }] + } +} +``` + +Key mechanics: +- A named preset inherits from `"*"` for its type; put shared formatting in `"*"` and only deltas in each preset name +- `"*" -> stylePreset -> [{name}]` sets which preset is the post-import default +- Presets are scoped to one visual type; you cannot define a cross-type preset +- A `visual.json` referencing a deleted preset name shows a "can't be found" error and falls back to `"*"`; grep for preset references before renaming or deleting + +```bash +# Check for visual.json references to a named preset before deleting it +grep -r "\"styleName\"" Report.Report/definition/pages/ +``` + +No `pbir` command authors theme presets; edit the visual-type file in the serialized `.Theme` folder and rebuild. + +--- + +## Base Theme and the Layering Model + +The cascade described in SKILL.md has a hidden first layer: a base theme Microsoft ships with Desktop that establishes factory defaults for every property. This base is frozen at report creation time; a report keeps its original base indefinitely (Desktop shows a banner in Customize Theme when a newer base is available). + +``` +Base theme (Microsoft-managed, frozen at creation) + | +Custom theme (your JSON; overrides only what it sets) + | +Theme visual-type overrides + | +Visual-level overrides (visual.json) +``` + +This explains three behaviors: +- Two reports with the same custom theme may render differently if created months apart (frozen bases differ) +- "Reset to default" reverts only to the custom theme's values; for keys the custom theme does not set, the base takes effect (not truly blank) +- A minimal custom theme (just `dataColors` and a name) is fragile: almost every formatting decision falls through to a base you cannot pin or version-control in PBIR + +The defense is completeness: set the wildcard and structural keys explicitly so the base is effectively displaced. A theme that fully specifies its intended defaults is portable across base versions. + +Side effect of "reset to default": clearing `objects`/`visualContainerObjects` via `pbir visuals clear-formatting` strips data-bound items, CF rules, button/image actions, and URL/field-bound images along with chrome overrides. The CLI's `--keep-cf` guards conditional formatting but a blanket clear removes more than chrome. Audit before batch-clearing on reports with rich interactive content. + +--- + +## Organizational Themes + +The tenant-level distribution mechanism for one corporate look across hundreds of reports. A Fabric tenant admin uploads validated theme JSON in the admin portal; those themes appear in the Themes dropdown for every author in Desktop and Service. + +Key behaviors: +- Applying an org theme removes any existing custom theme and replaces it wholesale; org theme and a hand-authored custom theme are mutually exclusive on one report +- The admin portal runs the same schema validation as Desktop import, plus a unique-name constraint; a theme failing `pbir theme validate` also fails org upload +- An agent cannot publish to the gallery from the terminal (admin-portal only) + +Authoring for org deployment: write valid JSON, name it uniquely, and make it complete enough to stand alone (no reliance on a local base the recipients will not have). `pbir theme validate` is the pre-upload gate. + +For compliance audits: the right question becomes "does this report match the org theme?" rather than only "is the JSON internally consistent?". Compare with: + +```bash +# Download the org theme from any report that has it applied, then diff +pbir theme diff "Report.Report" "OrgTheme.json" +``` + +Scriptable theme deployment: `sempy_labs.report` (Semantic Link Labs) is the closest path to programmatic theme application across a workspace without the admin portal. + +--- + +## Mobile-Only Formatting Overrides + +`mobile.json` accepts its own `objects` and `visualContainerObjects` alongside the required `position` block. This means the phone copy of a visual can be formatted differently from the web copy without touching the desktop visual. + +The cascade on phone: + +``` +Theme defaults + -> Theme wildcard/visual-type + -> visual.json overrides + -> mobile.json overrides (most specific; portrait only) +``` + +Practical use: a card showing a 28pt callout on desktop can show 16pt on phone via a `mobile.json` with only a `fontSize` override in its `objects` block. Strip non-essential chrome per Microsoft's mobile guidelines: axis titles, gridlines, and legends rarely survive portrait width. + +```json +{ + "$schema": "...", + "position": { "x": 0, "y": 0, "z": 0, "width": 320, "height": 80 }, + "objects": { + "title": [{ "properties": { "fontSize": { "expr": { "Literal": { "Value": "10" } } } } }] + } +} +``` + +Rules to keep overrides minimal: +- Store only properties that must differ from the desktop; every omitted property inherits from the desktop visual +- A change that applies on all surfaces (brand colors, shared typography) goes in the theme, not `mobile.json`; otherwise you reintroduce per-visual override sprawl +- Duplicating the full desktop `objects` into `mobile.json` is an anti-pattern: it doubles the maintenance surface and drifts; store deltas only + +Cascade trap: a desktop-visual formatting fix does not propagate to any property already pinned in `mobile.json`; the mobile value wins on phone. After editing a visual's desktop formatting, check whether a corresponding `mobile.json` override needs updating. diff --git a/plugins/reports/skills/modifying-theme-json/references/theme-authoring.md b/plugins/reports/skills/modifying-theme-json/references/theme-authoring.md index 4e3a718d..4e8bf3be 100644 --- a/plugins/reports/skills/modifying-theme-json/references/theme-authoring.md +++ b/plugins/reports/skills/modifying-theme-json/references/theme-authoring.md @@ -42,22 +42,29 @@ Never author a theme from an empty object. Start from: Add a `$schema` property as the first key to enable IDE autocomplete and inline validation in VS Code. Two schema URLs are used in practice: ```json -// Generic Power BI schema reference (used in exported themes) +// Generic Power BI schema reference (used in exported themes — triggers no real validation) { "$schema": "https://powerbi.com/product/schema#reportTheme" } // Versioned GitHub schema (recommended for authoring — enables full validation) -{ "$schema": "https://raw.githubusercontent.com/microsoft/powerbi-desktop-samples/main/Report%20Theme%20JSON%20Schema/reportThemeSchema-2.152.json" } +{ "$schema": "https://raw.githubusercontent.com/microsoft/powerbi-desktop-samples/main/Report%20Theme%20JSON%20Schema/reportThemeSchema-2.154.json" } ``` -Use the versioned GitHub URL when authoring or editing themes. Use the generic `powerbi.com` URL only if it was already present in an exported theme and you're not changing it. +Use the versioned GitHub URL when authoring or editing themes. The generic `powerbi.com` URL is a marker that exported themes carry; it triggers no schema validation. Do not use it for authoring. -The schema is versioned monthly alongside Power BI Desktop releases (pattern: `reportThemeSchema-2.{version}.json`). The latest as of March 2026 is `2.152` (exploration version 5.71). Target the version matching the Desktop release the report consumers are using. +The schema is versioned monthly alongside Power BI Desktop releases (pattern: `reportThemeSchema-2.{version}.json`). Resolve the newest version at author time rather than hardcoding a remembered number: -- Schema index (check for newer versions — schemas are released monthly): https://github.com/microsoft/powerbi-desktop-samples/tree/main/Report%20Theme%20JSON%20Schema -- The schema is Draft 7 compliant and is used verbatim by Desktop to validate themes on import. Invalid themes are rejected. -- In VS Code, trigger autocomplete with Ctrl+Space to see valid property names and enum values from the Format pane. +```bash +gh api repos/microsoft/powerbi-desktop-samples/contents/"Report Theme JSON Schema" \ + --jq '.[].name' | sort | tail -1 +``` + +Target the schema version matching the Desktop release the report consumers are using, not necessarily the absolute latest. + +Critical: Power BI validates an imported theme against the schema baked into the Desktop build, not the file's `$schema` URL. Validation is reject-unknown-fields: a single misspelled key refuses the whole theme and prompts for a corrected file. A theme that passes `jq` validation can fail Desktop import on one typo. This is stricter than `visual.json` (where unknown properties are usually dropped silently). -The `visualStyles` section of the schema documents every property available for each visual type — this is the most reliable reference for which properties exist and what their valid values are. +- Schema index (check for newer versions; schemas are released monthly): https://github.com/microsoft/powerbi-desktop-samples/tree/main/Report%20Theme%20JSON%20Schema +- The schema is Draft 7 compliant. The `visualStyles` section documents every property available for each visual type. +- In VS Code, trigger autocomplete with Ctrl+Space to see valid property names and enum values. --- @@ -81,7 +88,7 @@ Rules: ### 2. Semantic Colors -Flat top-level hex string keys used by conditional formatting measures that return color name strings (`"good"`, `"bad"`, `"neutral"`). These are NOT nested under a `sentimentColors` object — they are individual keys at the root level of the theme JSON. +Flat top-level hex string keys used by conditional formatting measures that return color name strings (`"good"`, `"bad"`, `"neutral"`). These are NOT nested under a `sentimentColors` object; they are individual keys at the root level of the theme JSON. ```json "good": "#2f9e44", @@ -89,12 +96,45 @@ Flat top-level hex string keys used by conditional formatting measures that retu "neutral": "#868e96", "maximum": "#1971c2", "center": "#f8f9fa", -"minimum": "#e03131" +"minimum": "#e03131", +"null": "#e9ecef" ``` > Conditional formatting measures that return `"good"` will use whatever hex is set here. This centralizes CF color control in one place. -### 3. Background/Foreground Variants +The gradient dialog pulls four colors: `minimum`, `center`, `maximum`, and `null` (applied to blank values in data-bar and background CF). If `null` is unset, blanks render Power BI's default (an off-orange `#FF7F48`) which clashes with most themes. Reports with sparse measures show blanks constantly; set all four together. The key is the literal string `"null"` (not JSON `null`). `pbir theme set-colors` exposes `--minimum/--center/--maximum` but not `--null`; write it directly in `_config.json`. + +### 3. Structural Colors + +Six flat top-level hex keys that recolor non-data chrome across every visual (gridlines, axis labels, table grid, trend lines, slicer outlines, filter-card and tooltip backgrounds). These are the highest-leverage move when building a dark or branded theme. + +The canonical names as written by the Customize-theme dialog: + +```yaml +firstLevelElements: # values/totals font, trend lines, card data labels, KPI text, filter/tooltip text +secondLevelElements: # axis labels, legend labels, table headers, slicer item font+outline, light secondary text classes +thirdLevelElements: # gridlines, shape fill, gauge arc background, applied filter-card background +fourthLevelElements: # legend dimmed, card category labels, multi-row card bars +background: # in-data-point label background, slicer dropdown, donut/treemap stroke, button fill, available filter-card/tooltip background +secondaryBackground: # grid outline, shape-map default, ribbon fill, tooltip separator +tableAccent: # table/matrix grid outline when present +``` + +Aliases used in the pbir CLI and older exports: + +```yaml +firstLevelElements: foreground +secondLevelElements: foregroundNeutralSecondary +thirdLevelElements: backgroundLight +fourthLevelElements: foregroundNeutralTertiary +secondaryBackground: backgroundNeutral +``` + +`pbir theme set-colors` exposes only the alias subset; for `firstLevelElements` through `fourthLevelElements` and `secondaryBackground`, write keys directly in `_config.json` then rebuild. Pick the level-N names for new themes (Desktop's dialog round-trips them). + +Dark theme trap: flipping only `background` to a dark hex while leaving element levels black makes gridlines and axis labels invisible (they inherit `firstLevelElements` and `secondLevelElements`). Set `firstLevelElements`, `secondLevelElements`, and `background` together with mutual contrast. Light text-class variants pull their color from these structural keys and break silently if skipped. + +### 4. Background/Foreground Variants Extended palette for container surfaces, canvas backgrounds, and foreground text. These feed into `visualContainerObjects` backgrounds and the filter pane. @@ -109,7 +149,7 @@ Extended palette for container surfaces, canvas backgrounds, and foreground text "backgroundDark": "#dee2e6" ``` -### 4. Additional Accent Colors +### 5. Additional Accent Colors ```json "tableAccent": "#1971c2", @@ -121,7 +161,7 @@ Extended palette for container surfaces, canvas backgrounds, and foreground text ### Color Principles - Refer to `pbi-report-design` skill → `references/visual-colors.md` for WCAG contrast requirements and accessibility guidance -- Use `ThemeDataColor` references (ColorId + Percent) in theme JSON rather than hardcoded hex wherever possible — this keeps the theme internally consistent if the palette changes +- Use `ThemeDataColor` references (ColorId + Percent) in theme JSON rather than hardcoded hex wherever possible; this keeps the theme internally consistent if the palette changes - Keep `dataColors[0]` as the "primary" color that appears most frequently across the report --- @@ -130,18 +170,48 @@ Extended palette for container surfaces, canvas backgrounds, and foreground text Text classes define font properties by semantic role. Every defined class overrides Power BI's defaults for that role across all visuals. -### Standard Roles +Power BI has 12 text classes: 4 primary ones you set explicitly, and 8 secondary ones that derive automatically from a primary (lighter shade or size delta). Over-specifying all 12 is wasted effort and a drift trap; set the four primaries, then override a secondary only when you need to break an inheritance. + +### Primary Roles (set these) + +```yaml +callout: "card data labels, KPI indicators" +title: "axis titles, multi-row card title, slicer header" +header: "key-influencers headers, tab headers" +label: "table/matrix headers, grid values, column headers" +``` + +### Secondary Roles (derive from primaries; override only when needed) + +```yaml +largeTitle: # derived from title, larger +semiboldLabel: # derived from label, semibold weight +largeLabel: # derived from label, larger +smallLabel: # derived from label, smaller +lightLabel: # derived from label, lighter color (from structural colors) +boldLabel: # derived from label, bold (used for table totals) +largeLightLabel: # derived from label, large + light +smallLightLabel: # derived from label, small + light +``` + +Common override: make table totals non-bold via `"boldLabel": {"bold": false}`. Without it, `boldLabel` inherits bold-weight from `label`. + +Caveat: `title` and slicer header partly derive their color from `dataColors[0]`, so changing the first data color shifts those text colors unexpectedly. Explicitly set `color` in `title` to override this. + +Use a plain hex string for `color` in `textClasses` (NOT the `{"solid":{"color":"..."}}` wrapper used in `visualStyles`). + +### Standard Roles Reference | Role | Typical Use | Recommended Size | |------|-------------|-----------------| -| `title` | Visual titles, page titles | 14–16pt | -| `header` | Section headers, column headers | 12–14pt | -| `label` | Axis labels, data labels | 11–12pt | -| `callout` | KPI values, prominent numbers | 28–36pt | +| `title` | Visual titles, page titles | 14-16pt | +| `header` | Section headers, column headers | 12-14pt | +| `label` | Axis labels, data labels | 11-12pt | +| `callout` | KPI values, prominent numbers | 28-36pt | | `dataTitle` | KPI subtitles / labels | 12pt | -| `boldLabel` | Emphasized labels | 12pt | -| `largeTitle` | Large section titles | 20–24pt | -| `largeLabel` | Larger variant of label | 13–14pt | +| `boldLabel` | Table totals, emphasized labels | 12pt | +| `largeTitle` | Large section titles | 20-24pt | +| `largeLabel` | Larger variant of label | 13-14pt | ### Font Choice @@ -219,7 +289,20 @@ At a minimum, set: - **`divider`** — `show: false` unless design calls for it - **`visualHeader`** — `show: true` to keep the visual header (focus mode, filter icon, etc.) - **`outspacePane`** — filter pane styling (see `pbir-format` → `theme.md`) -- **`filterCard`** — filter card styling for Available and Applied states +- **`filterCard`** — filter card styling; use the `$id` discriminator to style Available and Applied states differently (see below) + +### Filter-Card States with `$id` + +A single `filterCard` container styled without a `$id` applies identically to both states; you cannot make applied filters visually distinct from available ones. Use the `$id` discriminator to target each state independently: + +```json +"filterCard": [ + { "$id": "Available", "border": true, "backgroundColor": { "solid": { "color": "#f8f9fa" } } }, + { "$id": "Applied", "foregroundColor": { "solid": { "color": "#252423" } }, "backgroundColor": { "solid": { "color": "#e9ecef" } } } +] +``` + +`$id` values are fixed enumerations for this container (`Available` / `Applied`); values are case-sensitive. Mis-cased or invented values are silently ignored. This is one of the few containers in theme JSON that uses `$id`; get valid values from the schema or a Desktop-formatted export. ### Design Guidelines @@ -265,12 +348,14 @@ For detailed guidance on promoting bespoke visual.json formatting back into the Before considering a theme complete: -- [ ] `dataColors` has 6–12 entries; first color is the "primary" +- [ ] `dataColors` has 6-12 entries; first color is the "primary" - [ ] Semantic colors (`good`, `bad`, `neutral`) are set and distinct from series colors -- [ ] `textClasses` covers at minimum: `title`, `header`, `label`, `callout` +- [ ] Gradient colors (`minimum`, `center`, `maximum`, `null`) are all set; `null` prevents the default off-orange on blank values +- [ ] Structural colors (`firstLevelElements`, `secondLevelElements`, `background`) set with mutual contrast (critical for dark themes) +- [ ] `textClasses` covers at minimum the four primaries: `title`, `header`, `label`, `callout`; secondary classes overridden only where inheritance breaks - [ ] Wildcard sets container defaults: `title`, `background`, `border`, `dropShadow`, `padding` - [ ] `dropShadow.show: false` in wildcard - [ ] At least `textbox` and `image` have type-specific overrides disabling container chrome -- [ ] Filter pane (`outspacePane` and `filterCard`) styled in wildcard +- [ ] Filter pane (`outspacePane`) and filter cards (`filterCard` with `$id: Available/Applied`) styled in wildcard - [ ] Theme validates with `pbir theme validate "Report.Report"` (or `jq empty` as fallback) - [ ] Deployed and visually verified on at least 3 visual types diff --git a/plugins/reports/skills/pbi-report-design/SKILL.md b/plugins/reports/skills/pbi-report-design/SKILL.md index 74b02b2e..879f2704 100644 --- a/plugins/reports/skills/pbi-report-design/SKILL.md +++ b/plugins/reports/skills/pbi-report-design/SKILL.md @@ -1,11 +1,9 @@ --- name: pbi-report-design -version: 26.20 +version: 26.24 description: Power BI report design principles, layout guidance, and formatting best practices. Automatically invoke when the user asks about "report layout", "design best practices", "visual hierarchy", "3-30-300 rule", "KPI card design", "page layout", "accessibility in reports", "visual spacing", "report canvas", "card design patterns", "table formatting", "matrix formatting", or mentions report design principles. --- -Warning: This skill is incomplete and still in progress, but may provide value already as-is -- Kurt - # Power BI Report Design > **Report modification requires tooling.** Two paths exist: @@ -113,12 +111,10 @@ Before modifying visual formatting: ### When to Modify Theme vs Visual -| Scenario | Modify | -|----------|--------| -| All visuals of type need change | Theme | -| Single visual exception | Visual | -| Establishing design standards | Theme | -| Content-specific highlight | Visual | +- All visuals of a type need change -> theme +- Establishing design standards -> theme +- Single visual exception to a theme rule -> visual +- Content-specific highlight (one callout, one reference line) -> visual ### Theme Color Usage @@ -203,6 +199,8 @@ For complete guidance on KPI design, targets, trends, formatting hierarchy, icon - Use muted colors for non-essential elements - Highlight key data points sparingly +For chart-type selection (encoding hierarchy, Cleveland-McGill ranking), data-label discipline, and small-multiples guidance, consult **`references/chart-selection.md`**. + ### Tables and Matrices Tables require deliberate design -- "easy to create" differs from "easy to read." Key rules: @@ -226,6 +224,14 @@ For complete guidance on table vs matrix selection, formatting philosophy, condi - Use filter pane for additional filters - Consider sync slicers across pages +### Filter Pane + +The filter pane has its own information architecture beyond color and chrome. Key decisions: lock vs hide per filter card, how to name cards without renaming model fields, and report-level settings (Apply button, search, allow-change-filter-type). See **`references/filter-pane.md`**. + +### Mobile + +Power BI does not reflow a desktop page for portrait; the phone layout is a hand-picked subset re-placed on a narrow grid. Nothing renders on a phone until explicitly opted in with a `mobile.json`. See **`references/mobile.md`** for the subset-selection model and file mechanics. + ## Report Evaluation Criteria When asked to evaluate or audit a report, focus on objective criteria. Subjective evaluation is difficult for AI -- the report cannot be "seen" directly, and there is no intuitive sense of aesthetics, cognitive load, or effectiveness. Emphasize this limitation to users. @@ -233,7 +239,7 @@ When asked to evaluate or audit a report, focus on objective criteria. Subjectiv ### Objective Checklist 1. **Page count:** More than 5-8 pages is typically excessive -2. **Visuals per page:** More than 12-15 can cause performance issues (exceptions: simple visuals like images, textboxes, shapes) +2. **Visuals per page:** Count is a proxy; query cost per visual is the driver (see `references/layout-guidelines.md` performance section). Textboxes, images, shapes, and buttons do not emit queries. 3. **Theme usage:** Reports should use a custom theme for consistent formatting 4. **Layout consistency:** - Equal spacing between visuals? @@ -295,13 +301,16 @@ When evaluating, provide: ## References -For detailed documentation: - -- **`references/cards-and-kpis.md`** - KPI card design: targets, gaps, trends, formatting hierarchy, icons, accessible palettes, anti-patterns, and review checklist -- **`references/tables-and-matrices.md`** - Table and matrix design: decision-making framework, formatting philosophy (subtract don't add), conditional formatting (data bars, color scales), sorting, sparklines, matrix hierarchies, anti-patterns -- **`references/layout-guidelines.md`** - Complete layout specifications -- **`references/visual-colors.md`** - Color usage patterns -- **`references/page-titles.md`** - Title implementation +- **`references/cards-and-kpis.md`** -- KPI card design: targets, gaps, trends, formatting hierarchy, icons, accessible palettes, anti-patterns, review checklist +- **`references/tables-and-matrices.md`** -- Table and matrix design: decision-making framework, subtract-don't-add philosophy, conditional formatting, sorting, sparklines, matrix hierarchies, anti-patterns +- **`references/layout-guidelines.md`** -- Canvas dimensions, spacing tiers (intra-group/inter-group/margin), alignment rules, performance cost model +- **`references/visual-colors.md`** -- Color principles, CF basis decision (gradient vs rules vs field-value vs icons), semantic tokens, accessibility +- **`references/page-titles.md`** -- Title implementation, accessible title wording, hidden-title/alt-text rule +- **`references/chart-selection.md`** -- Encoding hierarchy (Cleveland-McGill ranking), chart type routing, data-label discipline, small multiples +- **`references/tooltips-and-annotations.md`** -- Report-page tooltip design, when not to use one, annotation primitives for guided analytics +- **`references/filter-pane.md`** -- Lock vs hide, card naming, card order, Applied/Available styling, report-level settings +- **`references/mobile.md`** -- Phone layout as a curated subset, `mobile.json` mechanics, what to include/exclude +- **`references/custom-visuals.md`** -- Build-vs-buy ranking, AppSource/org-store tradeoffs, licensing gaps ## Related Skills @@ -312,12 +321,13 @@ For detailed documentation: ### Custom Visuals -Reports often need visuals beyond what Power BI provides natively. Choose the right tool: +Reports often need visuals beyond what Power BI provides natively. Always consider in-repo code paths before reaching for a packaged third-party visual. See **`references/custom-visuals.md`** for the build-vs-buy decision and AppSource/org-store tradeoffs. -- **`deneb-visuals`** -- Vega/Vega-Lite declarative visuals. Preferred for advanced custom interactive charts (cross-filtering, tooltips, hover). Use when native visuals can't express the chart type needed. -- **`svg-visuals`** -- SVG via DAX measures. Preferred for simple inline graphics in tables, matrices, and cards (sparklines, data bars, progress bars, status indicators). No interactivity but lightweight and no custom visual registration needed. -- **`python-visuals`** -- matplotlib/seaborn scripts (static PNG). Preferred for statistical visualizations (distributions, regressions, correlations). No interactivity. -- **`r-visuals`** -- ggplot2 scripts (static PNG). Preferred for statistical visualizations, particularly where R's ecosystem excels (forecast, pheatmap, corrplot). No interactivity. +Skill routing for in-repo code visuals: +- **`deneb-visuals`** -- Vega/Vega-Lite declarative visuals; preferred for advanced custom interactive charts (cross-filtering, tooltips, hover) +- **`svg-visuals`** -- SVG via DAX measures; preferred for inline table/matrix/card graphics with no row cap issues +- **`python-visuals`** -- matplotlib/seaborn scripts (static PNG); for statistical visualizations that must compute at render time +- **`r-visuals`** -- ggplot2 scripts (static PNG); where R's statistical ecosystem has no Python peer (forecast, pheatmap, corrplot) ### Semantic Model diff --git a/plugins/reports/skills/pbi-report-design/references/chart-selection.md b/plugins/reports/skills/pbi-report-design/references/chart-selection.md new file mode 100644 index 00000000..edb01548 --- /dev/null +++ b/plugins/reports/skills/pbi-report-design/references/chart-selection.md @@ -0,0 +1,99 @@ +# Chart Selection, Data Labels, and Small Multiples + +## Encoding Hierarchy: Match the Task to the Channel + +Encode the value the reader needs to compare most precisely on the highest perceptual channel available. The Cleveland-McGill accuracy ranking: position on a common scale > length > angle/slope > area > color hue/saturation. + +Useful inverse for an agent: given a `visual.json` on disk, decide whether `visualType` matches the analytical task and what the minimal repair is. + +``` +ranking / magnitude compare -> length on common axis -> barChart / columnChart +trend over time -> position + slope -> lineChart / areaChart +two measures correlated -> 2D position -> scatterChart +part-to-whole, <=3-4 parts -> angle/area (accept) -> pie/donut OR stacked bar +part-to-whole, >4 parts -> length (prefer) -> stackedBarChart, not pie +single value vs target -> position vs marker -> kpi / card + reference line +``` + +Read the actual type and roles before judging: +```bash +pbir get "Page/MyVisual.Visual" .visual.visualType +pbir visuals bind "Page/MyVisual.Visual" --list-roles +``` + +Repair is usually a type swap preserving bindings (pie, donut, and stackedBarChart share Category/Y roles): +```bash +pbir set "Page/MyVisual.Visual" .visual.visualType '"clusteredBarChart"' +``` +Then sort and validate. + +### The sampling gate + +A type can be schema-valid and role-correct yet wrong because the data shape defeats the encoding. Sample with `pbir model -q` before trusting a line chart (are there enough distinct x-points to form a trend?) or a bar chart (are there more than two bars?). Two rows back means a line chart is wasted; one dominant slice plus a long tail means the pie should be a sorted bar with the tail grouped into "Other". + +### Pitfalls + +- Pie/donut beyond ~4 slices forces the weakest channels; apply Top N + "Other" rather than adding colors +- A combo chart second axis is justified only when two genuinely different units share a category axis +- Hue is last; do not solve a comparison problem by adding more colors + +--- + +## Data-Label Discipline + +Labels earn their place only when the exact value matters and cannot be read off the axis: line endpoints, a single highlighted bar, KPI deltas. Per-segment labels and stacked totals are two different objects: + +```bash +pbir visuals format "Page.Page/Visual.Visual" labels --on # per-data-point labels +pbir set "Page.Page/Visual.Visual" \ + '.visual.objects.totals[0].properties.show.expr.Literal.Value' "true" # stacked totals +``` + +### The stacked-total "incorrect number" trap + +The recurring symptom where a stacked total does not match a hand-summed expectation is almost always a measure problem, not a label problem. The engine's stacked-group sum diverges from the expected total when the underlying measure is non-additive: a ratio, distinct count, or time-intelligence expression. Verify with `pbir model -q "EVALUATE SUMMARIZECOLUMNS(...)"` before touching the label. If the total is the "wrong" number, the label is honest and the measure needs an explicit aggregation strategy. + +### Preferred pattern: endpoint-only labels + +Prefer binding a measure that returns the value at the last/relevant point and BLANK elsewhere, rather than global labels plus conditional-formatting-based hiding. The chart stays clean and only the number that matters is annotated. This is the extension-measure label pattern described in `thin-report-measures` references. + +### Pitfalls + +- Small multiples disable stacked total labels entirely (see below) +- Label display units must match the axis units +- A measure returning BLANK beats global-labels-plus-CF: less JSON, less fragile + +--- + +## Small Multiples + +Choose small multiples over a legend when the question is "does this pattern hold across a dimension" and there are more series than a legend reads cleanly (roughly more than 3-4). A legend overlays series in one frame for value comparison; small multiples separate them into a synchronized grid so you compare shapes across many categories. + +Supported only on bar, column, line, and area charts. The grid synchronizes axes and fills left-to-right then top-to-bottom in sort order, with overflow scrolling beyond the visible grid. + +### The Series vs SmallMultiples role distinction + +`Series`/`Legend` overlays series in a single frame; true small multiples use the dedicated `SmallMultiples` role that partitions into the grid. The `smallMultiplesLayout` object only takes effect when the `SmallMultiples` role is populated. Confirm before editing: + +```bash +pbir visuals bind "Page/MyVisual.Visual" --list-roles +``` + +### Features that are inert once a visual is trellised + +Do not waste edits on these; they silently have no effect in a small-multiples context: + +- Total labels for stacked charts +- Trend lines and forecasting +- Zoom sliders +- Line high-density sampling +- Concatenated axis labels and hierarchical axis (falls back to concatenated) +- Scroll-to-load-more +- Small-multiple cell title display units/decimals/format (control via the model's format string instead) + +Check for the `SmallMultiples` role before adding any analytics overlays. + +### Pitfalls + +- A 6x6 grid of dense charts defeats the purpose; apply Top N on the partition field and group the remainder into "Other" +- Synchronized axes are a feature, not a bug; do not give each cell its own scale unless the analytical intent is explicitly within-series comparison diff --git a/plugins/reports/skills/pbi-report-design/references/custom-visuals.md b/plugins/reports/skills/pbi-report-design/references/custom-visuals.md new file mode 100644 index 00000000..250b43f0 --- /dev/null +++ b/plugins/reports/skills/pbi-report-design/references/custom-visuals.md @@ -0,0 +1,38 @@ +# Custom Visual Selection: Build vs Buy + +## Default Ranking + +Prefer in-repo code paths over packaged third-party visuals. In-repo code (Deneb, SVG-via-DAX, Python/R) lives inside the report or model, travels with it, renders anywhere the engine is enabled, and requires no per-visual admin approval. + +``` +1. Core Power BI visual -- no dependencies, fastest +2. Deneb (Vega/Vega-Lite) -- declarative; preferred for advanced interactive charts +3. SVG-via-DAX measure -- preferred for table/card inline graphics; no row cap issues +4. Python/R script visual -- statistical plots that must compute at render time +5. Packaged AppSource visual -- only when the interaction model genuinely needs it +6. Private .pbiviz -- avoid; external risk plus you own the build pipeline +``` + +Reach for a packaged custom visual only when the interaction model genuinely requires it: a rich pre-built hierarchy/network slicer, a specialized gauge type, or a visual category with no reasonable in-repo alternative. + +## AppSource and Org-Store Tradeoffs + +An AppSource visual is an external dependency on a third party's code, AppSource availability, and three tenant settings (allow custom visuals, allow specific visual, allow uncertified). It can vanish, lose certification, or be policy-blocked at any time. + +When a packaged custom visual is warranted: +- Prefer the org store over ad-hoc AppSource (centralizes the approved version, single admin toggle) +- Prefer certified visuals (Microsoft-reviewed, sandboxed) +- Record the dependency in a PBIR annotation: name, source, certification status, approver + +## Licensing and Deployment Gaps + +Licensed AppSource visuals do not enforce or report licensing in: +- Power BI Report Server (sovereign clouds, on-premises) +- App-owns-data embed +- Publish-to-web + +Org-store/AppSource visuals are unavailable in Report Server entirely. If the report will be distributed via any of these paths, an AppSource dependency is a portability blocker. + +## The Real Cost of a Custom Visual Dependency + +Reaching for a custom visual to avoid learning Deneb or SVG trades a one-time authoring cost for a permanent governance and portability cost. A Deneb spec is version-controlled, diffable, and testable; an AppSource registration is not. diff --git a/plugins/reports/skills/pbi-report-design/references/filter-pane.md b/plugins/reports/skills/pbi-report-design/references/filter-pane.md new file mode 100644 index 00000000..16790078 --- /dev/null +++ b/plugins/reports/skills/pbi-report-design/references/filter-pane.md @@ -0,0 +1,38 @@ +# Filter Pane Information Architecture + +The decisions behind a filter pane are distinct from its color and chrome. Three core decisions per filter card: should it be locked, hidden, or visible? What should it be called? Where in the card order does it sit? + +## Lock vs Hide + +These encode different intents and are not interchangeable: + +- Lock: keeps a filter card visible but read-only. Use for scope guards the reader should know about ("Region = EMEA", "Status = Active"). The reader can see why numbers are filtered. +- Hide: removes the card entirely. Use for data-cleanup filters (exclude nulls, exclude test SKUs). These exist to prevent junk from appearing; the reader does not need to know about them. + +Rule: **lock to inform, hide to clean.** Never hide a business filter the reader expects to see; they will not know why the numbers look filtered. + +## Renaming Filter Cards + +The card title is editable independently of the field it filters; renaming does not rename the model field. Replace jargon field names like `D_SHOP[Reporting Row Name]` with a readable label like "Report Line". Renamed cards do not automatically rename matching slicers; rename the slicer header too. + +## Card Order + +The pane always groups cards by scope: report-level, then page-level, then visual-level. Custom ordering is only possible within a scope level. If a page-level filter needs to be the reader's first reach, promote it to a slicer on the canvas. + +## Applied vs Available Card Styling + +A filter card's Applied state (a filter is set) should be visually distinct from Available (no value set). This is the built-in restatement mechanism; readers scanning the pane can tell at a glance what is active. Set distinct styles in the theme via the `$id` discriminator (`Available`/`Applied`); see the `modifying-theme-json` skill for the `filterCard` theme pattern. + +## Report-Level Settings + +These persist in the report definition and affect all readers: + +- Search in filter pane: leave on when the pane has many cards; turn off on a curated 2-3 card pane +- Allow users to change filter types: off when a card is deliberately set to Top-N or relative-date (prevent readers from overriding it) +- Apply filters button: on for slow DirectQuery or large Import models where instant feedback creates multiple unnecessary queries; off for snappy Import models where immediate cross-filter is the better experience + +## Pitfalls + +- Hiding a business filter removes the reader's ability to see why numbers look filtered; lock it instead +- Publish-to-web does not render the filter pane at all; move essential filters to on-canvas slicers +- Renaming a filter card does not propagate to slicers on the same field; update both diff --git a/plugins/reports/skills/pbi-report-design/references/layout-guidelines.md b/plugins/reports/skills/pbi-report-design/references/layout-guidelines.md index c2f2f00e..dbabf887 100644 --- a/plugins/reports/skills/pbi-report-design/references/layout-guidelines.md +++ b/plugins/reports/skills/pbi-report-design/references/layout-guidelines.md @@ -23,6 +23,22 @@ Height: 720px ## Margins and Spacing +On a fixed PBIR canvas, white space is a finite budget spent explicitly through gaps between `position` rectangles. Two failure modes both come from treating gaps as residual: cramming (proximity collapses, groups stop reading) or scattering (page feels empty). + +Spend at three tiers, all multiples of the grid unit: + +```yaml +intra-group: ~8-16px # within a group (a KPI row) +inter-group: ~24-32px # between groups; this gap creates visual proximity grouping +margin: ~24-32px # canvas edge to first/last visual, all four sides +``` + +The inter-group gap must be strictly larger than the intra-group gap; that inequality is what turns spacing into hierarchy. + +Reserve the margin first (usable area = `canvas - 2*margin`). Avoid double-spending a gap and a box border for the same separation; prefer the gap. + +Padding (inside a visual container) and gaps (between visual containers) are different budgets controlled in different places; tightening one does not buy the other. + ### Page Margins ``` @@ -35,8 +51,8 @@ Right: 24-32px ### Visual Spacing ``` -Minimum gap between visuals: 16px -Recommended gap: 24px +Intra-group (within a visual group): 8-16px +Inter-group (between logical groups): 24-32px ``` ### Grid System @@ -246,19 +262,18 @@ This applies to any multi-row layout where visuals share implicit column boundar ## Performance Considerations -### Visual Count Limits +Opening a page refreshes every visible visual; each emits at least one DAX query. Parallelism is capped (DirectQuery default 10 concurrent connections), so latency grows non-linearly past that cap. Visual COUNT is a proxy; query COST per visual is the actual driver. 12 cheap card visuals are fine; 8 matrices with totals and measure filters may not be. -| Level | Visual Count | Notes | -|-------|--------------|-------| -| Optimal | 6-8 | Best performance | -| Acceptable | 9-12 | Slight impact | -| Warning | 13-15 | Noticeable delay | -| Critical | 16+ | Performance issues | +Visuals that emit more than one query per page load (multipliers): +- Tables/matrices with totals or subtotals (one query per band; DistinctCount and Median are worst) +- Measure filters (two queries) +- Top N filters (two queries; can hit the 1M-row intermediate limit in DirectQuery) +- Field parameters (an extra evaluated-parameters phase) +- Custom/Deneb/Python/R visuals (render phase dominates) ### Exceptions -Simple visuals with minimal impact: - +These do not emit queries and do not count toward visual density: - Textboxes - Images - Shapes diff --git a/plugins/reports/skills/pbi-report-design/references/mobile.md b/plugins/reports/skills/pbi-report-design/references/mobile.md new file mode 100644 index 00000000..0abd8610 --- /dev/null +++ b/plugins/reports/skills/pbi-report-design/references/mobile.md @@ -0,0 +1,38 @@ +# Mobile Layout: Phone View as a Curated Subset + +Power BI does not reflow a desktop page onto a phone. The portrait view is a hand-picked subset re-placed on a fixed narrow portrait grid. A visual appears in portrait only if it has a `mobile.json` alongside its `visual.json`. This inverts the desktop default: on the web every visual renders; on a phone nothing renders until explicitly opted in. + +A page with no mobile-placed visuals falls back to rotated landscape, which is usually unreadable on a phone held in portrait. + +## Design Principles + +Pick the few visuals that answer the headline question for a mobile reader: KPI cards, the one primary chart, the key slicer. Wide tables, scatter charts, and dense matrices rarely survive the narrow portrait grid. Stack vertically with the answer at the top. Lay slicers and navigation buttons horizontally so they consume one short band rather than stacking tall. + +**Build the phone layout alongside the desktop page** so a page copy duplicates placements from the start rather than being retrofitted. + +## File Mechanics + +The phone layout is stored per-visual, not per-page. Each visual folder may hold a `mobile.json` beside its `visual.json` (validated by the `visualContainerMobileState` schema). Required keys are `$schema` and `position`; the `position` block uses the same field names as desktop (`x, y, z, height, width, tabOrder, angle`) but in the phone-canvas coordinate space, which is independent of the desktop canvas. + +"The page has a phone layout" is emergent: it is true if at least one visual has a `mobile.json`. + +Inspect and set via the CLI: +```bash +pbir visuals mobile "Report.Report/Page.Page/Visual.Visual" +``` + +When hand-writing a `mobile.json`, copy the `$schema` URL from another `mobile.json` in the same report; a stale schema version is the most common validation failure. + +## What to Include and Exclude + +Rank visuals by their value for the mobile reader's primary question, then place the top 4-8 in a single column. Record which visuals you intentionally excluded. The over-faithful miniature anti-pattern (placing everything) produces a portrait page that scrolls forever and is harder to read than the rotated landscape fallback. + +Strip non-essential chrome for the mobile copy via mobile-only formatting overrides in `mobile.json`: smaller axis titles, drop legends, remove gridlines. These overrides are deltas; list only properties that must differ from the desktop visual. The theme-first cascade still applies: a formatting change that should apply on all surfaces belongs in the theme, not in `mobile.json`. + +## Pitfalls + +- A page that looks mobile-ready in the editor but has zero `mobile.json` files silently falls back to rotated landscape +- Mobile-optimized views render only in native iOS/Android apps; a browser (Service, Playwright, Chrome) always shows the landscape layout, so you cannot verify the portrait layout by browser screenshot +- Editing `visual.json` position does nothing to the phone layout and vice versa; they are independent coordinate records +- Deleting a visual's folder must also drop its `mobile.json` +- Absence of `mobile.json` is not automatically a defect; tie severity to intent (a headline KPI page with no mobile layout is high severity; a back-of-house detail page is usually fine) diff --git a/plugins/reports/skills/pbi-report-design/references/page-titles.md b/plugins/reports/skills/pbi-report-design/references/page-titles.md index e99b4bbf..3bc64099 100644 --- a/plugins/reports/skills/pbi-report-design/references/page-titles.md +++ b/plugins/reports/skills/pbi-report-design/references/page-titles.md @@ -166,10 +166,30 @@ For titles, typically disable: } ``` +## Accessible Titles + +A screen reader speaks a visual's title and type before any alt text. This means an acronym or jargon title is unintelligible spoken aloud; the accessibility constraint is stronger than the visual-design constraint. + +- Spell out the subject ("Current year sales vs prior year", not "CY Sales vs PY") +- Reserve abbreviations for axis labels inside the chart, not the title +- Do not encode the chart type in the title; the reader announces it already +- A dynamic measure-bound title is spoken on every filter change; keep it a plain readable phrase with no glyphs or unit-suffix soup + +Scan titles across a page: +```bash +pbir visuals format "MyPage/*" -p title.text +``` + +### Title visibility and alt text + +A hidden title (`title.show=false`) makes the reader fall back to a worthless auto name ("chart 4"). If the title is hidden for layout reasons, supply descriptive alt text on the visual's `general.altText` property instead. + +Decorative title textboxes (section headers, visual labels) should be removed from the tab order (set `tabOrder` to -1) so readers skip them rather than announcing them as navigation stops. + ## Best Practices 1. **Consistent positioning** - Same x, y across all pages 2. **Consistent sizing** - Same width, height, font size -3. **Descriptive text** - Clearly describe page purpose +3. **Descriptive text** - Clearly describe page purpose, spell out abbreviations 4. **Avoid redundancy** - Don't repeat report name if obvious -5. **Consider mobile** - Ensure readable on smaller screens +5. **Consider mobile** - Ensure readable on smaller screens; see `mobile.md` diff --git a/plugins/reports/skills/pbi-report-design/references/tooltips-and-annotations.md b/plugins/reports/skills/pbi-report-design/references/tooltips-and-annotations.md new file mode 100644 index 00000000..4c334f43 --- /dev/null +++ b/plugins/reports/skills/pbi-report-design/references/tooltips-and-annotations.md @@ -0,0 +1,71 @@ +# Tooltips, Annotations, and Guided Analytics + +## Tooltip Page Design + +A report-page tooltip is a designed mini-canvas, not a bigger hover list. Three legitimate intents: + +- Different perspective: same data pivoted (hover a monthly bar, see the full-year trend) +- Add detail: same grain, attributes the source visual omitted +- Add help: static explainer wired to the visual header icon, not data points + +### When not to build one + +- One or two extra numbers: use the Tooltip field well, not a report page +- Anything the user must interact with: use drillthrough, not a tooltip + +### Design rules + +Keep the page small and `ActualSize` (320x240 default; 240x180 for a single chart); design at final pixels since `ActualSize` does not scale. Strip chrome harder than a normal page: + +```bash +pbir visuals format "TooltipPage/*" background --show false +``` + +Titles off, one focal visual plus at most a card or two. One tooltip page can serve many source visuals; reuse rather than clone. + +### Pitfalls + +- Tooltip pages count toward page/visual budgets even though hidden; consolidate near-duplicates +- A tooltip that restates the data point already visible adds a query per hover for zero insight; delete it +- Do not reach for a tooltip when the need is comparison or navigation; those want drillthrough or a page link +- Never place a page-level `filterConfig` on a tooltip page; the hover is the filter (see `pbir-cli` references for filter-compatibility grain checks) +- "Tooltip size affected by canvas size" autoscale is report-level, not settable per page from PBIR; design for actual pixels and flag for the owner + +--- + +## Annotations and Guided Analytics + +Annotation-as-design is the chrome that carries the analytical argument. This describes how to compose existing PBIR primitives into a storytelling layer. + +### Building blocks + +**Shaded "what happened here" band:** a reference line with `shade*` properties spanning a date range behind the hero series: +```bash +pbir reference-line add "Page/Hero.Visual" --type constant --value "2024-01-01" --shade-to "2024-03-31" +# reference-line adds are not idempotent; capture the returned id +``` + +**Target line stating the finding:** a reference line bound to a target measure with `dataLabel` on, label driven by the measure ("Plan: 11.0M"). + +**Highlight-and-grey:** hero series in a brand data color, others grey via per-series selectors (`series(col=val)`) or a measure-driven CF that returns grey for non-hero categories. The measure-driven approach is more robust; per-series selectors break if the series set changes. + +**Callout with leader:** a chrome-off textbox (title/background/border/shadow all `show=false`) plus a thin shape as the leader line. Keep callouts as separate visuals so they are independently `cp`-able and themeable. + +**Reveal pacing:** prefer a page sequence plus a page navigator over bookmarks for step-by-step story reveals. Bookmarks are fragile and capped; reserve them for in-page state toggles. + +**Reading order / scent:** set `tabOrder` so traversal is headline -> hero visual -> callouts -> detail panel. + +### Review checklist: flag weak narrative + +These are not caught by `pbir bpa`: + +- Headline textbox is a chart-type label ("Bar Chart: Revenue") rather than a finding +- Every series at full saturation with no grey context series +- A reference or target line exists but `dataLabel` is off +- Callouts present but `tabOrder` does not follow the story +- Story carried entirely by a Copilot Narrative visual with no deterministic fallback + +### Pitfalls + +- Reference-line and error-bar adds are not idempotent; always capture the returned id and use it for subsequent edits +- A wall of callouts harms accessibility; decorative chrome is budget-exempt but a screen reader still walks the `tabOrder` sequence diff --git a/plugins/reports/skills/pbi-report-design/references/visual-colors.md b/plugins/reports/skills/pbi-report-design/references/visual-colors.md index fa7b02c3..f54b0eb8 100644 --- a/plugins/reports/skills/pbi-report-design/references/visual-colors.md +++ b/plugins/reports/skills/pbi-report-design/references/visual-colors.md @@ -90,15 +90,36 @@ For highlights and emphasis: - Reserve bright colors for important data - Don't use red/orange unless indicating problems +## Choosing the CF Basis + +Pick the basis before touching `pbir visuals cf`; picking wrong produces valid JSON that misleads readers. + +Each basis encodes a claim about the data: + +- **Gradient:** the measure is continuous and comparable across rows ("more is darker"). Valid only for a magnitude on a single scale. A signed variance with no explicit center miscolors the midpoint and re-stretches on every refresh. +- **Rules:** discrete business-defined bands (RAG, SLA met/missed) whose cut points come from policy, not the data's min/max. These survive refresh without shifting. +- **Field value (measure-driven):** the color or icon is itself data that a measure computed. Most flexible; prefer it for anything non-trivial. Rule: if the logic has more than two thresholds or depends on another measure, make it measure-driven rather than an inline rules array. +- **Icons:** a status faster to scan than a number. Use sparingly on a triage/status column; never on the primary value column. + +Apply CF to the secondary signal (variance, gap, status), not the headline value. Data bars on the one primary magnitude, color scale on the variance column; never both on the same column. + +Prefer theme tokens over hex in every basis (`pbir visuals cf ... --theme-colors` to convert existing hex assignments). + +### CF Pitfalls + +- A signed-measure gradient with no center colors zero as mid-gray noise; set the center explicitly to 0 or use rules +- `IconOnly` hides the number; only use it where the number itself is irrelevant +- Overlapping or gappy rule bounds silently leave rows uncolored; promote fiddly logic to a measure you can test with `pbir model -q` + ## Conditional Formatting Colors ### Best Practices -1. **Theme tokens over hex** -- CF should use theme sentiment tokens ("good", "bad", "neutral", "minColor", "maxColor") not hardcoded hex. Theme tokens mean changing the theme cascades to all CF across all reports. Use `--theme-colors` to convert existing hex CF to tokens. -2. **Measure-driven preferred** -- Prefer extension measures returning theme tokens over built-in gradient/rules. Measure logic lives in one place; change the measure or theme and it propagates. Use `--to-measure` to convert built-in CF. -3. **Sparingly applied** -- CF should highlight exceptions, not decorate everything. Formatting everything means formatting nothing. Apply to variance/gap columns, not raw values. -4. **Accessible** -- Use blue/orange instead of red/green for colorblind safety. Always pair color with a secondary cue (icon, text, shape). -5. **Theme-first hierarchy** -- Check theme sentiment colors exist before applying CF. Create them if missing by setting `good`, `bad`, and `neutral` sentiment colors in the theme.json file (e.g., good="#00B050", bad="#FF0000", neutral="#FFC000") +1. **Theme tokens over hex** -- use `--theme-colors` to convert existing hex CF assignments to tokens; changing the theme then cascades everywhere +2. **Measure-driven conversion** -- use `--to-measure` to promote built-in gradient/rules CF to a measure expression; logic becomes testable and versionable +3. **Sparingly applied** -- CF should highlight exceptions; formatting every column means nothing stands out +4. **Accessible** -- use blue/orange instead of red/green; always pair color with a secondary cue (icon, text, shape) +5. **Theme-first** -- check that `good`, `bad`, and `neutral` sentiment colors exist in the theme before applying CF; add them if missing (e.g., `good="#00B050"`, `bad="#FF0000"`, `neutral="#FFC000"`) ### Positive/Negative Pattern diff --git a/plugins/reports/skills/pbir-cli/SKILL.md b/plugins/reports/skills/pbir-cli/SKILL.md index 85546744..36e5b242 100644 --- a/plugins/reports/skills/pbir-cli/SKILL.md +++ b/plugins/reports/skills/pbir-cli/SKILL.md @@ -1,7 +1,7 @@ --- name: pbir-cli -version: 26.20 -description: This skill should be used whenever the user mentions "pbir", "pbir-cli", "Power BI reports", or "PBI reports", or works with .pbir, .pbip, or .pbix files. Covers creating, exploring, formatting, validating, and publishing Power BI reports through the pbir CLI and object model. +version: 26.24 +description: This skill should be used whenever the user mentions "pbir", "pbir-cli", "Power BI reports", or "PBI reports", works with .pbir, .pbip, or .pbix files, or wants to refresh, screenshot, or visually verify a report that is open in Power BI Desktop. Covers creating, exploring, formatting, validating, and publishing Power BI reports through the pbir CLI and object model, plus driving Power BI Desktop (canvas reload, page screenshots) and querying connected or local semantic models. --- # Working with Power BI reports using `pbir` @@ -31,7 +31,7 @@ Keep entries concise and generalizable. The memory file is not a change log. Pru 3. Clarify intent. For vague or open-ended instructions, consult **`references/vague-prompts.md`** and use `AskUserQuestion` to understand expectations and report context before mutating anything. 4. Plan changes. For new reports, pages, or visuals, draft a wireframe or mock-up for the user to approve before building. 5. Make changes. Reach for relevant files in `references/`, `examples/`, and related skills like `pbi-report-design`. -6. Validate. Run `pbir validate` after every mutation. For visual confirmation, ask permission to publish to a sandbox workspace with `pbir publish` and inspect rendering via Chrome MCP, devtools CLI, or Playwright. +6. Validate. Run `pbir validate` after every mutation. For visual confirmation, prefer the local loop when the report is open in Power BI Desktop: `pbir desktop refresh` then `pbir desktop screenshot` and inspect the PNG (see "Desktop Integration" below). Otherwise ask permission to publish to a sandbox workspace with `pbir publish` and inspect rendering via Chrome MCP, devtools CLI, or Playwright. 7. Iterate. Expect multiple rounds. Push back on one-shot expectations from vague prompts. 8. Record learnings. Add concise, generalizable entries to the memory file noted above. @@ -52,6 +52,7 @@ Format: `ReportName.Report/PageName.Page/VisualName.Visual` - Properties via `get` or `set` and dot notation: `"Report.Report/Page.Page/Visual.Visual.title.fontSize"` - Filters/bookmarks: `"Report.Report/filter:Name"`, `"Report.Report/bookmark:Name"` - If multiple reports match, disambiguate with parent folder prefix +- Absolute filesystem paths work too: `"C:\Reports\Sales.Report"`, `"C:\Reports\Flash.pbix"` (globs do not combine with absolute paths) - Workspace destinations use `.Workspace` suffix: `"My Workspace.Workspace/Report.Report"` @@ -102,8 +103,24 @@ pbir model "Report.Report" -q "EVALUATE ROW(\"Revenue\", [Total Revenue])" # Te pbir fields list "Report.Report" # Fields already in use across report ``` +Routing depends on the report's model reference: thin reports (`byConnection`) query the Power BI / Fabric service; thick reports (`byPath`) query the local Analysis Services engine of the Power BI Desktop instance that has the report open, so `-q` and `-d` work fully offline against live model state. Local queries need the .NET Framework ADOMD client (found automatically from DAX Studio or Desktop installs; override with `PBIR_ADOMD_DIR`). + For full model query patterns and field binding workflows, consult **`references/fields-and-bindings.md`**. +### Desktop Integration (Refresh and Screenshot) + +When the report is open in Power BI Desktop (Windows, with the "external tool access" preview feature enabled), drive the running instance directly. This is the fastest way to visually verify changes; no publishing required. + +```bash +pbir desktop list # Running instances (PID, open file) +pbir desktop refresh "Report.Report" # Reload on-disk definition into the canvas +pbir desktop screenshot "Report.Report/Page.Page" -o verify.png +``` + +The edit-verify loop: mutate with `pbir set`/`add`, then `pbir desktop refresh`, then `pbir desktop screenshot`, then read the PNG. Inspect the rendered page after every meaningful change; screenshots catch what validation cannot (overlap, truncation, wrong field, illegible formatting). Set `PBIR_DESKTOP_AUTO_REFRESH=1` to fold the refresh step into every save. + +Screenshots need the Desktop window in the Report view. Refreshing an instance with unsaved changes makes Desktop save first, rewriting the whole definition on disk. PBIX files support screenshot but not refresh. For requirements, multi-instance behavior, and troubleshooting, consult **`references/desktop-integration.md`**. + ### Creating Reports New reports include out of the box: @@ -299,6 +316,7 @@ Data: fields, filters, dax, bookmarks, annotations Theme: theme (colors, text-classes, fonts, set-formatting, apply-template, diff) Schema discovery: schema (types, containers, describe), visuals properties, visuals format Workflow ops: validate, backup, restore, publish, download, batch, open, bpa +Desktop (Windows): desktop (list, refresh, screenshot) ``` Notes on the less-obvious groups: @@ -328,7 +346,8 @@ Top-level flags; place before the subcommand: `pbir -q new report ...`, NOT `pbi - **`pbir filters list` has no `-v` flag.** Use `--json` for detailed output. - **Do not convert to PBIX then publish the PBIR folder.** If converting to PBIX, publish the `.pbix` file directly. If publishing PBIR, skip conversion entirely. - **`pbir pages rename` renames folders only**; it does not change page IDs or display names. Use `--to` for single page folder rename. -- **Model schema is fetched via TMDL, not DMV**. `pbir model -q` runs EVALUATE DAX only; `INFO.TABLES()` and other DMVs return 400. Use `pbir model -d` for schema introspection. +- **DMV queries fail against service-connected models.** For thin reports (`byConnection`), `pbir model -q` runs EVALUATE DAX only; `INFO.TABLES()` and other DMVs return 400 from the service, and schema comes from TMDL. Use `pbir model -d` for schema introspection. Thick reports (`byPath`) open in Desktop query the local engine instead, where the live schema is used. +- **`pbir desktop refresh` does not work on PBIX files.** Desktop only reloads PBIP/PBIR definitions from disk; PBIX instances support `pbir desktop screenshot` only. - **Always run `pbir --help`** before using an unfamiliar command to confirm exact syntax. @@ -365,6 +384,7 @@ Run `pbir validate "Report.Report"` after **every mutation**. This catches broke ```yaml references/cli-reference.md: full syntax for any command with all flags references/exploration.md: exploring an unfamiliar report systematically +references/desktop-integration.md: driving Power BI Desktop; canvas refresh, page screenshots, auto-refresh, local model queries, troubleshooting references/create-new-report.md: building a report from scratch references/add-new-visual.md: adding visuals, layout patterns, bulk creation references/visual-groups.md: visual groups (create, add/remove members, ungroup) diff --git a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/54698b9cd0a0c57906b7.bookmark.json b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/54698b9cd0a0c57906b7.bookmark.json index c8fe1e67..ec06cce2 100644 --- a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/54698b9cd0a0c57906b7.bookmark.json +++ b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/54698b9cd0a0c57906b7.bookmark.json @@ -7,7 +7,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/71abeb475381792b035d.bookmark.json b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/71abeb475381792b035d.bookmark.json index c9a2c7eb..2fd1682e 100644 --- a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/71abeb475381792b035d.bookmark.json +++ b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/71abeb475381792b035d.bookmark.json @@ -10,7 +10,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/82ce9a7f49847d03190a.bookmark.json b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/82ce9a7f49847d03190a.bookmark.json index 77d17a35..16a2b8a1 100644 --- a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/82ce9a7f49847d03190a.bookmark.json +++ b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/82ce9a7f49847d03190a.bookmark.json @@ -10,7 +10,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/8f5a4883bde0d3bd075d.bookmark.json b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/8f5a4883bde0d3bd075d.bookmark.json index b8e5edba..fc054158 100644 --- a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/8f5a4883bde0d3bd075d.bookmark.json +++ b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/8f5a4883bde0d3bd075d.bookmark.json @@ -10,7 +10,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/958f29ad733c047ee0b8.bookmark.json b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/958f29ad733c047ee0b8.bookmark.json index d63ba894..7dedcee6 100644 --- a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/958f29ad733c047ee0b8.bookmark.json +++ b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/958f29ad733c047ee0b8.bookmark.json @@ -7,7 +7,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/e5b8aa0b8e0565be9ce0.bookmark.json b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/e5b8aa0b8e0565be9ce0.bookmark.json index 032c5731..2e3c6abf 100644 --- a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/e5b8aa0b8e0565be9ce0.bookmark.json +++ b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/bookmarks/e5b8aa0b8e0565be9ce0.bookmark.json @@ -10,7 +10,7 @@ }, "explorationState": { "version": "1.3", - "activeSection": "da2e63ebeb2179a994f1", + "activeSection": "56bda0fdb3f32a7d1fe9", "filters": { "byExpr": [ { diff --git a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/pages/AfterReport.Page/visuals/LineChart_TopLeft.Visual/visual.json b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/pages/AfterReport.Page/visuals/LineChart_TopLeft.Visual/visual.json index 141df89f..594dafe3 100644 --- a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/pages/AfterReport.Page/visuals/LineChart_TopLeft.Visual/visual.json +++ b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/pages/AfterReport.Page/visuals/LineChart_TopLeft.Visual/visual.json @@ -61,7 +61,7 @@ "Property": "OTD % (Value; PY REPT)" } }, - "queryRef": "On-Time Delivery.OTD % (Value; PY)", + "queryRef": "On-Time Delivery.OTD % (Value; PY REPT)", "nativeQueryRef": "PY", "displayName": "PY" } @@ -107,7 +107,7 @@ } }, "selector": { - "metadata": "On-Time Delivery.OTD % (Value; PY)" + "metadata": "On-Time Delivery.OTD % (Value; PY REPT)" } } ] diff --git a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/pages/pages.json b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/pages/pages.json index 335c73f3..93e3adfa 100644 --- a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/pages/pages.json +++ b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/pages/pages.json @@ -1,18 +1,8 @@ { "$schema": "https://developer.microsoft.com/json-schemas/fabric/item/report/definition/pagesMetadata/1.0.0/schema.json", "pageOrder": [ - "5d82d19c82c0d41db580", - "62e0e50a60b0575015eb", - "da2e63ebeb2179a994f1", - "0c32c81bce347402001e", - "1ce692e0ba0d1200c2ab", - "f11d192709500600d380", - "f1bf8244c8632606ac7d", - "57c5400a36b337093301", - "2dfd4022797c06129390", - "b1eca2da055007b1e7e5", - "b05b633226b8b89ed938", - "bf728418368097c42629" + "56bda0fdb3f32a7d1fe9", + "5b9de88f152744001bbe" ], - "activePageName": "da2e63ebeb2179a994f1" + "activePageName": "56bda0fdb3f32a7d1fe9" } diff --git a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/report.json b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/report.json index 854c1bc7..8c7743fd 100644 --- a/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/report.json +++ b/plugins/reports/skills/pbir-cli/examples/K201-MonthSlicer.Report/definition/report.json @@ -11,7 +11,7 @@ "type": "SharedResources" }, "customTheme": { - "name": "SqlbiDataGoblinTheme0699179381108107.json", + "name": "SqlbiDataGoblinTheme.json", "reportVersionAtImport": { "visual": "2.1.0", "report": "2.1.0", @@ -557,8 +557,8 @@ "type": "RegisteredResources", "items": [ { - "name": "SqlbiDataGoblinTheme0699179381108107.json", - "path": "SqlbiDataGoblinTheme0699179381108107.json", + "name": "SqlbiDataGoblinTheme.json", + "path": "SqlbiDataGoblinTheme.json", "type": "CustomTheme" } ] diff --git a/plugins/reports/skills/pbir-cli/references/cli-reference.md b/plugins/reports/skills/pbir-cli/references/cli-reference.md index 77d1191e..510e4208 100644 --- a/plugins/reports/skills/pbir-cli/references/cli-reference.md +++ b/plugins/reports/skills/pbir-cli/references/cli-reference.md @@ -16,6 +16,7 @@ Complete command reference for the pbir CLI. All commands prefixed with `pbir`. - [Annotation Operations](#annotation-operations) - [Best Practice Analyzer (BPA)](#best-practice-analyzer-bpa) - [Connection and Fabric](#connection-and-fabric) +- [Desktop Operations (Windows)](#desktop-operations-windows) - [Configuration and Setup](#configuration-and-setup) - [Visual Types Reference](#visual-types-reference) @@ -506,6 +507,24 @@ pbir model "Report.Report" -q "EVALUATE VALUES('Sales'[Region])" pbir model "Report.Report" -q "EVALUATE 'Sales'" -F json # JSON output ``` +Model query routing: thin reports (`byConnection`) query the Power BI / Fabric service (EVALUATE DAX only; DMVs rejected). Thick reports (`byPath`) open in Power BI Desktop query the local Analysis Services engine; requires the .NET Framework ADOMD client (auto-detected from DAX Studio or Desktop installs; override with `PBIR_ADOMD_DIR`). + +## Desktop Operations (Windows) + +Requires Power BI Desktop running with the report open and the preview feature "Enable external tool access to Power BI Desktop through secure local APIs" enabled. + +```bash +pbir desktop list # Running Desktop bridge instances +pbir desktop refresh "Report.Report" # Reload on-disk definition into the canvas +pbir desktop screenshot "Report.Report" # PNG of the first page +pbir desktop screenshot "Report.Report/Page.Page" # PNG of a specific page +pbir desktop screenshot "Report.Report/Page.Page" -o out.png # Explicit output path +pbir desktop screenshot "Report.Report" --scale 3 # Render scale (default 2) +pbir desktop screenshot "Report.Report" --pid 1234 # Disambiguate instances +``` + +Notes: screenshots need the Desktop window in the Report view. Refresh on an instance with unsaved changes makes Desktop save first (rewrites the definition on disk). PBIX files support screenshot but not refresh. `PBIR_DESKTOP_AUTO_REFRESH=1` auto-reloads the canvas after every pbir mutation. See `desktop-integration.md` for the full workflow. + ## Configuration and Setup ```bash diff --git a/plugins/reports/skills/pbir-cli/references/desktop-integration.md b/plugins/reports/skills/pbir-cli/references/desktop-integration.md new file mode 100644 index 00000000..d3cec8c5 --- /dev/null +++ b/plugins/reports/skills/pbir-cli/references/desktop-integration.md @@ -0,0 +1,101 @@ +# Power BI Desktop Integration; Live Canvas Refresh and Screenshots + +Drive a running Power BI Desktop instance from the CLI: reload the on-disk report into the canvas, capture page screenshots as PNG, and query the local model engine. This closes the edit-verify loop locally; no publishing to a sandbox workspace required. + +## Table of Contents +- [Requirements](#requirements) +- [Commands](#commands) +- [The Edit-Verify Loop](#the-edit-verify-loop) +- [Auto-Refresh on Save](#auto-refresh-on-save) +- [Live Local Model Queries](#live-local-model-queries) +- [PBIX Support and Limits](#pbix-support-and-limits) +- [Multiple Desktop Instances](#multiple-desktop-instances) +- [Caveats and Troubleshooting](#caveats-and-troubleshooting) + +## Requirements + +- Windows only; Power BI Desktop must be running with the report open +- The Desktop preview feature must be enabled: File > Options and settings > Options > Preview features > "Enable external tool access to Power BI Desktop through secure local APIs"; restart Desktop after enabling +- Screenshots additionally require the Desktop window to be in the **Report view** (not Model or Table view) + +When the bridge is unreachable, the CLI prints the preview-feature hint; when no instance has the target report open, it says so and names the report it looked for. + +## Commands + +```bash +pbir desktop list # Running Desktop instances (PID, open file) +pbir desktop refresh "Report.Report" # Reload on-disk definition into the canvas +pbir desktop screenshot "Report.Report" # PNG of the first page +pbir desktop screenshot "Report.Report/Page.Page" # PNG of a specific page +pbir desktop screenshot "Report.Report/Page.Page" -o out.png --scale 2 +pbir desktop screenshot "Report.Report" --pid 1234 # Target a specific instance +``` + +Paths accept the usual report resolution plus absolute filesystem paths: + +```bash +pbir desktop refresh "C:\Temp\Sales.Report" # Absolute path to a .Report folder +pbir desktop screenshot "C:\Reports\Flash.pbix" # Absolute path to an open .pbix +``` + +Default screenshot file name derives from the page display name (forbidden characters replaced); default scale is 2 (retina-quality, suitable for inspection). + +## The Edit-Verify Loop + +The core workflow this enables: make a change, reload the canvas, capture the page, and look at the rendered result. + +```bash +pbir set "Report.Report/Page.Page/Card.Visual.title.text" --value "Revenue" +pbir desktop refresh "Report.Report" +pbir desktop screenshot "Report.Report/Page.Page" -o verify.png +# Read verify.png to confirm the change rendered as intended +``` + +Inspect the PNG after every meaningful change; rendered output catches problems that validation cannot (overlap, truncation, color contrast, wrong field bound). Prefer this loop over publish-to-sandbox whenever the report is open in Desktop. + +## Auto-Refresh on Save + +Set `PBIR_DESKTOP_AUTO_REFRESH=1` to make every pbir mutation reload the canvas automatically, removing the explicit `pbir desktop refresh` step: + +```bash +# PowerShell +$env:PBIR_DESKTOP_AUTO_REFRESH = "1" +# bash +export PBIR_DESKTOP_AUTO_REFRESH=1 +``` + +Auto-refresh never raises; if no Desktop instance has the report open, saves proceed silently. It is off by default so that scripted bulk edits do not trigger a canvas reload per command. + +## Live Local Model Queries + +For thick reports (model referenced `byPath`) open in Desktop, `pbir model -q` executes DAX against Desktop's local Analysis Services engine, and field validation resolves the schema live: + +```bash +pbir model "Thick.Report" -q "EVALUATE TOPN(5, VALUES(Brands[Brand]))" +pbir model "Thick.Report" -d # Live schema: tables, columns, measures +``` + +This requires the .NET Framework build of the ADOMD client library. The CLI finds one automatically from DAX Studio or Power BI Desktop installs; override the search with `PBIR_ADOMD_DIR` pointing at a folder containing `Microsoft.AnalysisServices.AdomdClient.dll`. Note that Tabular Editor 3 ships a .NET 8 build that cannot be loaded; install DAX Studio if no compatible library is found. + +Thin reports (`byConnection`) are unaffected: their queries go to the Power BI / Fabric service as before, and DMV-style queries (INFO.TABLES() etc.) remain unsupported on that route. + +## PBIX Support and Limits + +A `.pbix` file with PBIR metadata is readable everywhere (`ls`, `tree`, `cat`, `get`, `validate`, `model`) via a read-only extraction cache, and `pbir desktop screenshot` works against the open PBIX instance. + +- `pbir desktop refresh` on a PBIX is rejected by Desktop itself ("Reload is only supported for PBIP/PBIR files"); there is no on-disk PBIR definition to reload +- All write commands (`set`, `add`, `rm`, ...) are rejected with a conversion hint; convert to PBIP first to edit +- Glob patterns do not combine with absolute paths; run from the report's parent directory or use relative paths for bulk operations + +## Multiple Desktop Instances + +Each open report is a separate Desktop process with its own bridge endpoint and local engine. The CLI correlates by the report folder (or source .pbix) each instance reports as open, so refresh and screenshot target the right instance automatically; the local engine port is matched to the owning Desktop process when several engines are running. Use `--pid` (from `pbir desktop list`) only when the same report is open twice. + +## Caveats and Troubleshooting + +- **Refresh reloads the definition, not themes.** `file.reload` re-reads pages and visuals from disk but not StaticResources: edits to theme JSON files (custom or base) only render after closing and reopening the file in Desktop. Desktop also resolves base themes by name from its internal store; the materialized `BaseThemes/*.json` is a snapshot it writes, not a file it reads (verified empirically; custom themes in `RegisteredResources` are read at open). +- **Refresh on a dirty instance saves first.** If the report has unsaved changes in Desktop, `file.reload` makes Desktop save before reloading; the whole definition is rewritten (re-indented, schema versions bumped, active page recorded). Expect git churn; commit or stash before iterating on a report a user has open with edits. +- **"Report view is not active"**: switch the Desktop window to the Report view and retry the screenshot. +- **Transient errors right after a refresh** (HostNotReady): handled automatically; the CLI honors the bridge's retry protocol. +- **Bridge unreachable**: enable the preview feature (see Requirements) and restart Desktop. +- **ADOMD client not found** when querying a local model: install DAX Studio or set `PBIR_ADOMD_DIR`. diff --git a/plugins/reports/skills/python-visuals/SKILL.md b/plugins/reports/skills/python-visuals/SKILL.md index 3808427f..ae5efa90 100644 --- a/plugins/reports/skills/python-visuals/SKILL.md +++ b/plugins/reports/skills/python-visuals/SKILL.md @@ -1,6 +1,6 @@ --- name: python-visuals -version: 26.20 +version: 26.24 description: Python visual creation and matplotlib/seaborn patterns for PBIR reports. Automatically invoke when the user mentions "Python visual", "matplotlib in Power BI", "seaborn in Power BI", "pythonVisual", or asks to "create a Python visual", "add a matplotlib chart", "write a Python visual script". --- @@ -128,7 +128,7 @@ Any locally installed package works without restriction. | Cross-filter FROM | Not supported | Not supported | | Receive cross-filter | Yes | Yes | | Publish to web | Not supported | Not supported | -| Embed (app-owns-data) | Ending May 2026 | Ending May 2026 | +| Embed (app-owns-data) | Not supported | Not supported | ## Script Structure Template @@ -162,18 +162,24 @@ else: plt.show() ``` -## When to Use Python Visuals +## When to Use a Script Visual -Python visuals are appropriate for **statistical and analytical visualizations** where the focus is on data analysis rather than interactivity. Use Python visuals when you need: +Reach for a Python visual only when **all** of the following hold: -- Statistical charts (distributions, regressions, correlations, violin plots) -- Analytical visualizations leveraging scipy, scikit-learn, or statsmodels -- Chart types that seaborn/matplotlib handle well but Power BI natives don't support +- The chart has no native equivalent and no reasonable Deneb spec +- The value is in a statistical computation that must run at render time (model fit, kernel density, forecast band), not just a shape Vega could draw +- The visual does not need to be a cross-filter source, hover tooltips, publish-to-web, or app-owns-data embed +- The report is served in a Pro/PPU or higher capacity with a Fabric-enabled region -**Output is static PNG** -- no cross-filtering FROM the visual, no hover/tooltip interactivity. Use Deneb instead for interactive custom visuals. Use SVG measures for simple inline graphics in tables/cards. +If interactivity or cross-filtering matters, use **Deneb** (a static PNG cannot be a selection source). If the need is a small inline mark (sparkline, bar, status pill), use an **SVG measure** (no row cap, no timeout, no licensing/region gate, renders under publish-to-web). The script visual's niche is narrow: compute-at-render statistical plots for internal or org consumption. + +**Python vs R once a script visual is the right call:** use Python when the computation leans on scikit-learn, statsmodels, or scipy, or when surrounding report logic is already Python. Use R for publication-quality statistical defaults and packages with no Python peer (`forecast`, `corrplot`, `pheatmap`, ridgeline/violin). Where equal, default to whichever language the report's other scripts use; mixing doubles the publish-time package surface to validate. + +Do not default to a script visual because a chart type "looks statistical." A box plot, lollipop, or dumbbell is an SVG-measure or Deneb job; reserve scripts for charts that genuinely compute. ## References +- **`references/data-model.md`** -- `dataset` grouping mechanic, the row/byte caps, and how to force per-row input - **`references/community-examples.md`** -- seaborn gallery examples organized by chart type, plus matplotlib and Python Graph Gallery links - **`references/chart-patterns.md`** -- Common matplotlib/seaborn chart patterns (bar, heatmap, donut, KPI, area) - **`examples/script/`** -- Standalone Python scripts (bar-chart, trend-line) -- ready to inject into visual.json after escaping diff --git a/plugins/reports/skills/python-visuals/references/data-model.md b/plugins/reports/skills/python-visuals/references/data-model.md new file mode 100644 index 00000000..143024b0 --- /dev/null +++ b/plugins/reports/skills/python-visuals/references/data-model.md @@ -0,0 +1,30 @@ +# Python Visual Data Model + +Behavior of `dataset` and the row cap; traps that cause silent wrong input. + +## Distinct-row grouping + +The 150,000-row cap applies to the **deduplicated set**, not the raw fact table. Power BI groups the `dataset` exactly like a table visual before handing it to the script: identical rows across all bound `Values` columns collapse to one. A script bound to `Region, Category` over a million-row fact table receives at most as many rows as there are distinct `Region + Category` combinations. + +This is a property of the field bindings, not the script. Consequences: + +- Per-transaction charts (jitter, strip, raw scatter, ECDF) silently receive aggregated input if no unique key is bound +- Scripts expecting row-level variation (e.g. bootstrapping, individual observations) produce wrong output without error +- The cap is hit much faster when a unique key is included; pre-filter at the visual or page level first + +To force per-row input, bind a guaranteed-unique column to the `Values` role alongside your other fields: + +```bash +pbir visuals bind Page/StripPlot -t Column -d "Values:Sales.Region" +pbir visuals bind Page/StripPlot -t Column -d "Values:Sales.Amount" +pbir visuals bind Page/StripPlot -t Column -d "Values:Sales.TransactionKey" +``` + +Do not use a measure as the unique key; a measure changes the projection kind and does not produce distinct-row expansion. If the model has no natural key, add an index column in Power Query or accept grouped input by design. Verify the effective row count with `print(len(dataset))` during development. + +Two byte caps the skill header omits: + +- Input is capped at 250 MB regardless of row count (Desktop and Service) +- Any single string value over 32,766 chars is silently truncated; validate with `dataset.apply(lambda c: c.str.len().max())` before parsing a long bound string + +`dataset` arrives in arbitrary order; sort inside the script, never rely on positional joins. diff --git a/plugins/reports/skills/r-visuals/SKILL.md b/plugins/reports/skills/r-visuals/SKILL.md index 4141530f..67fc30ce 100644 --- a/plugins/reports/skills/r-visuals/SKILL.md +++ b/plugins/reports/skills/r-visuals/SKILL.md @@ -1,6 +1,6 @@ --- name: r-visuals -version: 26.20 +version: 26.24 description: R visual creation and ggplot2 patterns for PBIR reports. Automatically invoke when the user mentions "R visual", "ggplot2", "ggplot in Power BI", or asks to "create an R visual", "add an R chart", "write an R visual script", "inject an R script into Power BI". --- @@ -133,7 +133,7 @@ Any locally installed R package works without restriction. R must be installed s | Cross-filter FROM | Not supported | Not supported | | Receive cross-filter | Yes | Yes | | Publish to web | Not supported | Not supported | -| Embed (app-owns-data) | Ending May 2026 | Ending May 2026 | +| Embed (app-owns-data) | Not supported | Not supported | ## Script Structure Template @@ -165,30 +165,36 @@ if (nrow(dataset) == 0) { } ``` -## R vs Python Comparison +## R vs Python Syntax Reference + +For the language-choice decision, see the "When to Use a Script Visual" section above. This table covers only mechanical syntax differences for scripts already committed to R: | Aspect | R (`scriptVisual`) | Python (`pythonVisual`) | |--------|-------|--------| -| Primary library | ggplot2 | matplotlib | | Render call | `print(p)` | `plt.show()` | | Column access | `dataset[,1]` or `dataset$col` | `dataset.iloc[:,0]` or `dataset["col"]` | | Empty guard | `if (nrow(dataset) == 0)` | `if len(dataset) == 0:` | -| Factor control | `factor(x, levels=...)` | `pd.Categorical(x, categories=...)` | +| Factor/category order | `factor(x, levels=...)` | `pd.Categorical(x, categories=...)` | | Runtime (Service) | R 4.3.3 | Python 3.11 | -## When to Use R Visuals +## When to Use a Script Visual + +Reach for an R visual only when **all** of the following hold: + +- The chart has no native equivalent and no reasonable Deneb spec +- The value is in a statistical computation that must run at render time (model fit, kernel density, forecast band), not just a shape Vega could draw +- The visual does not need to be a cross-filter source, hover tooltips, publish-to-web, or app-owns-data embed +- The report is served in a Pro/PPU or higher capacity with a Fabric-enabled region -R visuals are the preferred choice for **statistical and analytical visualizations**, particularly where R's statistical ecosystem excels. Use R visuals when you need: +If interactivity or cross-filtering matters, use **Deneb** (a static PNG cannot be a selection source). If the need is a small inline mark (sparkline, bar, status pill), use an **SVG measure** (no row cap, no timeout, no licensing/region gate, renders under publish-to-web). The script visual's niche is narrow: compute-at-render statistical plots for internal or org consumption. -- Distribution analysis (violin, ridgeline, density, boxplot) -- Statistical modeling (regression, correlation, ANOVA) -- Publication-quality analytical charts with ggplot2 -- Packages like forecast, corrplot, pheatmap that have no Python equivalent of equal quality +**R vs Python once a script visual is the right call:** use R for publication-quality statistical defaults and packages with no Python peer (`forecast`, `corrplot`, `pheatmap`, ridgeline/violin). Use Python when the computation leans on scikit-learn, statsmodels, or scipy, or when surrounding report logic is already Python. Where equal, default to whichever language the report's other scripts use; mixing doubles the publish-time package surface to validate. -**Output is static PNG** -- no cross-filtering FROM the visual, no hover/tooltip interactivity. Use Deneb instead for interactive custom visuals. Use SVG measures for simple inline graphics in tables/cards. +Do not default to a script visual because a chart type "looks statistical." A box plot, lollipop, or dumbbell is an SVG-measure or Deneb job; reserve scripts for charts that genuinely compute. ## References +- **`references/data-model.md`** -- `dataset` grouping mechanic, row/byte caps, forcing per-row input, and R-specific traps (Time type, text rendering flags, CJK fonts) - **`references/community-examples.md`** -- R Graph Gallery examples organized by chart type (distribution, correlation, ranking, evolution, flow) - **`references/ggplot2-patterns.md`** -- Common ggplot2 chart patterns (bar, donut, line, heatmap, bullet) - **`examples/script/`** -- Standalone R scripts (bar-chart, trend-line) -- ready to inject into visual.json after escaping diff --git a/plugins/reports/skills/r-visuals/references/data-model.md b/plugins/reports/skills/r-visuals/references/data-model.md new file mode 100644 index 00000000..1656a3c4 --- /dev/null +++ b/plugins/reports/skills/r-visuals/references/data-model.md @@ -0,0 +1,56 @@ +# R Visual Data Model + +Behavior of `dataset` and the row cap; traps that cause silent wrong input. + +## Distinct-row grouping + +The 150,000-row cap applies to the **deduplicated set**, not the raw fact table. Power BI groups the `dataset` exactly like a table visual before handing it to the script: identical rows across all bound `Values` columns collapse to one. A script bound to `Region, Category` over a million-row fact table receives at most as many rows as there are distinct `Region + Category` combinations. + +This is a property of the field bindings, not the script. Consequences: + +- Per-observation charts (jitter, strip, raw scatter, ECDF) silently receive aggregated input if no unique key is bound +- Scripts expecting row-level variation (bootstrapping, individual observations) produce wrong output without error +- The cap is hit much faster when a unique key is included; pre-filter at the visual or page level first + +To force per-row input, bind a guaranteed-unique column to the `Values` role alongside your other fields: + +```bash +pbir visuals bind Page/StripPlot -t Column -d "Values:Sales.Region" +pbir visuals bind Page/StripPlot -t Column -d "Values:Sales.Amount" +pbir visuals bind Page/StripPlot -t Column -d "Values:Sales.TransactionKey" +``` + +Do not use a measure as the unique key; a measure changes the projection kind and does not produce distinct-row expansion. If the model has no natural key, add an index column in Power Query or accept grouped input by design. Verify the effective row count with `nrow(dataset)` during development. + +Two byte caps the skill header omits: + +- Input is capped at 250 MB regardless of row count (Desktop and Service) +- Output is additionally capped at 2 MB on Desktop; a dense ggplot (many `ggrepel` labels, fine `geom_tile`) can blow it silently; reduce mark density first +- Any single string value over 32,766 chars is silently truncated + +`dataset` arrives in arbitrary order; sort inside the script (`reorder()`, `factor(levels=...)`) and never rely on positional joins. + +## R-specific traps (no Python equivalent) + +### Time data type unsupported + +A `Time` column (not `Date/Time`) errors the visual. Cast or model the field as `Date/Time` before binding; this is a model fix, not a script fix. + +### Text rendering in Service + +R text-based output requires `powerbi_rEnableShowText = 1` set as the first line of the script in Service. Desktop silently ignores it; Service silently drops text without it. This is the most common Desktop-passes / Service-fails trap for R visuals. + +### CJK fonts in Service + +CJK characters render blank in Service unless the script sets `powerbi_rEnableShowTextForCJKLanguages = 1` and loads `showtext`. Both flags must appear before any `library()` call: + +```r +powerbi_rEnableShowText = 1 +powerbi_rEnableShowTextForCJKLanguages = 1 +if (!requireNamespace("showtext", quietly = TRUE)) install.packages("showtext") +library(showtext) +showtext_auto() +library(ggplot2) +``` + +`showtext` is on the approved CRAN list; the `install.packages()` call works in Service. Placing either flag after `library()` or inside a function makes it a no-op. diff --git a/plugins/reports/skills/review-report/SKILL.md b/plugins/reports/skills/review-report/SKILL.md index 05879571..c8080243 100644 --- a/plugins/reports/skills/review-report/SKILL.md +++ b/plugins/reports/skills/review-report/SKILL.md @@ -1,6 +1,6 @@ --- name: review-report -version: 26.20 +version: 26.24 description: Actionable feedback on the quality, usage, and effectiveness of Power BI reports. Automatically invoke when the user asks to "review a report", "audit a report", "report usage analysis", "report health check", "find unused reports", "check if a report is being used", "assess report performance", "evaluate report quality". --- @@ -55,13 +55,13 @@ python3 scripts/get_report_distribution.py -w -r **Evaluate usage signals:** -- **Audience reach** is the most important metric: what percentage of users with access have actually viewed the report in the last 7, 28, and 60 days? See `references/distribution.md` for how to calculate reach and what the numbers mean. -- **View trends:** Is viewership stable, growing, or declining? Use the rolling 7D average (see `references/usage-metrics.md`). -- **Page view distribution:** Are views concentrated on one page or spread across the report? Concentration may indicate low-value pages. -- **Last visited:** When was the report last accessed by anyone? Tier 1 (Admin Activity Events, 30-day rolling, admin role required) is the official path. The Tier 3 DataHub V2 `lastVisitedTimeUTC` field (`--include-datahub`) is the non-admin cross-workspace fallback; flag to the user it's undocumented and can break. -- **Load times:** Are P50 and P90 load times acceptable for the audience? See `references/performance.md` for interpretation. +- **Audience reach** is the most important metric: what percentage of users with access have actually viewed the report in the last 7, 28, and 60 days? See `references/distribution.md` for how to calculate reach and what the numbers mean +- **View trends:** Is viewership stable, growing, or declining? Use the rolling 7D average (see `references/usage-metrics.md`) +- **Page view distribution:** Are views concentrated on one page or spread across the report? Before calling a page unused, check whether it is a tooltip/drillthrough target (no direct views expected) and confirm reachability via `pbir pages list` +- **Last visited:** When was the report last accessed by anyone? Tier 1 (Admin Activity Events, 30-day rolling, admin role required) is the official path. The Tier 3 DataHub V2 `lastVisitedTimeUTC` field (`--include-datahub`) is the non-admin cross-workspace fallback; flag to the user it's undocumented and can break +- **Load times:** Are P50 and P90 load times acceptable for the audience? See `references/performance.md` for interpretation -Do not use arbitrary thresholds for what constitutes "healthy" or "concerning"; these depend entirely on the report's audience, purpose, and lifecycle stage. A report for 3 analysts has different expectations than one for 300 executives. +Do not use arbitrary thresholds for what constitutes "healthy" or "concerning"; these depend entirely on the report's audience, purpose, and lifecycle stage. A report for 3 analysts has different expectations than one for 300 executives. Match the review window to the report's cadence before drawing conclusions; see `references/usage-interpretation.md` for common misreads of the modern Usage Metrics report and the retire/keep/redesign decision framework. **Subscriptions are not views.** Email subscriptions deliver report snapshots without generating view events. Check `admin/reports/{id}/subscriptions` (requires Fabric Admin) for active subscribers. A report with 0 views but active subscriptions is being consumed passively. @@ -119,7 +119,7 @@ python3 scripts/performance_audit.py -w -r See `references/performance.md` for percentile interpretation, DAX query inference from visual field bindings, and common anti-patterns. -**Key indicators:** P50 and P90 load times, visual count per page (loosely 12-15 max, but depends on complexity), extension measure count. See the reference for interpretation; do not apply rigid thresholds. +**Key indicators:** P50 and P90 load times, visual count per page, extension measure count. Visual count is a proxy; query cost per visual is the real driver. See `references/performance-audit.md` for the full cost model, how to interpret a Performance Analyzer export, DirectQuery report-layer tuning levers, and the interaction/navigation audit. Do not apply rigid visual-count thresholds without reading the cost model first. ### 5. Report Metadata and Governance @@ -142,13 +142,17 @@ Evaluate whether the report meets accessibility, organizational standards, and d **Accessibility:** -- [ ] Alt text present on data visuals -- [ ] Color contrast meets WCAG 2.1 AA (4.5:1 for text, 3:1 for UI) -- [ ] No reliance on color alone to convey meaning -- [ ] Font sizes legible (min 9pt for data, 12pt for labels) -- [ ] Tab order and visual layer order established -- [ ] No unnecessary animations or shadows -- [ ] Tested across different screens, browsers, and contexts +Most accessibility checks can be run statically against PBIR files; only focus traversal and screen-reader readout need a live report. See `references/accessibility-audit.md` for the full procedure: geometry/alignment audit, tab-order reconciliation, static pass (alt text, decorative items, color-only encoding), SVG-measure checks, script visual checks, and mobile readiness. + +Summary checklist: +- [ ] Alt text present on all data visuals (including SVG-measure hosts and script visuals) +- [ ] Decorative items removed from tab sequence (`tabOrder = -1`) +- [ ] Tab order matches geometric reading pattern (top-to-bottom, left-to-right) +- [ ] No color-only encoding (paired with shape, glyph, or text) +- [ ] Color contrast meets WCAG 2.1 AA (4.5:1 text, 3:1 UI elements) +- [ ] Font sizes legible (min 9pt data, 12pt labels) +- [ ] Mobile layout present on consumption-intended pages +- [ ] Live keyboard Tab traversal verified (cannot be confirmed from files) **Standards:** @@ -182,7 +186,7 @@ Ask the user: - Do they have access to the underlying semantic model? - Are they the developer of both the report and model, or only one? -If the semantic model is in scope, use the `review-semantic-model` skill in parallel. Many report issues (slow visuals, (Blank) values, missing fields) originate in the model. See `references/best-practices.md` for model symptoms that surface in reports. +If the semantic model is in scope, use the `semantic-model` skill in parallel. Many report issues (slow visuals, (Blank) values, missing fields) originate in the model. See `references/best-practices.md` for model symptoms that surface in reports. ### Step 1b: Determine Report Lifecycle Stage @@ -256,12 +260,15 @@ LOW ## References - **`references/usage-metrics.md`** -- Full documentation of all usage data APIs (official and undocumented) +- **`references/usage-interpretation.md`** -- Reading modern Usage Metrics correctly; retire/keep/redesign decision framework - **`references/distribution.md`** -- All report access paths and how to audit them - **`scripts/get_report_usage.py`** -- Workspace-level usage overview - **`scripts/get_report_detail.py`** -- Single report deep-dive (daily, per-viewer, per-page) - **`scripts/get_report_distribution.py`** -- Distribution audit (ACL, apps, publish-to-web) - **`scripts/performance_audit.py`** -- Load times + visual complexity analysis - **`references/performance.md`** -- Percentile interpretation, DAX query inference from visual metadata +- **`references/performance-audit.md`** -- Query cost model, Performance Analyzer export, DirectQuery tuning, interaction/navigation audit +- **`references/accessibility-audit.md`** -- Alignment/tab-order audit, static accessibility pass, SVG/script/mobile checks - **`references/report-metadata.md`** -- Thick/thin, endorsement, sensitivity, pipeline, model properties - **`references/export-to-excel.md`** -- Export activity analysis, data governance implications - **`references/best-practices.md`** -- Data visualization principles, chart selection, color, interaction design @@ -269,7 +276,7 @@ LOW ## Related Skills -- **`review-semantic-model`** -- Companion skill for semantic model review (run in parallel when model is in scope) +- **`semantic-model`** -- Companion skill for semantic model design and review (run in parallel when model is in scope) - **`pbi-report-design`** -- Detailed report design guidelines and layout rules - **`modifying-theme-json`** -- Theme authoring, compliance auditing, formatting promotion - **`deneb-visuals`**, **`python-visuals`**, **`r-visuals`**, **`svg-visuals`** -- Visual-specific review criteria diff --git a/plugins/reports/skills/review-report/references/accessibility-audit.md b/plugins/reports/skills/review-report/references/accessibility-audit.md new file mode 100644 index 00000000..258d3fbe --- /dev/null +++ b/plugins/reports/skills/review-report/references/accessibility-audit.md @@ -0,0 +1,101 @@ +# Accessibility Audit + +File-level checks that can be run from the terminal, plus a short residue pass that requires a live report. Keep the two sets separate; never assert "passes accessibility" from a file scan alone. + +## Alignment and Gutter Consistency + +A pure-geometry audit from the PBIR position blocks; run before any screenshot. + +```bash +pbir visuals properties "Sales.Report/Overview.Page/*" -s position --json +``` + +Build four sorted lists from the output: left `x`, right `x+width`, top `y`, bottom `y+height`. Any cluster of near-but-not-identical values is a misalignment candidate; snap to the modal value. Compute gutters as gaps between a row's right edges and the next left edge; flag rows where the spread exceeds a couple of pixels. Verify that distinct x-edges on row 1 match row 2 (continuous vertical lines). + +Round survivors to the grid unit with `pbir set .position.x `, then re-validate. + +Notes: +- `basicShape`, decorative `image`/`textbox`, and `actionButton` participate in the grid for alignment checks but should be excluded from visual-density counts +- A `visualGroup` child's `position` is relative to the group, which has its own `position` + `ScaleMode`; resolve group offsets before comparing or you get phantom misalignment +- Geometry alignment is necessary but not sufficient; pair with the tab-order check below + +## Tab Order vs Reading Pattern + +Spatial reading order (F/Z) and `tabOrder` (keyboard-tab and screen-reader announce sequence) are set independently; Power BI does not derive one from the other. A flawless Z-pattern can still ship a scrambled announced order. + +```bash +pbir visuals properties "Sales.Report/Overview.Page/*" -s tabOrder --json +``` + +Reconstruct intended order from geometry: sort by `(y_band, x)` where `y_band` buckets visuals into rows. Where a visual's `tabOrder` rank diverges from its geometric rank, the announced order fights the layout; flag it. + +Key conventions: +- A negative `tabOrder` (e.g. `-1`) removes an item from the tab sequence; Desktop writes this via the Selection pane. A meaningful visual with a negative value becomes unreachable by keyboard; flag unexpected negatives +- Decorative visuals should be removed from the tab sequence (`tabOrder = -1`); a decorative item still in sequence is a 1.3.2 risk +- Zero `tabOrder` set on every visual means the author never made the order intentional; that is a 1.3.2 risk, not a hard failure +- `position.z` (stacking order) is unrelated to `tabOrder`; never use z to fix the reading sequence +- Grouped visuals announce within their group + +## Static Accessibility Pass + +The following checks produce findings from `visual.json` alone: + +- **Missing alt text:** each data visual (exclude `basicShape`, decorative `image`/`textbox`, `actionButton`) must have non-empty `general.altText`; find gaps with `pbir visuals format "MyPage/*" -p general.altText` +- **Decorative items in tab sequence:** any `basicShape`, brand `image`, or divider with `tabOrder >= 0` should be `-1` +- **Tab-order coherence:** dump `(name, visualType, x, y, tabOrder)`; flag where ascending `tabOrder` does not track top-to-bottom, left-to-right +- **Color-only encoding:** a series/CF color with no paired data label, icon, or marker shape (WCAG 1.4.1) +- **Unreachable meaningful visual:** any data visual with a negative `tabOrder` + +Tag each finding `[Accessibility]` with its WCAG SC: 1.1.1 alt, 1.3.2/2.4.3 reading/focus, 1.4.1 color-only, 2.1.1 keyboard. + +## SVG-Driven Visuals + +Detect SVG measures by grepping `reportExtensions.json` expressions for `image/svg+xml`. Three checks not covered by the standard accessibility pass: + +- Each table/matrix/card/image whose bound measure returns an svg+xml URI must have non-empty `general.altText` (static or measure-driven) AND the SVG-encoded numbers must also appear as a readable adjacent column. Severity high for primary KPIs, medium for decorative micro-charts +- Flag SVG measures whose only context-varying element is a `fill`/`stroke` color with no accompanying shape, glyph, or text run (color-only encoding inside the SVG) +- Flag SVG measures that recompute base aggregations inside the string-builder (CONCATENATEX over fact rows rather than pre-aggregated values), and matrix value-axis SVGs lacking a `HASONEVALUE`/`ISINSCOPE` total guard (these are both an accessibility concern and a performance one) + +The Desktop Bridge screenshot does not confirm accessibility; alt text is invisible in a screenshot. + +## Script Visuals (Python/R) + +`pythonVisual` and `scriptVisual` have additional accessibility exposure because they render a static PNG with no data table fallback and no per-cell alt text slot. + +```bash +pbir visuals query --type pythonVisual --json +pbir visuals query --type scriptVisual --json +``` + +For each script visual found: +- Confirm the host visual has non-empty `general.altText` +- Flag the `Publish to web` distribution path: Python/R visuals render empty there; this is a blocking finding for public embeds +- Flag the `app-owns-data` embed path: R/Python visuals do not render in app-owns-data scenarios (not a future risk; broken now) +- Note the "Show visuals as tables" fallback (`Ctrl+Shift+F11`) does not include script visuals; the PNG is the only output + +Also audit the script literal: +```bash +pbir get 'Page/MyScriptVisual.Visual' objects.script +``` +An unreviewed third-party script is a security finding. + +## Mobile Readiness + +A page can be flawless on web and unusable in portrait. These are entirely file-based signals: + +- **Has a phone layout:** count `mobile.json` files under the page's `visuals/`; zero means the page falls back to rotated landscape on phones. Flag consumption-intended pages with none +- **Coverage vs over-stuffing:** ratio of mobile-placed to total visuals; near-zero on a key page is a gap, near-1.0 is the over-faithful-miniature anti-pattern +- **Stale placement:** a `visual.json` edited long after its `mobile.json` (heuristic; flag for manual review) +- **Orphan records:** a `mobile.json` whose sibling `visual.json` is gone (a blocking state; `pbir validate` catches it) +- **Mobile-only override sprawl:** large `objects` blocks in `mobile.json` that duplicate rather than delta desktop formatting + +Tie severity to intent: a back-of-house detail page with no phone layout is fine; a headline KPI page visible in an org app is high severity. Mobile-optimized views render only in the native iOS/Android apps; a browser always shows the landscape layout, so you cannot verify portrait by browser screenshot. + +## Must-Render Residue + +These cannot be confirmed from files; state them as not-file-verifiable: +- Tab/Shift+Tab focus traversal between regions (`Ctrl+F6`) +- Screen-reader readout via "Show data" (`Alt+Shift+F11`) and "Show visuals as tables" (`Ctrl+Shift+F11`) +- `card`/`slicer`/smart-narrative/Q&A/Key-Influencers/paginated are excluded from "Show visuals as tables"; the table fallback does not cover them + +Screenshots do not capture focus; the only check for traversal is a live keyboard pass. diff --git a/plugins/reports/skills/review-report/references/best-practices.md b/plugins/reports/skills/review-report/references/best-practices.md index e4633721..d9d8e3b0 100644 --- a/plugins/reports/skills/review-report/references/best-practices.md +++ b/plugins/reports/skills/review-report/references/best-practices.md @@ -148,7 +148,7 @@ Many report issues originate in the underlying semantic model. Ask the user: 1. Do they have access to the underlying semantic model? 2. Are they the developer of both the report and model, or only one of them? -If the model is in scope, use the `review-semantic-model` skill in parallel. The following are model-related issues that surface as report symptoms: +If the model is in scope, use the `semantic-model` skill in parallel. The following are model-related issues that surface as report symptoms: - (Blank) values from referential integrity violations (missing keys) or incorrect relationships - Repeating/inflated values from many-to-many or bidirectional relationships @@ -158,7 +158,7 @@ If the model is in scope, use the `review-semantic-model` skill in parallel. The - Refresh frequency not matching business needs - Unused columns/tables inflating model size -These are documented in detail in the `review-semantic-model` skill. For the report review, note these as symptoms and flag them for model-level investigation. +These are documented in detail in the `semantic-model` skill. For the report review, note these as symptoms and flag them for model-level investigation. ## Design for Agents diff --git a/plugins/reports/skills/review-report/references/performance-audit.md b/plugins/reports/skills/review-report/references/performance-audit.md new file mode 100644 index 00000000..085cb977 --- /dev/null +++ b/plugins/reports/skills/review-report/references/performance-audit.md @@ -0,0 +1,109 @@ +# Performance Audit + +Complements `performance.md` (which covers load-time telemetry and DAX query inference). This reference covers the query cost model, the Performance Analyzer export artifact, DirectQuery report-layer tuning, and the interaction/navigation audit. + +## Query Cost Model + +Visual count is a proxy; query cost per visual is the real driver. Opening a page refreshes every visual. Each emits at least one DAX query; several emit more. Parallelism is capped (DirectQuery `Maximum Connections per Data Source` defaults to 10; service capacity imposes additional limits), so total page-load latency grows non-linearly once the parallel cap is hit. + +Practical implication: 12 cheap card visuals can be fine; 8 matrices with totals and measure filters may not be. + +Visuals that emit more than one query (multipliers): +- Tables/matrices with totals/subtotals (one query per band; DistinctCount/Median are worst) +- Measure filters (two queries) +- Top N filters (two queries; can blow the 1M-row intermediate limit under DirectQuery) +- Field parameters (an extra evaluated-parameters phase) +- Custom/Deneb/Python/R visuals (display phase dominates) + +Review without Desktop: +```bash +pbir visuals query # read queryState per visual; flag measure-filters, TopN, totals, field params +``` + +Hidden and off-canvas visuals still query on load; include them in counts. Do not clear a flag based on visual count alone; a low-count page can be slow from one expensive visual. + +## Performance Analyzer Export + +Performance Analyzer (Desktop-only export) produces a JSON log of every recorded operation. It is the one perf artifact parseable from the terminal without a live model. It breaks each visual's wall time into named phases: + +```yaml +DAX query: model/measure work (or DirectQuery source query); fix belongs with the modeler +Direct query: only present for DirectQuery tables; confirms live source round-trip +Visual display: render time; high here + low DAX = report-layer cost (fixable) +Other: queuing/serialization; large value here is a serialization symptom +Evaluated parameters: field-parameter overhead +``` + +Workflow: generate timings in Desktop, export the JSON, map each `objectId`/`objectName` to a `visual.json` name, apply fixes in PBIR, re-validate. Use the documented export format from `microsoft/powerbi-desktop-samples` rather than reverse-engineering fields. + +When Desktop is unavailable, fall back to WABI `reportloads` telemetry (see `performance.md`) for absolute load times; use the Performance Analyzer export for relative attribution. + +Notes: +- Durations are queue-inclusive; a high number does not prove a visual is intrinsically slow. Isolate with single-visual refresh before drawing conclusions +- The capturing machine differs from the service; use the export to find the relatively-worst visual, not to promise a specific millisecond figure + +## DirectQuery Report-Layer Tuning + +Report-layer-only levers, all fixable inside `.Report` with `pbir-cli`. Separates "report agent can fix" from "kick to the modeler". Matters more under DirectQuery because every interaction is a live source round-trip (4-minute service timeout; 5s/30s usable/unusable guideline in practice). + +Detect DirectQuery from the report side via the Performance Analyzer "Direct query" phase, or confirm storage mode via model skills before applying. + +**Apply before binding fields:** write the `filterConfig` block before field bindings, or order `pbir filters` calls before `pbir visuals bind`. An unfiltered intermediate can hit the 1M-row limit. + +**Turn off unused totals/subtotals:** +```bash +pbir visuals format --property totals.show false +``` +These generate extra source queries; always extra cost for DistinctCount/Median. + +**Avoid measure filters and Top N on high-cardinality columns:** they generate two source queries and can exceed the 1M-row limit. If Top N is required, scope it tight and prefer a model-side aggregation. + +**Prefer single-select slicers, or gate multi-select behind an Apply button:** the highest-leverage fix; multi-select fires a query per item added. + +**Disable cross-highlight from slicers to expensive visuals:** +```bash +# set a NoFilter pair from slicer to matrix visual +``` +Use `visualInteractions[]` with type `NoFilter` for slicer-to-expensive-visual pairs. + +**Keep visuals-per-page low:** past the parallel-connection cap, visuals serialize and can show time-inconsistent results; this is a correctness argument, not just a speed one. + +**What not to change from `.Report`:** `Maximum Connections per Data Source` is a model setting; recommend raising it but do not attempt to set it from `.Report`. Auto page refresh multiplies everything by frequency and concurrent users; flag it if present. + +Do not blanket-apply DirectQuery tuning to Import reports; disabling cross-highlight removes interactivity for no gain. + +## Interaction and Navigation Audit + +Defects that are invisible in a screenshot and do not fail `pbir validate`. + +### Cross-filter graph + +Read each page's `visualInteractions`: +- Flag zero overrides on a page with both slicers and KPI cards; the cards likely jump on chart clicks (usually unintended) +- Flag a wall of `NoFilter` everywhere; interactivity may have been disabled rather than configured +- Flag overrides referencing visual `name`s that no longer exist (stale, no-op; find by resolving names against actual visuals on the page) + +Note: a `NoFilter` pair is sometimes correct (intentional KPI stability). Flag patterns and stale references, then ask the author about intent. + +### Drill propagation + +Grep `drillFilterOtherVisuals`: +- Flag drillable visuals where page-wide drill response was clearly intended but left `false` +- Flag the inverse (drill response active on a page where it would confuse users) + +Note: `drillFilterOtherVisuals` (hierarchy drill on same page) is distinct from drillthrough (page navigation). + +### Navigation integrity + +Collect every `visualLink` with `navigationSection`/`drillthroughSection`/`bookmark`, resolve each target against actual page/bookmark `name`s (not `displayName`): +- Flag dangling references +- Flag `WebUrl` pointing at non-HTTPS or empty URLs +- Flag navigator vs button sprawl: a row of near-identical `PageNavigation` buttons that a single `pageNavigator` would replace + +### Drillthrough hygiene + +- Confirm drillthrough pages are hidden in the view mode or carry a clear-filters reset +- Confirm a `Back` button exists on drillthrough pages +- A missing local target page is expected for cross-report drillthrough; do not flag it + +Resolve all targets against `name`, not `displayName`. Stale references resolve empty at runtime without a validation error. diff --git a/plugins/reports/skills/review-report/references/performance.md b/plugins/reports/skills/review-report/references/performance.md index 19d4d2eb..604794ff 100644 --- a/plugins/reports/skills/review-report/references/performance.md +++ b/plugins/reports/skills/review-report/references/performance.md @@ -155,4 +155,4 @@ The most expensive queries reveal the visuals that need optimization. Common fix - Reduce grouping columns or filter the data (page or report filters) (fewer dimensions = smaller result set) - Simplify or remove conditional formatting where not essential - Avoid or remove custom visuals that are overly-complex -- Audit the DAX and semantic model for issues there (see the semantic-models plugin and related skills like `review-semantic-model`) +- Audit the DAX and semantic model for issues there (see the semantic-models plugin and the `semantic-model` skill) diff --git a/plugins/reports/skills/review-report/references/usage-interpretation.md b/plugins/reports/skills/review-report/references/usage-interpretation.md new file mode 100644 index 00000000..f7c5a235 --- /dev/null +++ b/plugins/reports/skills/review-report/references/usage-interpretation.md @@ -0,0 +1,100 @@ +# Usage Interpretation + +Guidance for reading modern Usage Metrics data accurately and converting view counts into a retire/keep/redesign decision. Avoids the failure mode of drawing wrong conclusions from structural telemetry limitations. + +## Modern Usage Metrics: What the Numbers Actually Mean + +The modern Usage Metrics report has different metric definitions from the legacy report; treating them as equivalent produces wrong adoption verdicts. + +### Report View vs Report Page View + +- **Report View** (server-side): one event per report open, reliable, matches audit logs +- **Report Page View** (client-side): one event per page render + +Opening a report increments page views only for the landing page. "Page X has fewer page views than the report has views" is normal, not evidence the page is unused. + +### Structural Undercounting + +Page-view undercounting is structural: ad blockers, firewalls, offline sessions, and embedded scenarios drop client telemetry silently. Low page-view counts are a floor, not a ceiling. Conclude "unused page" only when the whole report is unused, not from a page-level view count alone. + +Before flagging an unused page, check: +- `pbir pages list` + navigators to confirm the page is reachable +- Whether the page is a tooltip page or drillthrough target (these legitimately have no direct views) + +### The `Blank` Page Entry + +`Blank` in the page slicer is not a real page; it represents pages added in the last 24h or since deleted. Do not investigate it. + +### Window and Retention + +- 30-day window, 30-day retention, daily refresh with up to 24h lag +- "No views" means no views in 30 days; that is the wrong window for quarterly or annual reports +- Archive via Analyze in Excel or a scheduled extract for longer trends + +### Blind Spots + +- App report pages and paginated reports are not in the Report pages table; absence of page views does not mean no engagement +- The platform slicer understates mobile/embedded usage precisely where client telemetry drops hardest; do not cite it as proof nobody uses mobile + +### Confidence Calibration + +Tag conclusions by confidence: +- Report-level verdicts (views, viewers, rank): high confidence +- Page-level verdicts: lower confidence; always add an explicit telemetry-loss caveat +- "No views in 30 days": medium confidence; check window vs cadence before acting + +Other notes: +- `Unnamed Users` is a tenant privacy setting, not missing data +- A modern-vs-legacy "drop" in metrics is a definition change, not a regression + +## Retire / Keep / Redesign Verdict + +Low usage is a question, not an answer. Use the following steps to turn view counts into a defensible action. + +### 1. Match the window to the cadence + +Read date-slicer grain, "as of" titles, and period filters to infer the report's cadence before applying a 30-day lens. An annual auditor model untouched for 11 months must be kept; a weekly operational report with no views in 30 days is a real signal. + +### 2. Separate reach from adoption + +Potential reach (people with access) vs actual reach (people who viewed it) are different numbers. A 500-person group with 8 viewers is an adoption problem, not necessarily a report problem. Investigate distribution and onboarding before touching the report. + +### 3. Filter the viewer list + +Strip before counting: +- Service principals (`principalType: App`) +- Report creators/owners (identified via `UpdateReportContent`/`CreateReport` activity events) +- IT/support/admin users whose only activity is maintenance access +- One-or-two-time viewers in a narrow window near the report's creation date (likely developers or testers) + +After filtering, work with the remaining consumer-only count. + +### 4. Honor exception classes + +Do not retire: +- Exec scorecards viewed by a small number of high-value users +- Compliance or audit content used intermittently but critical when needed +- Reports serving a seasonal cadence outside the 30-day window + +### 5. Read trend, not level + +A downward trend on a previously-used report is a stronger retire signal than flat-low with no prior history. + +### 6. Emit a verdict + +One of four outcomes: +- **Keep:** active reach, trend stable or rising, exception class, or cadence outside window +- **Investigate-distribution:** low actual reach relative to potential reach; distribution or onboarding issue +- **Redesign:** used but poorly (low per-viewer frequency, declining trend, design issues explaining the drop) +- **Retire:** low actual reach + downward trend + no exception class + confirmed by owner/SME + +Always require owner/SME confirmation as the final gate before retiring. + +### 7. Soft-retire for code-managed reports + +Rename with a deprecation prefix rather than delete; leave the definition in source control. The admin REST `Get Unused Artifacts as Admin` only looks back 30 days, so trend decisions need archived activity history. + +Notes: +- Never recommend deletion off a single 30-day "no views" +- Permission breadth is not consumption; a widely-shared report can still be unused +- A low-but-loyal personal/team-BI report with a small dedicated audience is a valid scenario, not a failure diff --git a/plugins/reports/skills/svg-visuals/SKILL.md b/plugins/reports/skills/svg-visuals/SKILL.md index 872d4115..1ed4dbe7 100644 --- a/plugins/reports/skills/svg-visuals/SKILL.md +++ b/plugins/reports/skills/svg-visuals/SKILL.md @@ -1,6 +1,6 @@ --- name: svg-visuals -version: 26.20 +version: 26.24 description: SVG generation via DAX measures and extension measures with ImageUrl data category for inline visualizations in PBIR reports. Automatically invoke when the user mentions "SVG visual", "DAX sparkline", "SVG measure", "inline graphics with DAX", "ImageUrl data category", "extension measure", or asks to create any DAX-generated chart (progress bars, bullet charts, KPI indicators, data bars, gauges, donut charts, lollipop charts, dumbbell charts, status pills, overlapping bars, boxplots, IBCS bars, jitter plots, box-and-whisker charts). --- @@ -22,13 +22,11 @@ Generate inline SVG graphics using DAX measures that return SVG markup strings. ## Supported Visuals -| Visual | visualType | Binding | Reference | -|--------|------------|---------|-----------| -| Table | `tableEx` | `grid.imageHeight` / `grid.imageWidth` | `references/svg-table-matrix.md` | -| Matrix | `pivotTable` | Same as table | `references/svg-table-matrix.md` | -| Image | `image` | `sourceType='imageData'` + `sourceField` | `references/svg-image-visual.md` | -| Card (New) | `cardVisual` | `callout.imageFX` | `references/svg-card-slicer.md` | -| Slicer (New) | `advancedSlicerVisual` | Header images | `references/svg-card-slicer.md` | +- Table (`tableEx`): `grid.imageHeight` / `grid.imageWidth` -- `references/svg-table-matrix.md` +- Matrix (`pivotTable`): same as table -- `references/svg-table-matrix.md` +- Image (`image`): `sourceType='imageData'` + `sourceField` -- `references/svg-image-visual.md` +- Card/New (`cardVisual`): `callout.imageFX` -- `references/svg-card-slicer.md` +- Slicer/New (`advancedSlicerVisual`): header images -- `references/svg-card-slicer.md` ## Workflow: Creating an SVG Measure @@ -216,19 +214,21 @@ VAR _Points = CONCATENATEX( ## Best Practices -1. **Check UDF libraries first** -- use DaxLib.SVG or MacGuyver Toolbox functions before writing custom DAX -2. **VAR pattern mandatory** -- one VAR per config value, one VAR per SVG element, assembly at the end -3. **Normalize all values** -- raw measure values must be scaled to SVG coordinate range -4. **HASONEVALUE guard** -- always guard against total/subtotal rows in table/matrix context -5. **`` sort trick** -- embed formatted value in `` for sortable SVG columns -6. **Use `viewBox`** for responsive scaling instead of fixed width/height -7. **Round coordinates** to 1-2 decimal places for performance -8. **Store as extension measures** -- SVG measures don't belong in the semantic model -9. **Use `display_folder`** to organize SVG measures together (e.g., `"SVG Charts"`) -10. **Preview first** -- save static SVG to `/tmp/`, open in browser, iterate before writing DAX -11. **Limit complexity** -- ~32K character limit on rendered SVG string (not DAX expression) -12. **Hex colors only** -- `#` directly, never `%23` URL encoding -13. **Image visuals need no `query` block** -- only `objects.image` with `sourceType='imageData'` and `sourceField` +- Check UDF libraries first -- use DaxLib.SVG or MacGuyver Toolbox functions before writing custom DAX +- VAR pattern mandatory -- one VAR per config value, one VAR per SVG element, assembly at the end +- Normalize all values -- raw measure values must be scaled to SVG coordinate range +- HASONEVALUE guard -- always guard against total/subtotal rows in table/matrix context; use `ISINSCOPE` for nested hierarchy levels +- `` sort trick -- embed formatted value in `` for sortable SVG columns +- Use `viewBox` for responsive scaling instead of fixed width/height +- Round coordinates to integers for performance (shorter strings, cheaper FORMAT calls) +- Store as extension measures -- SVG measures don't belong in the semantic model +- Use `display_folder` to organize SVG measures (e.g., `"SVG Charts"`) +- Preview first -- save static SVG to `/tmp/`, open in browser, iterate before writing DAX +- 32K character limit on the rendered SVG string per cell (not the DAX expression); see `references/svg-table-matrix.md` for diagnosis and mitigation +- Pre-aggregate in model measures -- let the storage engine cache aggregations; the SVG measure maps numbers to coordinates only +- Hex colors only -- `#` directly, never `%23` URL encoding +- Image visuals need no `query` block -- only `objects.image` with `sourceType='imageData'` and `sourceField` +- Accessibility: every SVG encoding primary data needs adjacent readable columns and a dynamic alt-text measure; see `references/svg-accessibility.md` ## reportExtensions.json Format @@ -251,11 +251,12 @@ VAR _Points = CONCATENATEX( ## Limitations -- **No interactivity** -- SVG images are static (no hover, click, tooltip) -- **No JavaScript** -- inline scripts are stripped -- **32K character limit** -- on the rendered SVG string, not the DAX expression. CONCATENATEX over 30+ rows easily exceeds this. Prefer polylines over individual dots, integer coordinates over decimals. Split complex designs into multiple simpler measures or use Deneb instead. -- **Performance** -- complex SVG with many elements impacts rendering -- **Classic card** does NOT support SVG -- use `cardVisual` instead +- No interactivity -- SVG images are static (no hover, click, tooltip) +- No JavaScript -- inline scripts are stripped +- 32K character limit per rendered cell string (not the DAX expression); `CONCATENATEX` over 30+ series points easily approaches this; prefer `` over individual shapes, integer coordinates, and pre-aggregated series; see `references/svg-table-matrix.md` for full diagnosis +- Per-cell formula-engine cost -- each visible cell evaluates the string-building expression; push aggregations into model measures so SVG assembly is coordinate-mapping only +- Accessibility gap -- screen readers receive no per-cell data from an SVG URI; mitigate with adjacent readable columns and dynamic alt text; see `references/svg-accessibility.md` +- Classic card (`card`) does NOT support SVG -- use `cardVisual` instead ## When to Use SVG Measures @@ -276,9 +277,13 @@ SVG measures are the preferred choice for **simple inline graphics** embedded in ### By Visual Type -- **`references/svg-table-matrix.md`** -- Patterns for Table/Matrix: data bar, bullet chart, dumbbell, overlapping bars, lollipop, status pill, sparkline, bar sparkline, area sparkline, UDF patterns. Includes axis normalization, sort trick, and image size configuration. -- **`references/svg-image-visual.md`** -- Patterns for Image visuals: KPI header, sparkline with endpoint, dashboard tile. Covers sourceType, Python API, and design considerations. -- **`references/svg-card-slicer.md`** -- Patterns for Card/Slicer: arrow indicator, mini gauge, mini donut, progress bar. Card binding via `callout.imageFX`. +- **`references/svg-table-matrix.md`** -- Patterns for Table/Matrix: data bar, bullet chart, dumbbell, overlapping bars, lollipop, status pill, sparkline, bar sparkline, area sparkline, UDF patterns; axis normalization, sort trick, image size configuration, and per-cell performance guidance +- **`references/svg-image-visual.md`** -- Patterns for Image visuals: KPI header, sparkline with endpoint, dashboard tile; sourceType binding, dynamic/conditional layout, responsive width guidance +- **`references/svg-card-slicer.md`** -- Patterns for Card/Slicer: arrow indicator, mini gauge, mini donut, progress bar, narrative sentence; card binding via `callout.imageFX` + +### Accessibility + +- **`references/svg-accessibility.md`** -- Accessibility for SVG measures: adjacent readable columns, dynamic alt text, color-only encoding, contrast requirements, and severity guidance for audit findings ### General diff --git a/plugins/reports/skills/svg-visuals/references/svg-accessibility.md b/plugins/reports/skills/svg-visuals/references/svg-accessibility.md new file mode 100644 index 00000000..c4091eec --- /dev/null +++ b/plugins/reports/skills/svg-visuals/references/svg-accessibility.md @@ -0,0 +1,102 @@ +# SVG Measures: Accessibility + +A DAX SVG measure renders as an image keyed off `ImageUrl`; the screen-reader path reads the visual's title, type, author alt text, and the "Show data" table. A cell whose value is an `svg+xml` URI contributes nothing meaningful to either path (Show-data lists the literal markup string), and there is no per-cell alt-text slot. Every inline SVG chart is an accessibility dead zone by default. + +## Mitigation at the host-visual level + +Apply both of the following for any SVG measure encoding primary KPI data; the second alone is sufficient for decorative micro-charts. + +### 1. Keep the numbers in adjacent readable columns + +For an SVG sparkline column, also bind the numeric measures the SVG encodes (min, max, last value; actual, target, variance) as plain value columns in the same table or matrix. Show-data then carries real numbers rather than markup strings, satisfying WCAG 1.1.1 for the data-table fallback. + +Decide per project which columns to expose vs hide at the visual level (they can be hidden from the visual but still present in the query so Show-data picks them up). + +### 2. Set dynamic alt text on the host container + +Author a sibling `_Alt` measure returning a spoken sentence (under ~250 characters) that narrates the SVG's content for the current filter context: + +```dax +KPI Sparkline Alt = +VAR _Last = [Sales Amount] +VAR _Trend = IF([Sales Amount] > [Sales Amount PY], "up", "down") +RETURN + "Sales: " & FORMAT(_Last, "$#,0,, M") & ", trending " & _Trend & " vs prior year" +``` + +Bind via `pbir set`: + +```bash +pbir set "MyPage/MyTable.Visual" "general.altText" --measure "_Report.KPI Sparkline Alt" +``` + +Or set directly in `visual.json` under `visualContainerObjects.general[0].properties.altText`: + +```json +"general": [{ + "properties": { + "altText": { + "expr": { + "Measure": { + "Expression": { "SourceRef": { "Schema": "extension", "Entity": "Sales" } }, + "Property": "KPI Sparkline Alt" + } + } + } + } +}] +``` + +The measure must guard `BLANK()` (a blank alt text is no better than none): + +```dax +IF(COUNTROWS(VALUES('Date'[Month])) > 0, + "Sales: " & FORMAT(_Last, "$#,0,, M") & "...", + "No data for the current selection." +) +``` + +For an `image` visual rendering a single SVG, the container has a `general.altText` slot; always set it. The static literal form is acceptable when the SVG is not filter-sensitive. + +### Decorative SVGs + +For purely decorative SVGs (dividers, brand backgrounds), mark the host hidden in tab order so screen readers skip it entirely: + +```bash +pbir set "MyPage/MyDecorativeSVG.Visual" "position.tabOrder" -1 +``` + +## Color-only encoding + +An SVG that conveys status purely via `fill`/`stroke` color fails WCAG 1.4.1. Pair every semantic color with at least one of: + +- A shape or glyph change (circle vs diamond, filled vs hollow) +- A text label or abbreviation inside the SVG +- A `` element inside the SVG markup itself + +The status pill pattern already does this (text label alongside fill color). The dumbbell and bullet patterns do not; add a symbol or label if they are primary status indicators. + +## Contrast + +SVG elements are subject to the same WCAG contrast requirements as rasterized content: +- Text inside the SVG: >= 4.5:1 against the cell background (3:1 for large text >= 18pt) +- Graphical elements (bars, lines, dots used for data): >= 3:1 against adjacent colors + +Theme tokens from the host report's theme do not automatically propagate into an SVG string; verify contrast manually, especially on dark themes or high-contrast accessibility themes. + +## What the Desktop Bridge screenshot does not confirm + +The Bridge screenshot is useful for layout verification; it does not confirm: +- Alt text is populated and readable +- Adjacent readable columns are present in Show-data +- Tab order positions the visual correctly for keyboard users + +Confirm accessibility by reading back the bound measure expression and auditing `visual.json` for `general.altText` and the presence of non-SVG bound columns. + +## Severity guidance for audit findings + +When surfaced by the `review-report` skill: + +- Primary KPI SVGs with no alt text and no adjacent numeric column: high severity +- SVG measures with color-only encoding and no paired shape or label: high severity +- Decorative micro-charts with no alt text: medium severity (low if clearly decorative and tab-order is -1) diff --git a/plugins/reports/skills/svg-visuals/references/svg-card-slicer.md b/plugins/reports/skills/svg-visuals/references/svg-card-slicer.md index bc7ba250..05e17909 100644 --- a/plugins/reports/skills/svg-visuals/references/svg-card-slicer.md +++ b/plugins/reports/skills/svg-visuals/references/svg-card-slicer.md @@ -102,6 +102,56 @@ RETURN "<text x='" & (_W / 2) & "' y='" & (_H / 2 + 5) & "' font-size='11' text-anchor='middle' fill='white' font-weight='bold'>" & _Label & "</text></svg>" ``` +### Pattern: Narrative Sentence + +When the story needs inline formatting that reacts to data (a clause that changes color on a miss, a bold figure, a verdict word keyed off performance), compose the whole sentence as a DAX measure returning an SVG with `<text>` and `<tspan>` runs. This is the only path to "X out of Y targets hit (~Z%)" where the number, color, and verdict word all key off data in a single declarative, version-controlled measure. + +Decision guide for narrative elements: + +- Plain "label: value" -> dynamic-value textbox run (simpler, inherits measure format string) +- Whole-string title with no per-clause styling -> expression-based DAX visual title +- Conditionally-styled clauses or sentence with inline micro-chart -> SVG narrative measure (below) +- Multi-paragraph AI summary -> Narrative visual (non-deterministic, license-gated, cannot be diffed; avoid for deterministic reporting) + +```dax +Performance Narrative = +VAR _Actual = [Sales Amount] +VAR _Target = [Sales Target] +VAR _Variance = DIVIDE(_Actual - _Target, _Target) +VAR _Hit = _Actual >= _Target +VAR _VerdictText = IF(_Hit, "on track", "behind") +VAR _VerdictColor = IF(_Hit, "#2D6A2E", "#982F2F") +VAR _ActualFmt = FORMAT(_Actual, "$#,0,, M") +VAR _TargetFmt = FORMAT(_Target, "$#,0,, M") +VAR _VarFmt = FORMAT(ABS(_Variance), "+0.0%;0.0%") + +RETURN +"data:image/svg+xml;utf8," & +"<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 400 30'>" & +-- Plain prefix +"<text x='4' y='20' font-family='Segoe UI' font-size='12' fill='#333'>" & +"Sales " & +-- Bold actual value, brand color +"<tspan font-weight='700' fill='#1f4e79'>" & _ActualFmt & "</tspan>" & +" vs target " & +"<tspan fill='#555'>" & _TargetFmt & "</tspan>" & +" -- " & +-- Verdict: color changes with performance +"<tspan fill='" & _VerdictColor & "' font-weight='700'>" & _VerdictText & " (" & _VarFmt & ")</tspan>" & +"</text>" & +"</svg>" +``` + +Key notes: + +- Each styled clause is a separate `<tspan>` carrying its own `fill` and `font-weight`; mix styled and unstyled runs in one `<text>` element +- Colors come from `IF`/`SWITCH` over data; use hex codes aligned with the report's theme tokens +- Format numbers inside the measure via `FORMAT()`; the SVG string ignores the measure's model format string +- The 32K limit and no-interactivity constraints apply; keep sentences concise +- The classic card (`card`) does not support `ImageUrl` callouts; use `cardVisual`, an `image` visual, or a table single-row image column + +For the binding, wire the measure to `callout.imageFX` in a `cardVisual` (see the binding block above), or to `sourceField` in an image visual. To embed a tiny inline sparkline mid-sentence, add a `<polyline>` element after the text run within the same SVG. + --- ## Slicer Visual (advancedSlicerVisual) diff --git a/plugins/reports/skills/svg-visuals/references/svg-image-visual.md b/plugins/reports/skills/svg-visuals/references/svg-image-visual.md index feb193e0..6714cb41 100644 --- a/plugins/reports/skills/svg-visuals/references/svg-image-visual.md +++ b/plugins/reports/skills/svg-visuals/references/svg-image-visual.md @@ -204,3 +204,68 @@ Disable default image effects to prevent unwanted styling: ### No Query Block Image visuals bound to SVG measures do not need a `query` block. The measure is evaluated directly via `sourceField`. Adding a query block can cause duplicate evaluations. + +--- + +## Dynamic and Responsive SVG + +### Responsive width (image and card only) + +Set the SVG root to `width='100%' height='100%'` and drive geometry off the `viewBox`; the host scales the rendered image to fill its container: + +```dax +VAR _Prefix = "data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' width='100%' height='100%' viewBox='0 0 300 60' preserveAspectRatio='xMidYMid meet'>" +``` + +This works in image visuals and card callouts. Table and matrix cells ignore percentage width; they use `grid.imageWidth` from the visual's `objects.grid`. Fix the `viewBox` geometry to the configured `imageWidth` when targeting table cells. + +SVG can scale to fill the host, but it cannot read the container width as a number, so true reflow (re-binning bars to available width) is not possible in a DAX SVG measure. Reach for Deneb when the layout must adapt to container width. + +### Conditional layout based on context + +Branch the entire element assembly on filter context so the measure emits a different shape depending on what is selected, keeping each branch under the 32K string ceiling: + +```dax +Conditional SVG = +VAR _Mode = + SWITCH( + TRUE(), + ISBLANK([Sales Amount]), "empty", + HASONEVALUE('Date'[Month]), "single", + "trend" + ) + +-- Assemble the data-URI prefix once, outside the branch +VAR _Prefix = "data:image/svg+xml;utf8,<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 300 60'>" +VAR _Suffix = "</svg>" + +-- Build only the branch that will be returned +VAR _EmptyMarkup = "<text x='10' y='35' font-family='Segoe UI' font-size='11' fill='#999'>n/a</text>" +VAR _BarMarkup = -- single-bar assembly (HASONEVALUE context) + "<rect x='10' y='20' width='" & INT(DIVIDE([Sales Amount], [Sales Target]) * 200) & "' height='20' fill='#448FD6'/>" +VAR _SparkMarkup = -- sparkline assembly (multi-period context) + "<polyline fill='none' stroke='#448FD6' stroke-width='2' points='" & + CONCATENATEX( + ADDCOLUMNS(VALUES('Date'[Month]), "@V", [Sales Amount]), + INT(RANKX(VALUES('Date'[Month]), 'Date'[Month], , ASC) * 25) & "," & + INT(50 - DIVIDE([@V], MAXX(VALUES('Date'[Month]), [Sales Amount])) * 40), + " ", 'Date'[Month] + ) & "'/>" + +VAR _Body = + SWITCH( + _Mode, + "empty", _EmptyMarkup, + "single", _BarMarkup, + _SparkMarkup + ) + +RETURN _Prefix & _Body & _Suffix +``` + +Key rules for conditional layout measures: + +- Always include a blank/empty-state branch so filter contexts with no data emit a readable fallback rather than a degenerate zero-width SVG or BLANK +- Assemble the data-URI prefix and `</svg>` suffix once outside the SWITCH so the empty branch cannot accidentally omit them +- Switch on report/page-level state for the whole measure (e.g., `HASONEVALUE` over a slicer's dimension); switching per row on a different condition confuses readers who see different chart metaphors in adjacent cells +- Subtotal and total rows should usually return the empty markup or `BLANK()` rather than a coarced single-value chart diff --git a/plugins/reports/skills/svg-visuals/references/svg-table-matrix.md b/plugins/reports/skills/svg-visuals/references/svg-table-matrix.md index 0dfbb955..39ff9c10 100644 --- a/plugins/reports/skills/svg-visuals/references/svg-table-matrix.md +++ b/plugins/reports/skills/svg-visuals/references/svg-table-matrix.md @@ -500,3 +500,57 @@ Available UDF libraries: - `SVG.Chart.Waterfall` -- waterfall chart with connectors See the PowerBI MacGuyver Toolbox and DaxLib.SVG libraries for more UDFs. + +--- + +## Performance in Table and Matrix Cells + +An SVG measure is a string-building DAX expression evaluated once per visible cell (per row in a table; per row x column intersection in a matrix), all in the single-threaded formula engine with no storage-engine acceleration for the string concatenation. Cost scales with iteration count, not byte size. + +### Pre-aggregate in model measures, not inside the SVG string + +A sparkline that runs `CONCATENATEX` over 24 monthly points for every row is ~720 point-iterations of string assembly per page refresh, re-evaluated on every cross-filter. Push the base aggregations into reusable model measures or a precomputed table so the storage-engine cache absorbs that work; the SVG measure should only map computed numbers to coordinates. + +### Prefer one `<polyline>` over N individual elements + +A single `CONCATENATEX` producing a `points='...'` string for `<polyline>` iterates once and produces one element. Replacing it with N `<circle>` or `<line>` elements runs N iterations and emits N elements for the parser. Use `<polyline>` for line/area sparklines; reserve individual shapes for endpoints or markers only. + +### Round coordinates to integers + +Shorter strings mean cheaper `FORMAT` calls and less markup: + +```dax +-- Preferred: integer coordinates +VAR _X = INT(DIVIDE(_Month - _XMin, _XMax - _XMin) * 100) +-- Avoid: floating-point strings like "47.3829..." +``` + +### The 32K limit is per rendered cell, not per expression + +The ceiling applies to the string returned by the measure for each individual cell. Diagnose by running `LEN([Your SVG Measure])` via `pbir model -q` for a worst-case category member. Near the ceiling, the cell silently drops to blank text. Options when approaching the limit: + +- Reduce the time-series window (12 months instead of 24) +- Split into two simpler measures rendered in adjacent columns +- Move the visualization to Deneb, which has no comparable string ceiling + +### The `HASONEVALUE`/`ISINSCOPE` total guard matters for both correctness and cost + +A matrix evaluates SVG measures at subtotal and grand-total rows too. Without a guard, the measure runs at a coarser grain and may return a meaningless or oversized SVG. Gate explicitly: + +```dax +IF( + HASONEVALUE('Product'[Category]), + -- SVG assembly + BLANK() +) +``` + +For nested matrices, `ISINSCOPE('Product'[SubCategory])` targets a specific hierarchy level and avoids building SVGs at every subtotal band. + +### Caching and volatile inputs + +Identical SVG strings for identical filter contexts hit the formula engine result cache. Inputs that change every evaluation defeat caching: `NOW()`, `TODAY()`, and unseed random jitter. The jitter-plot example uses a hash-based pseudo-random from a key column; preserve that pattern rather than introducing `RAND()`. + +### `grid.imageHeight`/`grid.imageWidth` and column width + +Setting large image dimensions in the visual's `objects.grid` does not change the formula-engine cost of generating the SVG, but it adds rasterization and paint cost in the renderer. Keep dimensions proportional to the cell content; avoid inflating them for visual effect. Auto-size-width ON with a wide SVG column triggers horizontal scroll and re-layout churn; set an explicit `columnWidth` in the visual formatting. diff --git a/plugins/semantic-models/.claude-plugin/plugin.json b/plugins/semantic-models/.claude-plugin/plugin.json index 813b41eb..48901044 100644 --- a/plugins/semantic-models/.claude-plugin/plugin.json +++ b/plugins/semantic-models/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "semantic-models", - "version": "26.20", + "version": "26.24", "description": "Get this plugin for agentic development and management of semantic models.", "author": { "name": "Kurt Buhler", diff --git a/plugins/semantic-models/agents/semantic-model-auditor.agent.md b/plugins/semantic-models/agents/semantic-model-auditor.agent.md index 2728cb18..668691c8 100644 --- a/plugins/semantic-models/agents/semantic-model-auditor.agent.md +++ b/plugins/semantic-models/agents/semantic-model-auditor.agent.md @@ -210,7 +210,7 @@ Perform the following checks, categorized by severity: ## Related Skills - **[`dax`](../skills/dax/)** — DAX performance optimization -- **[`review-semantic-model`](../skills/review-semantic-model/)** — Full model quality review +- **[`semantic-model`](../skills/semantic-model/)** — Full model design, build, and quality review - **[`standardize-naming-conventions`](../skills/standardize-naming-conventions/)** — Naming remediation ## Model Design Issues diff --git a/plugins/semantic-models/skills/dax/SKILL.md b/plugins/semantic-models/skills/dax/SKILL.md index 5a889b32..dbfc7f40 100644 --- a/plugins/semantic-models/skills/dax/SKILL.md +++ b/plugins/semantic-models/skills/dax/SKILL.md @@ -1,6 +1,6 @@ --- name: dax -version: 26.20 +version: 26.24 description: DAX performance optimization for semantic models. Automatically invoke when the user asks to "optimize DAX", "fix slow DAX", "DAX performance", "tune a measure", "debug a measure", "DAX anti-patterns", or mentions slow queries, server timings, or DAX authoring. --- @@ -18,7 +18,7 @@ Detailed reference files (progressive disclosure — consult as directed by the - **[`references/engine-internals.md`](./references/engine-internals.md)** — FE/SE architecture, xmSQL, compression/segments, SE fusion, trace diagnostics - **[`references/dax-patterns.md`](./references/dax-patterns.md)** — Tier 1 DAX patterns (DAX001–DAX021) + Tier 2 query structure (QRY001–QRY004) -- **[`references/model-optimization.md`](./references/model-optimization.md)** — Tier 3 model patterns (MDL001–MDL010) + Tier 4 Direct Lake (DL001–DL002) +- **[`references/model-optimization.md`](./references/model-optimization.md)** — Tier 3 model patterns (MDL001–MDL009) + Tier 4 Direct Lake (DL001–DL002) Trace capture and performance profiling: @@ -27,6 +27,6 @@ Trace capture and performance profiling: ## Related Skills -- [`review-semantic-model`](../review-semantic-model/) — Model auditing including DAX anti-patterns and best practices +- [`semantic-model`](../semantic-model/) — Model design, build, and auditing including DAX anti-patterns and best practices - [`connect-pbid` (pbi-desktop plugin)](../../../pbi-desktop/skills/connect-pbid/) — Trace capture, performance profiling, EVALUATEANDLOG debugging - [`lineage-analysis`](../lineage-analysis/) — Impact analysis before model changes diff --git a/plugins/semantic-models/skills/dax/references/dax-patterns.md b/plugins/semantic-models/skills/dax/references/dax-patterns.md index 17f2b639..27b18009 100644 --- a/plugins/semantic-models/skills/dax/references/dax-patterns.md +++ b/plugins/semantic-models/skills/dax/references/dax-patterns.md @@ -10,11 +10,15 @@ Tier 1 DAX patterns (DAX001-DAX021) and Tier 2 query structure patterns (QRY001- > **Autonomy: Auto-apply freely. Modify only measure/UDF definitions in the DEFINE block. Keep EVALUATE and SUMMARIZECOLUMNS grouping identical.** +> **Candidate optimizations, not guarantees:** Each pattern is a hypothesis to test. Cardinality, data layout, relationships, filters, storage mode, and engine folding can make a rewrite help, do nothing, or hurt. Apply one or more matching patterns, validate trace/runtime and semantic equivalence after each step, and continue iterating because one rewrite can expose the next optimization opportunity. Keep changes that help; revert changes that do not. + > **Prefer SUMMARIZECOLUMNS:** Fully supported inside measure definitions — earlier restrictions no longer apply. Use it to replace `ADDCOLUMNS`/`SUMMARIZE` patterns (DAX002), pre-materialize context transitions before iterating (DAX006), and cache repeated evaluations into a single virtual table (DAX003). Prefer it over `ADDCOLUMNS(VALUES(...), ...)` unless a specific scenario prevents it. ### DAX001: Use Simple Column Filter Predicates as CALCULATE Arguments -CALCULATE accepts simple boolean column predicates directly — these are more efficient than wrapping a table in FILTER (causes excessive materialization). Split `&&` into separate filter arguments. +**Identifier:** `FILTER(Table, ...)` as a filter argument, or combined predicates. Verify trace; simple cases can fold to `WHERE`. + +**Action:** Keep filters column-scoped. Replace table filters with boolean predicates, and split `&&` into separate filter arguments. **Anti-pattern — FILTER with table expression uses an iterator:** ```dax @@ -44,9 +48,11 @@ CALCULATETABLE( 'Sales', 'Sales'[Region] = "West", 'Sales'[Amount] > 1000 ) --- -### DAX002: Replace ADDCOLUMNS/SUMMARIZE with SUMMARIZECOLUMNS +### DAX002: Use SUMMARIZECOLUMNS for Grouped Calculations + +**Identifier:** grouped calculation built with `ADDCOLUMNS`/`SUMMARIZE`. Compare trace against a direct `SUMMARIZECOLUMNS` shape. -SUMMARIZECOLUMNS defines grouping + calculation in one step, enabling better SE fusion. Replace all ADDCOLUMNS/SUMMARIZE patterns. +**Action:** Use `SUMMARIZECOLUMNS` for grouping + calculation when row shape and filter semantics stay identical. **Anti-patterns:** ```dax @@ -63,9 +69,11 @@ SUMMARIZECOLUMNS ( 'Sales'[ProductKey], "Total Profit", [Profit] ) --- -### DAX003: Cache Repeated and Context-Independent Expressions in Variables +### DAX003: Cache Repeated Expressions in Variables -Evaluating the same measure multiple times or placing context-independent expressions inside iterators causes redundant SE queries. Cache in a variable. +**Identifier:** repeated measure or expression references. Verify with FE time, repeated SE requests, or cache matches. + +**Action:** Cache repeated or row-independent expressions in variables at the smallest safe scope. **Anti-pattern — repeated measure reference:** ```dax @@ -109,9 +117,11 @@ RETURN SUMX( 'Sales', 'Sales'[Quantity] * _AvgPrice * 1.1 ) --- -### DAX004: Remove Duplicate and Redundant Filters +### DAX004: Remove Redundant Filters + +**Identifier:** repeated predicates, duplicate filter tables, or redundant key-set filters. -Applying the same filter condition twice — whether as duplicate CALCULATE arguments or as a variable that restates an existing predicate — causes redundant SE evaluation. +**Action:** Keep one copy of each filter. Remove key sets or variables that restate an active predicate. **Anti-pattern — same predicate in CALCULATE + FILTER:** ```dax @@ -146,16 +156,18 @@ VAR Result = --- -### DAX005: SUMMARIZE with Complex Table Expression +### DAX005: Move Complex SUMMARIZE Inputs to CALCULATETABLE -Instead of using SUMMARIZE with complex table expressions as the first argument, wrap with CALCULATETABLE instead. +**Identifier:** `SUMMARIZE` starts from a filtered or computed table expression. + +**Action:** Keep grouping simple; move filters to an outer `CALCULATETABLE`. **Anti-pattern:** ```dax SUMMARIZE( - CALCULATETABLE('Sales', 'Sales'[Year] = 2023, 'Sales'[CustomerKey] IN SellingPOCs), + CALCULATETABLE('Sales', 'Sales'[Year] = 2023, 'Customer'[Segment] = "Enterprise"), 'Sales'[CustomerKey], - "DistinctSKUs", DISTINCTCOUNT('Sales'[StoreKey]) + "DistinctStores", DISTINCTCOUNT('Sales'[StoreKey]) ) ``` @@ -164,18 +176,20 @@ SUMMARIZE( CALCULATETABLE( SUMMARIZECOLUMNS( 'Sales'[CustomerKey], - "DistinctSKUs", DISTINCTCOUNT('Sales'[StoreKey]) + "DistinctStores", DISTINCTCOUNT('Sales'[StoreKey]) ), 'Sales'[Year] = 2023, - 'Sales'[CustomerKey] IN SellingPOCs + 'Customer'[Segment] = "Enterprise" ) ``` --- -### DAX006: Pre-Materialize Context Transitions with SUMMARIZECOLUMNS +### DAX006: Precompute Iterator Inputs with SUMMARIZECOLUMNS + +**Identifier:** iterator over `VALUES(...)` with `CALCULATE` or measure evaluation per row. -Materializing context transition results in SUMMARIZECOLUMNS and iterating over pre-calculated values can improve query plan. +**Action:** Precompute iterator rows and values with `SUMMARIZECOLUMNS`, then iterate the materialized result. **Anti-pattern:** ```dax @@ -198,9 +212,11 @@ SUMX( --- -### DAX007: Replace IF with INT for Boolean Conversion +### DAX007: Convert Boolean Tests Without IF -INT with boolean expressions avoids conditional logic callbacks that IF statements trigger. +**Identifier:** row-by-row `IF`/`SWITCH` boolean conversion. Trace may show `CallbackDataID` when it cannot fold. + +**Action:** Convert boolean-to-1/0 tests without `IF`; use a predicate + `COUNTROWS` when counting rows. **Anti-pattern:** ```dax @@ -218,7 +234,7 @@ SUMX( ) ``` -**When the result is a count of qualifying rows, eliminate the iterator and callback entirely with a simple predicate:** +**Count rows case:** eliminate the iterator and callback with a simple predicate. ```dax -- Anti-pattern: iterator + conditional = callback SUMX( 'Sales', IF('Sales'[Amount] > 1000, 1, 0) ) @@ -231,7 +247,9 @@ CALCULATE( COUNTROWS('Sales'), 'Sales'[Amount] > 1000 ) ### DAX008: Context Transition in Iterator -Context transition is powerful but expensive. Optimize by: +**Identifier:** measure reference or `CALCULATE` inside an iterator. Verify with FE time or repeated short SE events. + +**Action:** Reduce or remove iterator context transitions: 1. **Remove it completely:** ```dax @@ -239,23 +257,29 @@ Context transition is powerful but expensive. Optimize by: // Use: SUMX( 'Sales', 'Sales'[Unit Price] * 'Sales'[Quantity] ) ``` -2. **Reduce number of columns:** +2. **Iterate over a narrow key table:** ```dax -// Instead of: SUMX( 'Account', [Total Sales] ) -// Use: SUMX( VALUES ( 'Account'[Account Key] ), [Total Sales] ) +// Instead of: SUMX( 'Customer', [Total Sales] ) +// Use: SUMX( VALUES('Customer'[CustomerKey]), [Total Sales] ) ``` 3. **Reduce cardinality before iteration:** ```dax -// Instead of: SUMX( 'Account', [Total Sales] * 'Account'[Corporate Discount] ) -// Use: SUMX( VALUES ( 'Account'[Corporate Discount] ), [Total Sales] * 'Account'[Corporate Discount] ) +// Instead of: SUMX( 'Customer', [Total Sales] * 'Customer'[DiscountRate] ) +// Use only when grouping customers by DiscountRate preserves the result: +SUMX( + VALUES('Customer'[DiscountRate]), + [Total Sales] * 'Customer'[DiscountRate] +) ``` --- -### DAX009: Wrap SUMMARIZECOLUMNS Filters with CALCULATETABLE +### DAX009: Externalize SUMMARIZECOLUMNS Filters -Filters passed as direct arguments to SUMMARIZECOLUMNS inside measures can produce unexpected results. Move filters to a wrapping CALCULATETABLE instead. +**Identifier:** filter arguments inside `SUMMARIZECOLUMNS`. Verify materialization and result shape. + +**Action:** Move filters to a wrapping `CALCULATETABLE`. **Anti-pattern:** ```dax @@ -279,9 +303,11 @@ CALCULATETABLE ( --- -### DAX010: Apply Filters Using CALCULATETABLE Instead of FILTER +### DAX010: Push Table Filters with CALCULATETABLE + +**Identifier:** standalone `FILTER(...)` table where filter context could be applied directly. Simple cases can fold to `WHERE`. -CALCULATETABLE modifies filter context directly for better query plans. +**Action:** Use `CALCULATETABLE` so the filter applies before the table is consumed. **Anti-pattern:** ```dax @@ -295,9 +321,11 @@ CALCULATETABLE( 'Sales', 'Sales'[Year] = 2023 ) --- -### DAX011: Distinct Count Alternatives +### DAX011: Test DISTINCTCOUNT Alternatives -Depending on cardinality and data layout, moving DISTINCTCOUNT to SUMX(VALUES(),1) can improve performance by forcing FE evaluation. +**Identifier:** `DISTINCTCOUNT`; trace cue is `DCOUNT`, especially when cells have different filter sets. + +**Action:** Benchmark `DISTINCTCOUNT` against `SUMX(DISTINCT(), 1)`; keep only if equivalent and faster for the target visual. **Storage Engine Bound:** ```dax @@ -306,60 +334,60 @@ DISTINCTCOUNT('Sales'[CustomerKey]) **Formula Engine Bound (sometimes faster):** ```dax -SUMX(VALUES('Sales'[CustomerKey]), 1) +SUMX(DISTINCT('Sales'[CustomerKey]), 1) ``` --- -### DAX012: Use ALLEXCEPT Instead of ALL and VALUES Restoration +### DAX012: Preserve Filters Deliberately -When clearing filter context with ALL() and then restoring specific columns via VALUES(), ALLEXCEPT achieves the same in one operation. +**Identifier:** `ALLEXCEPT` or `REMOVEFILTERS/ALL + VALUES` used to preserve filters. Validate across report groupings. -**Anti-pattern:** -```dax -CALCULATE( [Total Sales], ALL('Sales'), VALUES('Sales'[Region]) ) -``` +**Action:** Choose the preservation form that matches how the retained value is filtered. -**Preferred:** +**Use `ALLEXCEPT` only when the preserved column is already directly filtered:** ```dax CALCULATE( [Total Sales], ALLEXCEPT('Sales', 'Sales'[Region]) ) ``` -> **Note:** Only valid when `'Sales'[Region]` is actively filtered. Without it, `VALUES` returns all regions (no-op restore) while `ALLEXCEPT` still clears other filters — the two forms are not equivalent, and `ALL + VALUES` is required. +**Keep `ALL/REMOVEFILTERS + VALUES` when the preserved value may be cross-filtered by another column:** +```dax +CALCULATE( [Total Sales], REMOVEFILTERS('Sales'), VALUES('Sales'[Region]) ) +``` --- -### DAX013: SWITCH/IF Branch Optimization in SUMMARIZECOLUMNS +### DAX013: Keep Branch Measures SE-Friendly -SWITCH/IF inside SUMMARIZECOLUMNS enables branch optimization — the engine evaluates only the matching branch. When this fails, it materializes a full cartesian product. Three things break it: +**Identifier:** `SWITCH`/`IF` chooses between measure branches. Verify FE time and unused branch work in trace/query plan. -1. **Multiple aggregations in one branch** — merge into single SUMX: `SUMX('Sales', 'Sales'[SalesAmount] - 'Sales'[TotalCost])` -2. **Mismatched data types across branches** — an implicit cast breaks the optimization; use explicit conversion: `CONVERT(SUM('Sales'[OrderQuantity]), CURRENCY)` -3. **Context transition inside a branch iterator** — a measure reference that requires a context transition (e.g., `SUMX(Sales, 'Sales'[Quantity] * [selection])`) forces a full crossjoin. If the measure is context-independent, cache it before the iterator: `VAR _UnitDiscount = [Unit Discount]` +**Action checklist:** +- Read the selector column directly filtered by the slicer/query; avoid hidden sort keys unless filtered directly. +- Keep branches SE-native: one simple aggregation or one fact iterator with row-level arithmetic, not multiple separate aggregations. +- Use one numeric type across branches; cast mixed branches explicitly. +- Avoid iterator context transition; lift row-independent measures into variables. + +For disconnected parameter tables, prefer field parameters or aligning the selector/filter column before making model metadata changes. --- -### DAX014: Use COUNTROWS Instead of DISTINCTCOUNT on Key Columns +### DAX014: Use COUNTROWS for Recognized Keys -Use when a column is a primary key (one-side of a relationship). +**Identifier:** `DISTINCTCOUNT` over a recognized unique key. Trace may compile to `COUNT`; verify before rewriting. -**Anti-pattern:** -```dax -DISTINCTCOUNT ( 'Product'[ProductKey] ) -``` +**Action:** Prefer `COUNTROWS` only when the counted column is a recognized unique key. -**Preferred:** -```dax -COUNTROWS ( 'Product' ) -``` - -For non-key columns where DISTINCTCOUNT is a bottleneck, see DAX011 for alternatives. +- Safe candidates: table key, or one-side key of a regular active relationship. +- Test first: inactive relationships, `USERELATIONSHIP`, or unmarked keys. +- Non-key/high-cardinality bottleneck → see DAX011. --- -### DAX015: Move Calculation to Lower Granularity +### DAX015: Iterate at the Required Grain -When an iterator scans a high-cardinality table but the calculation depends on a low-cardinality attribute, iterate over the attribute instead. +**Identifier:** iterator scans more rows than a repeated measure/context-transition dependency requires. Do not use for simple SE-native column sums. + +**Action:** Iterate the lowest distinct grain only when it removes repeated measure or context-transition work. **Anti-pattern:** ```dax @@ -375,26 +403,28 @@ SUMX( VALUES('Customer'[DiscountRate]), CALCULATE(SUM('Sales'[Amount])) * 'Custo --- -### DAX016: Experiment with Relationship Overrides via TREATAS and CROSSFILTER +### DAX016: Test Relationship Overrides Locally + +**Identifier:** bidirectional/M2M filter path, or `TREATAS`/`CROSSFILTER` override. Verify joins in xmSQL. -Relationship direction and filter propagation directly affect SE query plans. Sometimes bidirectional is faster; sometimes explicit filter propagation wins. Use TREATAS and CROSSFILTER to experiment without model changes. +**Action:** Test alternate filter propagation in the measure with `TREATAS`/`CROSSFILTER` before changing the model. **Example — replace bidirectional bridge with explicit filter:** ```dax CALCULATE( SUM('Sales'[Amount]), - CROSSFILTER('Customer'[CustomerKey], 'SportBridge'[CustomerKey], NONE), - TREATAS(VALUES('SportBridge'[CustomerKey]), 'Customer'[CustomerKey]) + CROSSFILTER('Customer'[CustomerKey], 'CustomerBridge'[CustomerKey], NONE), + TREATAS(VALUES('CustomerBridge'[CustomerKey]), 'Customer'[CustomerKey]) ) ``` --- -### DAX017: Apply Boolean Multiplier to Unblock Fusion +### DAX017: Align Scan Shape with Boolean Multipliers -**SE signal:** Near-identical SE queries on the same fact table that differ only by a column filter value or by per-measure `VAND` tuple predicates on the same column. +**Identifier:** sibling measures differ only by per-measure filters on the same fact. Trace may show near-identical SE scans differing only by `WHERE` values. -**Fix:** Replace the per-measure filter with `SUMX(KEEPFILTERS(ALL(Column)), expr * boolean)` to move the filter from SE to FE, making SE queries structurally identical across measures. +**Action:** Move the per-measure filter into a boolean multiplier so competing SE scans can share shape; verify fusion in trace. ```dax -- Anti-pattern: separate SE query per measure @@ -402,21 +432,24 @@ CALCULATE( SUM('Sales'[Amount]), 'Product'[Category] = "Bikes" ) CALCULATE( SUM('Sales'[Amount]), 'Date'[Date] = _dateAnchor ) CALCULATE( MAX('Sales'[DateKey]), 'Sales'[Metric] <> 0 ) --- Fix: boolean multiplier — structurally identical SE queries → engine fuses +-- Fix: boolean multiplier — structurally similar SE queries can fuse; verify in trace SUMX( KEEPFILTERS(ALL('Product'[Category])), CALCULATE(SUM('Sales'[Amount])) * ('Product'[Category] = "Bikes") ) SUMX( KEEPFILTERS(ALL('Date'[Date])), CALCULATE(SUM('Sales'[Amount])) * ('Date'[Date] = _dateAnchor) ) MAXX( ALL('Date'[Date]), CALCULATE(MAX('Sales'[DateKey])) * INT(NOT ISBLANK(CALCULATE(SUM('Sales'[Metric])))) ) ``` -`KEEPFILTERS` preserves external context; when the column is in the groupby, detail cells iterate only 1 row. Works with all aggregation types. - -**BLANK → 0 caveat:** the boolean pattern returns 0 instead of BLANK when no data exists. If `ISBLANK()` checks matter downstream, wrap: `VAR _r = SUMX(...) RETURN IF(_r = 0, BLANK(), _r)`. +- `KEEPFILTERS` preserves external context; column in the groupby → detail cells iterate 1 row. +- Best for additive (`SUM`-style) aggregations. +- Validate `MIN`/`MAX`/`AVERAGE`: injected 0 values can corrupt the result. +- BLANK caveat: returns 0 instead of BLANK when no data exists; wrap if downstream `ISBLANK()` checks matter. --- -### DAX018: Replace DIVIDE with Division Operator in Iterators +### DAX018: Keep Iterator Division SE-Native + +**Identifier:** `DIVIDE()` inside an iterator. Trace may show `CallbackDataID` when it cannot fold. -DIVIDE() includes divide-by-zero protection that forces FE callbacks inside iterators. Use the native `/` operator to keep the expression SE-native. Only use `/` when the denominator is guaranteed non-zero. If zero is possible, pre-filter: `CALCULATETABLE('Items', 'Items'[LocationAdjustment] <> 0)`. +**Action:** Use `/` only when the denominator is known non-zero; otherwise pre-filter zero denominators first. **Anti-pattern:** ```dax @@ -430,11 +463,13 @@ SUMX('Fact', 'Fact'[BaseAmount] * (RELATED('Items'[Discount]) / RELATED('Items'[ --- -### DAX019: Lift Time Intelligence to Outer CALCULATE for Vertical Fusion +### DAX019: Move Time Windows Outside Sibling Measures -TI functions (DATESYTD, DATEADD, etc.) break vertical fusion — each TI-modified measure gets its own SE query. Keep base measures TI-free and apply TI once in an outer wrapper. +**Identifier:** sibling measures each apply time-window filters. Verify whether trace shows separate SE scans. -> **Custom time intelligence (VAR-based predicates):** When measures use manual date anchoring via `CALCULATE(expr, Column = _var)` instead of built-in TI functions, DAX019 does not apply — see **DAX017** for the boolean multiplier workaround. +**Action:** Keep base measures time-window free; apply TI once in the outer `CALCULATE`. + +> **Custom time predicates:** `CALCULATE(expr, Column = _var)` does not match this rule; see **DAX017**. **Anti-pattern — each measure applies TI independently (no fusion):** ```dax @@ -452,9 +487,11 @@ MEASURE 'Sales'[Margin YTD] = --- -### DAX020: Unblock Horizontal Fusion by Lifting Filters +### DAX020: Keep Slice Measures Fusion-Friendly + +**Identifier:** slice measures differ only by simple filters or dynamic values. Verify whether trace fuses or separates scans. -Horizontal fusion merges SE queries that differ only by column-slice filter. It breaks when the filtered column is missing from groupby, or when table-valued / runtime-computed filters are applied per measure. Fix: keep only simple column-slice filters inside base measures; lift everything else (TI, dynamic variables) to an outer CALCULATE. +**Action:** Keep slice measures simple and literal; lift time-intelligence or dynamic filters to the combining measure. **Anti-pattern — TI inside each slice measure (no fusion):** ```dax @@ -469,17 +506,19 @@ MEASURE 'Sales'[Accessories] = CALCULATE ( SUM('Sales'[Amount]), 'Product'[Categ MEASURE 'Sales'[Combined YTD] = CALCULATE ( [Bikes] + [Accessories], DATESYTD('Date'[Date]) ) ``` -Same principle applies to runtime variable filters — move them to the consuming measure. See DAX017 when the filtered column is not in the groupby. +- Variable-driven slicers: leave base measures filter-free; put the dynamic predicate on the outer measure. +- Sliced column not in the groupby → see DAX017. --- -### DAX021: Pre-Compute and Join Instead of Filter Round-Trip +### DAX021: Join Precomputed Key Sets in FE -When a measure computes a qualifying key set from a filtered aggregation and then uses TREATAS or IN to filter a second aggregation by those keys, the outer SUMMARIZECOLUMNS context compounds the key filter with groupby columns — generating large tuple semi-joins (e.g., 500+ `(Brand, Key)` pairs in a single WHERE clause). The compound-tuple SE scan often dominates total query time. +**Identifier:** computed key set re-filters the same fact via `TREATAS`/`IN`. Trace cue is large `IN`/`INB`; `ININDEX` or compound tuples can also appear. -**SE signal:** `VertiPaqSEQueryEnd` with `DEFINE TABLE ... ININDEX` or `WHERE ... IN` containing hundreds of compound tuples. Single scan duration disproportionately high relative to others. +**Action:** Precompute both aggregations at the shared key grain, then join in FE. -**Fix:** Pre-compute both aggregations independently at the shared key grain, then join with NATURALINNERJOIN in the FE. The table expression used to build each side — `ADDCOLUMNS(VALUES(...), ...)`, `SUMMARIZECOLUMNS(...)`, etc. — does not matter; the key is that both sides share a common lineage column for the join. +- Avoid pushing computed key sets back to the fact scan with `TREATAS`/`IN`. +- Both sides must keep a shared lineage column for `NATURALINNERJOIN`. **Anti-pattern — TREATAS pushes key set back to SE, compounded by outer groupby:** ```dax @@ -510,7 +549,7 @@ VAR _Joined = NATURALINNERJOIN ( _Qualifying, _UnfilteredAgg ) VAR _Result = SUMX ( _Joined, [@Agg2] ) ``` -> **Why it works:** Each pre-computed table generates independent SE scans — clean, no tuple filters. NATURALINNERJOIN matches on the shared `'Fact'[Key]` lineage column in the FE, replacing the expensive compound-tuple SE round-trip with a fast in-memory join over small pre-materialized tables. +> **Goal:** replace the final key-set fact scan with precomputed tables and an FE join. Keep only if trace/runtime improves. --- @@ -519,7 +558,7 @@ VAR _Result = SUMX ( _Joined, [@Agg2] ) > **STOP — Requires user approval before applying any change. Explain the impact on query output and wait for explicit confirmation.** > **Scope: Desktop-Achievable Changes Only** -> +> > Every Tier 2 recommendation must map to an action the report author can perform in Power BI Desktop's UI. The agent optimizes the *generated* DAX query, but the user implements changes through the Desktop interface — not by editing DAX directly in the query pane. Examples of valid changes: > - **Changing the axis/groupby field** (e.g., swap `Calendar Date` for `Calendar Month` on a visual axis) > - **Removing or adding visual-level filters** (e.g., drop an unneeded slicer selection) @@ -529,11 +568,11 @@ VAR _Result = SUMX ( _Joined, [@Agg2] ) ### QRY001: Remove Unneeded Filters -Every filter adds a `WHERE` clause in xmSQL and may force an extra SE join. Users often apply global slicer or visual-level filters that don't actually affect the calculation being optimized. +Every filter adds a `WHERE` clause in xmSQL and may force an extra SE join. Users often apply slicer or visual-level filters that don't affect the calculation being optimized. **Detection:** `WHERE` clauses on columns not used in the measure logic, or filter variables that restrict to a single value (e.g., `Currency[Code] = "USD"` in a USD-only model). -**Fix:** Experiment — remove filters one at a time and re-run. If the result doesn't change, the filter might be unnecessary. Global filters that are needed across all visuals should be pushed to the data source (model-level change — see [Section 5](./model-optimization.md#section-5-tier-3-model-optimization-patterns)). +**Fix:** Remove filters one at a time and re-run; if the result doesn't change, the filter is unneeded. Filters needed across all visuals → push to the data source (model-level — see [Section 5](./model-optimization.md#section-5-tier-3-model-optimization-patterns)). ```dax -- Before: filter on Currency adds an SE join for no benefit @@ -551,7 +590,7 @@ SUMMARIZECOLUMNS ( 'Product'[Category], "Revenue", [Total Revenue] ) ### QRY002: Eliminate Report Measure Filters (__ValueFilterDM) -When a visual filters on a measure value (e.g., "Revenue > 1M"), Power BI generates a `__ValueFilterDM` variable that evaluates the measure twice — once for the filter check, once for display. Roughly doubles execution time. +When a visual filters on a measure value (e.g., "Revenue > 1M"), Power BI generates a `__ValueFilterDM` variable that can evaluate the measure twice — once for the filter check, once for display. **Detection:** `__ValueFilterDM` in the generated query. diff --git a/plugins/semantic-models/skills/dax/references/dax-performance-optimization.md b/plugins/semantic-models/skills/dax/references/dax-performance-optimization.md index 5d68e987..6da4fb3d 100644 --- a/plugins/semantic-models/skills/dax/references/dax-performance-optimization.md +++ b/plugins/semantic-models/skills/dax/references/dax-performance-optimization.md @@ -1,6 +1,6 @@ # DAX Performance Optimization Guide -Complete framework for optimizing DAX query performance: tier model, workflow phases, engine internals, trace diagnostics, and a full pattern catalog (DAX001–DL002). +Complete framework for optimizing DAX query performance: tier model, phased workflow, decision guide, trace diagnostics, and on-demand pattern routing. ## Reading Guide @@ -13,44 +13,35 @@ Always read these sections fully before starting any optimization session: - **[Phase 2: Optimization Iterations](#phase-2-optimization-iterations)** — apply, test, compare, iterate - **[Section 1: How the Engine Works](./engine-internals.md#section-1-how-the-engine-works)** — FE/SE architecture, xmSQL, segments, fusion - **[Section 2: Trace Diagnostics](./engine-internals.md#section-2-reading-and-diagnosing-traces)** — metrics, event waterfall, signal interpretation -- **[Section 3: Tier 1 — DAX Patterns](./dax-patterns.md#section-3-tier-1-dax-optimization-patterns)** — DAX001–DAX021 (auto-apply, no approval needed) ### Consult When Needed Read these only when directed by the Decision Guide or after Tier 1 is exhausted: +- **[Section 3: Tier 1 — DAX Patterns](./dax-patterns.md#section-3-tier-1-dax-optimization-patterns)** — DAX001–DAX021 — load only routed candidate patterns first - **[Section 4: Tier 2 — Query Structure](./dax-patterns.md#section-4-tier-2-query-structure-patterns)** — QRY001–QRY004 — requires user approval before applying -- **[Section 5: Tier 3 — Model Changes](./model-optimization.md#section-5-tier-3-model-optimization-patterns)** — MDL001–MDL010 — high caution, user approval, suggest model copy +- **[Section 5: Tier 3 — Model Changes](./model-optimization.md#section-5-tier-3-model-optimization-patterns)** — MDL001–MDL009 — high caution, user approval, suggest model copy - **[Section 6: Tier 4 — Direct Lake](./model-optimization.md#section-6-tier-4-direct-lake-optimization-patterns)** — DL001–DL002 — high caution, user approval, requires ETL/pipeline changes --- ## Decision Guide -Use to prioritize *where to start* within sections, not to skip them. Section 3 is always read in full — these signals tell you which patterns to try first. Sections 4–6 signals are escalation triggers; consult those sections only when the signal appears. +Use this table as a router into Section 3. Route by trace evidence when available; otherwise route by DAX shape and treat the match as a hypothesis until execution results confirm it. Load only the routed candidate patterns first; read the rest of Section 3 only if no signal matches or routed patterns are exhausted. Sections 4–6 signals are escalation triggers; consult those sections only when the signal appears. ### Section 3 — Where to Start (read all of §3) -| Signal | Start With | -|--------|------------| -| `CallbackDataID` or `EncodeCallback` in xmSQL | [DAX002](./dax-patterns.md#dax002-replace-addcolumnssummarize-with-summarizecolumns), [DAX007](./dax-patterns.md#dax007-replace-if-with-int-for-boolean-conversion), [DAX008](./dax-patterns.md#dax008-context-transition-in-iterator), [DAX018](./dax-patterns.md#dax018-replace-divide-with-division-operator-in-iterators) (highest priority) | -| `ADDCOLUMNS` or `SUMMARIZE` in measure expression | [DAX002](./dax-patterns.md#dax002-replace-addcolumnssummarize-with-summarizecolumns), [DAX006](./dax-patterns.md#dax006-pre-materialize-context-transitions-with-summarizecolumns) | -| `SUMMARIZE` with complex or filtered table as first argument | [DAX005](./dax-patterns.md#dax005-summarize-with-complex-table-expression) | -| `SUMX(VALUES(col), CALCULATE(...))` pattern in measure | [DAX006](./dax-patterns.md#dax006-pre-materialize-context-transitions-with-summarizecolumns) | -| Same measure evaluated multiple times | [DAX003](./dax-patterns.md#dax003-cache-repeated-and-context-independent-expressions-in-variables) | -| Duplicate or redundant `CALCULATE` filter predicates | [DAX004](./dax-patterns.md#dax004-remove-duplicate-and-redundant-filters) | -| `FILTER(Table, ...)` as `CALCULATE` argument, or `&&` joining predicates in single filter | [DAX001](./dax-patterns.md#dax001-use-simple-column-filter-predicates-as-calculate-arguments) | -| `ALL(table), VALUES(table[col])` in same `CALCULATE` | [DAX012](./dax-patterns.md#dax012-use-allexcept-instead-of-all-and-values-restoration) | -| Filter or `TREATAS` passed directly as `SUMMARIZECOLUMNS` argument (not wrapped in `CALCULATETABLE`) | [DAX009](./dax-patterns.md#dax009-wrap-summarizecolumns-filters-with-calculatetable) | -| SE rows far exceed final result count | [DAX010](./dax-patterns.md#dax010-apply-filters-using-calculatetable-instead-of-filter) | -| `DISTINCTCOUNT` in measure expression | [DAX011](./dax-patterns.md#dax011-distinct-count-alternatives), [DAX014](./dax-patterns.md#dax014-use-countrows-instead-of-distinctcount-on-key-columns) | -| Conditional logic (`IF`, `IIF`) or `DIVIDE()` inside row iterator | [DAX007](./dax-patterns.md#dax007-replace-if-with-int-for-boolean-conversion), [DAX018](./dax-patterns.md#dax018-replace-divide-with-division-operator-in-iterators) | -| `SWITCH` or `IF` as primary expression body in measure | [DAX013](./dax-patterns.md#dax013-switchif-branch-optimization-in-summarizecolumns) | -| Multiple SE queries hitting same fact table | [DAX019](./dax-patterns.md#dax019-lift-time-intelligence-to-outer-calculate-for-vertical-fusion) (vertical fusion), [DAX020](./dax-patterns.md#dax020-unblock-horizontal-fusion-by-lifting-filters) (horizontal), [DAX017](./dax-patterns.md#dax017-apply-boolean-multiplier-to-unblock-fusion) (boolean multiplier) | -| Near-identical SE queries on same fact table differing only by a column filter value or by per-measure `VAND` tuple predicates | [DAX017](./dax-patterns.md#dax017-apply-boolean-multiplier-to-unblock-fusion) | -| Bidirectional or M2M relationship causing unexpected SE join expansion, or existing `TREATAS`/`CROSSFILTER` in measure | [DAX016](./dax-patterns.md#dax016-experiment-with-relationship-overrides-via-treatas-and-crossfilter) | -| High-cardinality iterator (many distinct rows, low-cardinality attribute) | [DAX015](./dax-patterns.md#dax015-move-calculation-to-lower-granularity) | -| `TREATAS` or `IN` re-filtering same fact with a computed key set; or large compound-tuple semi-join in xmSQL | [DAX021](./dax-patterns.md#dax021-pre-compute-and-join-instead-of-filter-round-trip) | +| Route when trace shows | Or DAX shape shows | Start With | +|-----------------------|-------------------|------------| +| `CallbackDataID` / callback-like FE row work | `IF`/`SWITCH` or `DIVIDE()` inside row iterators; per-row context transition inside iterators; `ADDCOLUMNS`/`SUMMARIZE` extension patterns | [DAX002](./dax-patterns.md#dax002-use-summarizecolumns-for-grouped-calculations), [DAX007](./dax-patterns.md#dax007-convert-boolean-tests-without-if), [DAX008](./dax-patterns.md#dax008-context-transition-in-iterator), [DAX018](./dax-patterns.md#dax018-keep-iterator-division-se-native) | +| High FE time, many short SE events, or repeated cache hits | repeated measure/expression references; `SUMX(VALUES(col), CALCULATE(...))`; high-cardinality iterator with low-cardinality dependency | [DAX003](./dax-patterns.md#dax003-cache-repeated-expressions-in-variables), [DAX006](./dax-patterns.md#dax006-precompute-iterator-inputs-with-summarizecolumns), [DAX015](./dax-patterns.md#dax015-iterate-at-the-required-grain) | +| SE rows far exceed result rows, or FE filters a broad SE result | `FILTER(Table, ...)` as filter argument; combined predicates; complex `SUMMARIZE` source; filters inside `SUMMARIZECOLUMNS` | [DAX001](./dax-patterns.md#dax001-use-simple-column-filter-predicates-as-calculate-arguments), [DAX005](./dax-patterns.md#dax005-move-complex-summarize-inputs-to-calculatetable), [DAX009](./dax-patterns.md#dax009-externalize-summarizecolumns-filters), [DAX010](./dax-patterns.md#dax010-push-table-filters-with-calculatetable) | +| Multiple SE scans over the same fact with similar joins | sibling time-window measures; slice measures; `SWITCH`/`IF` branches choosing measures | [DAX019](./dax-patterns.md#dax019-move-time-windows-outside-sibling-measures), [DAX020](./dax-patterns.md#dax020-keep-slice-measures-fusion-friendly), [DAX013](./dax-patterns.md#dax013-keep-branch-measures-se-friendly) | +| Near-identical SE scans differ only by filter value | sibling measures differ only by per-measure filters on the same fact | [DAX017](./dax-patterns.md#dax017-align-scan-shape-with-boolean-multipliers) | +| Large `IN`/`INB`, `ININDEX`, or compound tuple predicates | `TREATAS`/`IN` re-filters the same fact with computed keys | [DAX021](./dax-patterns.md#dax021-join-precomputed-key-sets-in-fe) | +| `DCOUNT` in xmSQL | `DISTINCTCOUNT`, including distinct count over known unique key | [DAX011](./dax-patterns.md#dax011-test-distinctcount-alternatives), [DAX014](./dax-patterns.md#dax014-use-countrows-for-recognized-keys) | +| Result changes with grouping/filter context, or repeated predicates appear | `ALLEXCEPT`; `ALL/REMOVEFILTERS + VALUES`; duplicate predicates; redundant key-set filters | [DAX012](./dax-patterns.md#dax012-preserve-filters-deliberately), [DAX004](./dax-patterns.md#dax004-remove-redundant-filters) | +| Unexpected joins or expanded bridge/M2M paths | bidirectional/M2M relationship in filter path; `TREATAS`/`CROSSFILTER` in measure | [DAX016](./dax-patterns.md#dax016-test-relationship-overrides-locally) | > No signal matches? Read all of §3 — patterns DAX001–DAX021 cover the full range. @@ -65,7 +56,7 @@ Only consult these sections if the corresponding signal is present. All require | Tier 1 patterns exhausted; output change acceptable | §4 → [QRY001](./dax-patterns.md#qry001-remove-unneeded-filters)–[QRY004](./dax-patterns.md#qry004-remove-blank-suppression-changes-result-shape) | | Few SE queries, low parallelism, clean xmSQL, high SE duration | §5/§6 → [data layout](./model-optimization.md#section-5-tier-3-model-optimization-patterns) | | Many-to-many or bidirectional relationship overhead | §5 → [MDL001](./model-optimization.md#mdl001-many-to-many-relationship-optimization) | -| Direct Lake model + low parallelism or cold cache | §6 → [DL001](./model-optimization.md#dl001-v-ordering-for-optimal-vertipaq-compression)–[DL002](./model-optimization.md#dl002-segment-size-and-parallelism) | +| Direct Lake model + low parallelism or cold cache | §6 → [DL001](./model-optimization.md#dl001-v-ordering-delta-tables-for-direct-lake)–[DL002](./model-optimization.md#dl002-segment-size-and-parallelism) | --- @@ -80,27 +71,23 @@ Only consult these sections if the corresponding signal is present. All require | **Tier 3 — Model Changes** | Relationships, columns, agg tables, data types | High caution. Discuss trade-offs. Suggest model copy. Warn downstream risk. | | **Tier 4 — Direct Lake** | OneLake layout, V-ordering, rowgroup sizing | High caution. Requires ETL/pipeline changes outside the model. | -**Success criteria — Tier 1:** ≥10% duration improvement AND semantic equivalence (same row count, column count, data values). -**Success criteria — Tier 2/3/4:** ≥10% improvement AND explicit user approval of output or structural changes. +**Success criteria — Tier 1:** Query duration improvement AND semantic equivalence (same row count, column count, data values). +**Success criteria — Tier 2/3/4:** Query duration improvement AND explicit user approval of output or structural changes. ### Requirements -- **Semantic model connection** — Connect to the target semantic model before starting. For local Power BI Desktop models, use `connect-pbid`. For remote Fabric/XMLA models, use `powerbi-modeling-mcp` or an equivalent XMLA-capable tool. -- **Trace capture** — Requires the ability to execute DAX queries with server timing trace capture. See [Trace Capture Methods](#trace-capture-methods) below. -- **Model metadata** — Requires the ability to read measure definitions, function definitions, calculation group expressions, table metadata, and relationship metadata from the model. -- **Tier 2:** Present the change and its output impact, wait for user approval. -- **Tier 3/4:** Explain trade-offs, warn about downstream report risk, suggest working on a model copy, identify upstream changes (Lakehouse, Warehouse, Power Query) that may require changes beyond the semantic model itself. +- **Semantic model connection** — Any client that satisfies the Trace capture and Model metadata requirements below. See [Trace Capture Methods](#trace-capture-methods) for capability comparison across common clients. +- **Trace capture** — Ability to execute DAX queries with server timing trace capture. See [Trace Capture Methods](#trace-capture-methods). +- **Model metadata** — Ability to read measure definitions, function definitions, calculation group expressions, table metadata, and relationship metadata from the model. ### Trace Capture Methods -All methods use the same Analysis Services Trace API and produce identical trace events. - -| Method | Scope | Notes | -|--------|-------|-------| -| **`connect-pbid`** (PowerShell/ADOMD) | Local PBI Desktop | See [`performance-profiling.md`](../../../../pbi-desktop/skills/connect-pbid/references/performance-profiling.md). Derive FE/SE split manually. | -| **`powerbi-modeling-mcp`** (VS Code extension) | Local + remote (XMLA) | Returns pre-calculated FE/SE split, peak memory, result rows. Install: `code --install-extension analysis-services.powerbi-modeling-mcp` | -| **DAX Studio** | Local + remote | Server Timings pane. Manual, not scriptable. | -| **Fabric Workspace Monitoring** | Fabric workspaces | Built-in workspace-level query monitoring. | +| Method | Scope | Capture mode | How you drive it | Notes | +|--------|-------|--------------|------------------|-------| +| **`powerbi-modeling-mcp`** | Local PBI Desktop + remote (Fabric XMLA) | Live trace subscription | Tool calls (agent-friendly) | Returns pre-calculated FE/SE split, peak memory, and result rows. | +| **TOM Trace API (ADOMD.NET / PowerShell)** | Local PBI Desktop + remote (Fabric XMLA) | Live trace subscription | PowerShell / .NET scripts | Subscribe to `QueryEnd`, `VertiPaqSEQueryEnd`, `VertiPaqSEQueryCacheMatch` and derive FE/SE manually (`FE = TotalDuration − union(VertiPaqSEQueryEnd intervals)`; SE wall-clock is the union of overlapping intervals, not the sum). Direct Lake databases are not exposed via the PBI Desktop local AS proxy — connect to the Fabric workspace XMLA endpoint instead. | +| **DAX Studio** | Local PBI Desktop + remote (Fabric XMLA) | Live trace subscription | Interactive UI | Server Timings pane shows pre-calculated FE/SE. Best for hands-on investigation. | +| **Fabric Workspace Monitoring** (`SemanticModelLogs` Eventhouse table) | Fabric workspaces (Workspace Monitoring enabled) | Logged events, queried after the fact | KQL queries against the Eventhouse | Per-row `OperationName`, `DurationMs`, `CpuTimeMs`; correlate events for one query via `OperationId`. Best for after-the-fact production analysis at scale; not suited for tight iterate-and-rerun loops. | --- @@ -140,7 +127,9 @@ This context helps distinguish model design issues (missing star schema, bidirec For each run: -1. **Clear cache** — clear the model's VertiPaq cache to ensure cold-cache timing. +1. **Clear VertiPaq cache** — clears the SE query cache only; columns stay resident. + - Warm-up run: cold (on disk) → warm (resident). + - Measured runs: **warm + no-cache** — the ideal optimization-test state. 2. **Execute with trace capture** — run the DAX query with server timing trace enabled. 3. **Derive key metrics** — Total Duration, FE/SE split, SE query count, peak memory, and result row count. See [Understanding FE vs. SE Metrics](./engine-internals.md#understanding-formula-engine-fe-vs-storage-engine-se-metrics) for derivation from trace events. 4. Record all metrics, save the full trace events, and save the baseline result data for semantic equivalence checks. @@ -159,26 +148,16 @@ Apply **[Section 2: Trace Diagnostics](./engine-internals.md#section-2-reading-a ### Step 1: Select and Apply Optimizations -Using [Section 3 (Tier 1)](./dax-patterns.md#section-3-tier-1-dax-optimization-patterns), identify DAX patterns present in the baseline measures. Apply one or more of DAX001–DAX021. +Using [Section 3 (Tier 1)](./dax-patterns.md#section-3-tier-1-dax-optimization-patterns), start from trace identifiers when available; otherwise use the DAX-only fallback patterns as hypotheses. Apply one or more of DAX001–DAX021. **CRITICAL:** Modify only the **measure definitions in the DEFINE block**. Do NOT change the EVALUATE clause or SUMMARIZECOLUMNS grouping columns. Query structure must stay identical to preserve semantic equivalence. -```dax --- BASELINE measure -DEFINE - MEASURE Products[HighValueCount] = SUMX('Products', IF([Sales Amount] > 10000000, 1, 0)) - --- OPTIMIZED measure (DAX007: IF → INT) -DEFINE - MEASURE Products[HighValueCount] = SUMX('Products', INT([Sales Amount] > 10000000)) -``` - ### Step 2: Execute and Compare -1. Clear the model cache. +1. Clear the VertiPaq cache (returns the model to the warm + no-cache state — same condition as the baseline measured runs). 2. Execute the query with trace capture enabled. -**During iteration:** 1 run is sufficient — columns are already resident from baseline. Reserve the full protocol (1 warm-up + 3 measured, take median) for the **final confirmation** against the original baseline. +**During iteration:** 1 run is sufficient — columns are already resident from baseline, so no warm-up is needed; clearing only the SE cache keeps the warm + no-cache state. Reserve the full protocol (1 warm-up + 3 measured, take median) for the **final confirmation** against the original baseline. **Evaluate:** - **Improvement = (BaselineDuration − OptimizedDuration) / BaselineDuration × 100** @@ -186,11 +165,11 @@ DEFINE ### Step 3: Iterate and Escalate -- **≥10% improvement + semantically equivalent** → Success. Present optimized query and improvement to user. Offer to use it as new baseline for further rounds (compound improvements are common). -- **Further rounds:** When the user opts to continue, re-run Phase 1 Steps 3–4 on the new baseline. The optimized query has different structure — re-analyze against the Decision Guide and full pattern catalog. Patterns that didn't apply before (e.g., fusion opportunities, materialization candidates) may now be relevant. -- **<10% improvement** → Try another Section 3 pattern. Re-examine trace for additional bottlenecks. -- **Results differ** → Revert. The optimization changed calculation semantics. Try a different approach. -- **Tier 1 exhausted** → Move to Phase 3 (Tier 2) with user approval. +- **Meaningful improvement + semantically equivalent** → Success. "Meaningful" = exceeds the baseline's run-to-run noise band (e.g., baseline 1200/1280/1310 ms → 1200 ms is noise; 900 ms is real). Present to user; offer the optimized query as new baseline for further rounds (compound improvements are common). +- **Further rounds:** Re-run Phase 1 Steps 3–4 on the new baseline; re-analyze the new structure against the Decision Guide, as it may expose patterns that didn't apply before (fusion, materialization, etc.). +- **Within the noise band** → Try another Section 3 pattern, or combine patterns. Re-examine trace for other bottlenecks. +- **Results differ** → Revert; the optimization changed semantics. Try another approach. +- **Tier 1 exhausted** → Move to Phase 3 (Tier 2) with user approval. "Exhausted" = every signal-matching pattern tried (individually + combined), measures isolated for multi-measure queries, last 1–2 attempts at noise floor. --- @@ -213,7 +192,7 @@ Before applying any change: > **STOP — Do not modify the model without explicit user approval.** -Consult **[Section 5: Tier 3 — Model Patterns](./model-optimization.md#section-5-tier-3-model-optimization-patterns)** (MDL001–MDL010) and **[Section 6: Tier 4 — Direct Lake](./model-optimization.md#section-6-tier-4-direct-lake-optimization-patterns)** (DL001–DL002). +Consult **[Section 5: Tier 3 — Model Patterns](./model-optimization.md#section-5-tier-3-model-optimization-patterns)** (MDL001–MDL009) and **[Section 6: Tier 4 — Direct Lake](./model-optimization.md#section-6-tier-4-direct-lake-optimization-patterns)** (DL001–DL002). Before proceeding: @@ -237,11 +216,10 @@ Before proceeding: --- - ## Reference Files The detailed reference material is split into focused files for progressive disclosure: - **[Engine Internals](./engine-internals.md)** — FE/SE architecture, xmSQL, compression/segments, SE fusion, trace diagnostics (Sections 1-2) - **[DAX and Query Structure Patterns](./dax-patterns.md)** — Tier 1 DAX patterns DAX001-DAX021, Tier 2 query structure QRY001-QRY004 (Sections 3-4) -- **[Model and Direct Lake Optimization](./model-optimization.md)** — Tier 3 model patterns MDL001-MDL010, Tier 4 Direct Lake DL001-DL002 (Sections 5-6) +- **[Model and Direct Lake Optimization](./model-optimization.md)** — Tier 3 model patterns MDL001-MDL009, Tier 4 Direct Lake DL001-DL002 (Sections 5-6) diff --git a/plugins/semantic-models/skills/dax/references/engine-internals.md b/plugins/semantic-models/skills/dax/references/engine-internals.md index 2d883c32..e90df4af 100644 --- a/plugins/semantic-models/skills/dax/references/engine-internals.md +++ b/plugins/semantic-models/skills/dax/references/engine-internals.md @@ -16,9 +16,12 @@ The **FE** handles all DAX — branching logic, context transitions, complex ari The **SE** reads compressed columnar data from VertiPaq. It is **multi-threaded** and very fast, but supports only a limited set of operations: the four basic arithmetic operators, GROUP BY, LEFT OUTER JOINs, and basic aggregations (SUM, COUNT, MIN, MAX, DISTINCTCOUNT). -For **Direct Query models**, the SE role is played by the underlying data source (SQL, Spark, etc.). The FE generates SQL and pushes it down. The trade-off is network and source latency instead of in-memory scan cost. +For **DirectQuery models**, the data source serves as the SE role: the FE generates and pushes down SQL, trading network/source latency for in-memory scan cost. -**How they interact:** The FE requests data from the SE in one or more scans — each result is a **datacache** (a set of columns and aggregated values). Complex queries may require multiple datacaches: one to build a filter set, another to aggregate the fact. When the SE cannot evaluate an expression natively, it **calls back** to the FE row-by-row — making that SE scan effectively single-threaded. +**How they interact:** +- FE requests data via one or more SE scans; each result is a **datacache** (columns + aggregated values). +- Complex queries need multiple datacaches (e.g., one builds a filter set, one aggregates the fact). +- If the SE can't evaluate an expression natively → **callback** to FE row-by-row → that scan is effectively single-threaded. The core principle of DAX optimization: **push as much work as possible into the SE, minimize SE scans, and eliminate callbacks entirely.** @@ -38,17 +41,17 @@ FROM Sales LEFT OUTER JOIN Product ON 'Sales'[ProductKey] = Product[ProductKey] ``` -**Joins are always LEFT OUTER:** The many-side table is FROM, the one-side is joined in. +**Relationship joins are LEFT OUTER unless otherwise stated:** the many-side table is FROM, the one-side is joined in. Other internal forms (`INNER JOIN`, `REDUCED BY`, reverse joins) can also appear depending on the operation. **Semi-join projections:** Appear as `DEFINE TABLE $Filter0 ... ININDEX` in xmSQL — an initial dimension scan builds a key index injected into the fact WHERE clause. -**Callbacks:** Occur whenever the SE must compute an expression that exceeds VertiPaq's native capabilities — forcing row-by-row evaluation back in the FE. Example: `IF('Sales'[Amount] > 1000, 1, 0)` inside an iterator requires a callback because the SE cannot evaluate conditional logic. Replace with `INT('Sales'[Amount] > 1000)` to keep the expression SE-native. See [DAX002](./dax-patterns.md#dax002-replace-addcolumnssummarize-with-summarizecolumns), [DAX007](./dax-patterns.md#dax007-replace-if-with-int-for-boolean-conversion), [DAX008](./dax-patterns.md#dax008-context-transition-in-iterator), [DAX018](./dax-patterns.md#dax018-replace-divide-with-division-operator-in-iterators) for callback elimination patterns. +**Callbacks:** Occur whenever the SE must compute an expression that falls outside of VertiPaq's native capabilities. Forms include `CallbackDataID` (arbitrary expressions, most common) and specialized variants exposing individual FE functions (rounding, log/abs math). See [DAX002](./dax-patterns.md#dax002-use-summarizecolumns-for-grouped-calculations), [DAX007](./dax-patterns.md#dax007-convert-boolean-tests-without-if), [DAX008](./dax-patterns.md#dax008-context-transition-in-iterator), [DAX018](./dax-patterns.md#dax018-keep-iterator-division-se-native) for callback elimination patterns. --- ### Compression, Segments, and Parallelism -**Compression** determines scan speed. VertiPaq uses run-length encoding (RLE) and dictionary encoding. **V-ordering** reorders rows within segments to maximize RLE compression. Import models are V-ordered automatically. Direct Lake models are **not** — enable V-ordering explicitly (see [DL001](./model-optimization.md#dl001-v-ordering-for-optimal-vertipaq-compression)). +**Compression** determines scan speed. VertiPaq uses dictionary and run-length-style encodings to reduce scan work. For Direct Lake, source Delta/Parquet layout affects how quickly columns load into VertiPaq; V-Order improves RLE-friendly layout for read-heavy Power BI tables (see [DL001](./model-optimization.md#dl001-v-ordering-delta-tables-for-direct-lake)). **Segments** are fixed-size column chunks — the unit of both compression and parallel execution. The SE assigns one CPU thread per segment, so segment count determines how many cores a scan can utilize. @@ -56,30 +59,26 @@ FROM Sales **Segment skew matters equally:** if one segment has 15M rows and the rest have 1M, the scan bottlenecks on the oversized segment. Segments must be evenly sized for parallelism to be effective. -**Diagnosing low parallelism:** The **SE Parallelism Factor** (StorageEngineCpuTime ÷ StorageEngineDuration) shows thread utilization. Values near 1.0 mean single-threaded execution; values of 8–16 indicate strong multi-core use. When a trace shows few SE queries (1–4), high SE Duration, Parallelism Factor ≈ 1.0, and clean xmSQL — the bottleneck is too few segments or skewed segment sizes. This cannot be fixed with DAX; the fix is data layout (see [General Data Layout Best Practices](./model-optimization.md#section-5-tier-3-model-optimization-patterns) and [DL001–DL002](./model-optimization.md#section-6-tier-4-direct-lake-optimization-patterns)). +**Diagnosing low parallelism:** The **SE Parallelism Factor** (StorageEngineCpuTime ÷ StorageEngineDuration) shows thread utilization. Values near 1.0 mean single-threaded execution; values of 8–16 indicate strong multi-core use. When a trace shows few SE queries (1–4), high SE Duration, Parallelism Factor ≈ 1.0, and clean xmSQL — the bottleneck is likely too few segments or skewed segment sizes. DAX changes are unlikely to help; use data layout instead (see [General Data Layout Best Practices](./model-optimization.md#section-5-tier-3-model-optimization-patterns) and [DL001–DL002](./model-optimization.md#section-6-tier-4-direct-lake-optimization-patterns)). --- ### SE Query Fusion -Fusion is the engine's ability to combine multiple SE scans into fewer scans. There are two types: +Fusion is the engine's ability to combine multiple SE scans into fewer scans. Two flavors: -**Vertical fusion** merges multiple measure aggregations that share the same filter context into a single SE query. Three measures on the same fact table under the same filter = one scan instead of three. Gain scales with fact table size. +- **Vertical fusion** merges multiple measure aggregations that share the same filter context into a single SE query. Three measures on the same fact table under the same filter = one scan instead of three. Gain scales with fact table size. +- **Horizontal fusion** merges SE queries that differ only in which value(s) of a column they filter. N separate fact scans collapse to one; the FE partitions the result. -**What blocks vertical fusion:** -- **Time intelligence functions** (DATESYTD, DATEADD, SAMEPERIODLASTYEAR, etc.) — each TI-modified measure needs its own date-filtered SE scan → see [DAX019](./dax-patterns.md#dax019-lift-time-intelligence-to-outer-calculate-for-vertical-fusion) -- **Per-measure filter predicates** — can cause the FE to materialize separate `VAND` tuple predicates per measure, producing structurally different SE queries even when the underlying logic is identical → see [DAX017](./dax-patterns.md#dax017-apply-boolean-multiplier-to-unblock-fusion) -- **SWITCH/IF selecting between measures** — engine cannot determine at plan time which aggregation to include -- **Calculation group items** applying different filter modifications — each generates its own SE query +**Why fusion breaks.** Fusion needs the competing SE requests to have compatible scan shapes — the same filter context for vertical fusion, compatible single-column filter differences for horizontal. Common triggers that break that: -**Horizontal fusion** merges SE queries that differ only in which single value of a column they filter. N separate fact scans collapse to one; the FE partitions the result. +- **FE chooses the branch** — SWITCH/IF between measures, or per-measure filter predicates that materialize separate `VAND` tuples → structurally different SE queries → see [DAX017](./dax-patterns.md#dax017-align-scan-shape-with-boolean-multipliers) +- **Table- or range-valued filter** — time intelligence (DATESYTD, DATEADD, etc.) injects a per-measure date/range scan the SE can't fold in → see [DAX019](./dax-patterns.md#dax019-move-time-windows-outside-sibling-measures) +- **Slicing column not in the groupby** — horizontal fusion can only merge slices the result groups by; absent that, scans stay separate +- **Runtime-computed filter value** — a predicate held in a variable is treated as dynamic and won't fuse +- **Calculation group items** — each item applies its own filter modification → structurally different SE query -**What blocks horizontal fusion:** -- **Filtered column not in groupby** — engine cannot merge slices if the slicing column is absent from the groupby -- **Table-valued filter per measure** (e.g., time intelligence) — prevents slice merging even when column filters are identical -- **Filter value computed at runtime** (stored in a variable) — engine treats it as dynamic and will not fuse - -**Trace diagnosis:** Multiple SE queries hitting the same fact table with same joins → vertical fusion blocked. N near-identical SE queries with only the WHERE filter differing → horizontal fusion blocked. See [DAX patterns](./dax-patterns.md) and [Section 2 trace analysis](#section-2-reading-and-diagnosing-traces). +**Trace diagnosis:** Same fact table + same joins across multiple SE queries → missed vertical fusion. Near-identical queries differing only by `WHERE` values → blocked horizontal fusion. See [Section 3](./dax-patterns.md#section-3-tier-1-dax-optimization-patterns) and [Section 2 trace analysis](#section-2-reading-and-diagnosing-traces). --- @@ -165,3 +164,11 @@ Fusion is blocked, callbacks are present, or filters resolve iteratively. Fix th **Few SE queries + low FE time + high SE duration + low parallelism → Data layout problem** The DAX is clean but SE scans are slow due to insufficient segments or poor compression. DAX changes will not help — see [Section 5](./model-optimization.md#section-5-tier-3-model-optimization-patterns) / [Section 6](./model-optimization.md#section-6-tier-4-direct-lake-optimization-patterns) (General Data Layout Best Practices, DL001–DL002). + +--- + +## Further Reading + +- [Analysis Services trace events](https://learn.microsoft.com/analysis-services/trace-events/analysis-services-trace-events) +- [Horizontal fusion in Power BI and Analysis Services](https://powerbi.microsoft.com/en-us/blog/announcing-horizontal-fusion-a-query-performance-optimization-in-power-bi-and-analysis-services/) +- [Understand Direct Lake query performance](https://learn.microsoft.com/fabric/fundamentals/direct-lake-understand-storage) diff --git a/plugins/semantic-models/skills/dax/references/model-optimization.md b/plugins/semantic-models/skills/dax/references/model-optimization.md index 2d84dd27..c8aed3a4 100644 --- a/plugins/semantic-models/skills/dax/references/model-optimization.md +++ b/plugins/semantic-models/skills/dax/references/model-optimization.md @@ -1,6 +1,6 @@ # Model and Direct Lake Optimization -Tier 3 model patterns (MDL001-MDL010) and Tier 4 Direct Lake patterns (DL001-DL002). +Tier 3 model patterns (MDL001-MDL009) and Tier 4 Direct Lake patterns (DL001-DL002). > **Related references:** [Engine Internals](./engine-internals.md) · [DAX and Query Structure Patterns](./dax-patterns.md) @@ -8,7 +8,7 @@ Tier 3 model patterns (MDL001-MDL010) and Tier 4 Direct Lake patterns (DL001-DL0 ## Section 5: Tier 3 Model Optimization Patterns -> **STOP — Requires user approval before applying any change. Warn that model changes can break downstream reports. Suggest working on a model copy. Implement via `powerbi-semantic-model` skill; upstream source changes (Lakehouse, Warehouse, Power Query) require `fabric-cli` or pipeline coordination.** +> **STOP — Requires user approval before applying any change. Warn that model changes can break downstream reports. Suggest working on a model copy. Apply changes through whatever semantic-model authoring path is already in use; upstream source changes (Lakehouse, Warehouse, Power Query) must be coordinated with the user's data engineering or pipeline workflow.** ### General Data Layout Best Practices @@ -131,34 +131,24 @@ CASE WHEN SaleDate >= DATEADD(year, -1, GETDATE()) THEN SalesKey ELSE 'Historica --- -### MDL010: Set IsAvailableInMDX on Disconnected Slicer Tables - -Disconnected slicer tables (e.g., a `'Reporting Scenario'[Scenario]` parameter table with no model relationship) are commonly used with `SELECTEDVALUE` inside `IF`/`SWITCH`. When the slicer has no active selection, `SELECTEDVALUE` returns BLANK. With `IsAvailableInMDX = false`, the engine cannot determine this statically — it queries the table and generates two evaluation branches even though only one will execute. With `IsAvailableInMDX = true`, the engine statically resolves the unfiltered state and eliminates the dead branch without an extra SE scan. - -> **Scope:** This optimization only applies when the slicer column is **unfiltered**. When a selection is active, the branch is always evaluated regardless of this property — the static resolution path is not available. - ---- - ## Section 6: Tier 4 Direct Lake Optimization Patterns > **STOP — Requires user approval before applying any change. Changes here require Spark/ETL jobs or Fabric resource profile configuration outside the semantic model. Coordinate with the user's data engineering workflow.** -Direct Lake reads from OneLake Delta Parquet files instead of importing. Import-like speed when data is memory-resident, but unique characteristics around cold cache and segment loading. +Direct Lake reads OneLake Delta/Parquet files and loads columns into VertiPaq on demand. Its speed depends on source file layout, memory residency, and segment health. -### DL001: V-Ordering for Optimal VertiPaq Compression +### DL001: V-Ordering Delta Tables for Direct Lake -Import models are always V-ordered. Direct Lake models are **not** — enable it explicitly. V-ordering reorders rows within each rowgroup to maximize RLE compression (2–5× improvement). +Import refresh builds optimized VertiPaq storage. Direct Lake depends on Delta/Parquet layout at query time, so use V-Order for read-heavy Power BI tables to improve compression and column loading. Two approaches: -- **Spark:** `spark.conf.set("spark.microsoft.delta.vorder.enabled", "true")` then run `OPTIMIZE`. -- **Fabric resource profile:** Use the [`readHeavyForPBI` resource profile](https://learn.microsoft.com/en-us/fabric/data-engineering/configure-resource-profile-configurations) which enables V-ordering and optimized write settings automatically. +- **Spark:** `spark.conf.set("spark.sql.parquet.vorder.default", "true")`, then run `OPTIMIZE` for existing Delta tables. +- **Fabric resource profile:** Use the [`readHeavyForPBI` resource profile](https://learn.microsoft.com/en-us/fabric/data-engineering/configure-resource-profile-configurations), which enables V-Order-oriented write settings for Power BI reads. --- ### DL002: Segment Size and Parallelism -Delta rowgroups map directly to VertiPaq segments — one segment per CPU core. More segments = better CPU saturation (see SE Parallelism Factor in [Section 1](./engine-internals.md#section-1-how-the-engine-works)). +Parquet row groups shape VertiPaq column segment size/count (see SE Parallelism Factor in [Section 1](./engine-internals.md#section-1-how-the-engine-works)). **Target: 1–16M rows per rowgroup.** Too few rowgroups → single-threaded scans; too many tiny rowgroups → merge overhead. For small tables (< 1M rows) this rarely matters. Run `OPTIMIZE` regularly to consolidate small files into properly sized rowgroups. - -Maximize available cores by choosing a capacity SKU that matches table size — a table with 2 segments on an F64 wastes most of its parallelism budget. \ No newline at end of file diff --git a/plugins/semantic-models/skills/lineage-analysis/SKILL.md b/plugins/semantic-models/skills/lineage-analysis/SKILL.md index dad7d575..67f3aadf 100644 --- a/plugins/semantic-models/skills/lineage-analysis/SKILL.md +++ b/plugins/semantic-models/skills/lineage-analysis/SKILL.md @@ -1,6 +1,6 @@ --- name: lineage-analysis -version: 26.20 +version: 26.24 description: Trace relationships between semantic models and downstream reports across Fabric workspaces. Automatically invoke when the user asks to "find downstream reports", "show report lineage", "impact analysis", "what depends on this dataset", "cross-workspace lineage", "which reports are connected", "get model dependencies", or mentions model-to-report dependency tracing. --- @@ -63,6 +63,6 @@ The script only discovers Power BI reports. For full dependency mapping includin ## Related Skills -- **`review-semantic-model`** -- Audit model quality, memory, DAX, design +- **`semantic-model`** -- Design, build, refresh, and review models (quality, memory, DAX, design) - **`refreshing-semantic-model`** -- Trigger and monitor model refreshes - **`fabric-cli`** (fabric-cli plugin) -- Workspace and item management via `fab` CLI diff --git a/plugins/semantic-models/skills/power-query/SKILL.md b/plugins/semantic-models/skills/power-query/SKILL.md index e7eb9645..95b2235b 100644 --- a/plugins/semantic-models/skills/power-query/SKILL.md +++ b/plugins/semantic-models/skills/power-query/SKILL.md @@ -1,6 +1,6 @@ --- name: power-query -version: 26.20 +version: 26.24 description: Author, validate, and test Power Query M expressions in semantic model partitions. Automatically invoke when the user mentions "Power Query", "M code", "M expression", "partition expression", "query folding", or asks to "write Power Query", "fix Power Query", "test a partition", "preview partition data", "debug Power Query step", "optimize Power Query". --- diff --git a/plugins/semantic-models/skills/refresh-semantic-model/SKILL.md b/plugins/semantic-models/skills/refresh-semantic-model/SKILL.md index 620cbf0b..83452a07 100644 --- a/plugins/semantic-models/skills/refresh-semantic-model/SKILL.md +++ b/plugins/semantic-models/skills/refresh-semantic-model/SKILL.md @@ -1,6 +1,6 @@ --- name: refresh-semantic-model -version: 26.20 +version: 26.24 description: Automatically invoke this skill whenever the user asks to refresh a semantic model or a dataset. Can also be used to manage, optimize, troubleshoot, or configure a refresh or a refresh schedule. --- @@ -298,7 +298,7 @@ Pro capacity supports only full-model standard refreshes. Enhanced refresh featu ## Related Skills -- **`review-semantic-model`** -- Structured model quality and performance review +- **`semantic-model`** -- Model design, build, and quality/performance review - **`lineage-analysis`** -- Downstream report discovery and impact analysis - **`standardize-naming-conventions`** -- Naming audit and remediation - **`fabric-cli`** (fabric-cli plugin) -- Workspace and item management via `fab` CLI diff --git a/plugins/semantic-models/skills/review-semantic-model/SKILL.md b/plugins/semantic-models/skills/review-semantic-model/SKILL.md deleted file mode 100644 index c3867498..00000000 --- a/plugins/semantic-models/skills/review-semantic-model/SKILL.md +++ /dev/null @@ -1,136 +0,0 @@ ---- -name: review-semantic-model -version: 26.20 -description: Review, audit, and validate Power BI semantic models against quality, performance, and best practice standards. Automatically invoke when the user asks to "review a semantic model", "audit a semantic model", "check model quality", "optimize my model", "validate model design", "check AI readiness", "prepare model for Copilot", or mentions model validation or quality assessment. ---- - -Warning: This skill is incomplete and still in progress, but may provide value already as-is -- Kurt - -# Reviewing Semantic Models - -Structured evaluation of Power BI semantic models against quality, performance, and best practice standards. Produces actionable findings with prioritized recommendations. - -## Review Workflow - -### Step 0: Gather Context - -Before analyzing TMDL, collect metadata and understand the business context. - -**Run the model info script:** - -```bash -python3 scripts/get_model_info.py -w <workspace-id> -m <model-id> -``` - -This returns: storage mode, model size, connected reports, deployment pipeline, endorsement status, sensitivity label, data sources, refresh schedule, last refresh, and capacity SKU. - -**Ask the user:** - -- What business process does this model represent? -- Who are the primary consumers? (report developers, analysts, executives, AI/Copilot users?) -- Are they the developer of both the model and its reports, or only one? -- Is the model in development, testing, or production? -- Where should findings be documented? (scratchpad, agent-docs, wiki, etc.) - -Understanding the business context is critical. A model for 3 analysts has different requirements than one consumed by Copilot across the organization. The audit categories and their severity shift based on this context. - -### Step 1: Analyze Model Structure - -Inspect the model definition to evaluate its structure. The approach depends on available tooling -- use whatever is available to read the model's tables, columns, measures, relationships, and expressions. Do not prescribe a specific tool; common options include Tabular Editor, the `te-cli`, `fab export` to TMDL, or programmatic access via APIs. - -### Step 2: Audit Categories - -Evaluate findings across categories, ordered by severity: - -**Critical** -- Bidirectional relationships (ambiguity risk) -- Circular dependencies between tables -- Missing data types on columns -- Tables without relationships (orphaned) - -**Memory and Size** -- High-cardinality columns with large dictionaries (GUIDs, transaction IDs, composite keys) -- IsAvailableInMdx enabled on hidden or high-cardinality columns (wastes memory on attribute hierarchies unused by DAX; disable for columns not consumed via Analyze in Excel / MDX) -- Unsplit DateTime columns (near-unique precision creating massive dictionaries) -- Auto Date/Time tables (hidden LocalDateTable_* bloating memory) -- Inappropriate data types (Double for currency, String for numeric) -- Calculated columns that could be measures -- Unused columns or tables (no references in measures or visuals or other downstream items) - -**Data Reduction** -- Unfiltered history in fact tables (no date-range filter or incremental refresh) -- Columns that aren't necessary for reporting or calculations or consumption -- Pre-summarization opportunities (detail grain not needed for reporting) -- Columns better handled upstream (i.e. calculations not done in calc columns or PQ) - -**DAX Anti-Patterns** (for systematic DAX query optimization, use the [`dax` skill](../dax/)) -- Filtering tables instead of columns in CALCULATE (causes both correctness and performance issues) -- Unhandled division by zero (use DIVIDE() or explicit zero-check; note: plain `/` is fine when the denominator is guaranteed non-zero and can be faster) -- Iterators with callbacks or nested iterators over large tables (use aggregators like SUM/AVERAGE when possible; iterators over large tables are fine if the expression is Storage Engine-pushable) -- Missing KEEPFILTERS around non-equality filter predicates in CALCULATE - -**Measure Hygiene** -- Implicit measures used where explicit measures should exist -- Report-scoped extension measures that should be model-level -- Duplicate or overlapping measures with ambiguous names - -**Documentation** -- Tables or columns missing descriptions -- Missing display folders for measures -- Inconsistent naming conventions (use the `standardize-naming-conventions` skill) - -**Design** -- Star schema violations (direct fact-to-fact relationships, snowflake patterns) -- Missing or misconfigured date table: must be marked (`dataCategory: Time` in TMDL, with a key Date column), have continuous daily dates (no gaps), span the full range of fact data, and relate to fact tables via a single-column relationship. Missing any of these causes time intelligence functions (DATEADD, SAMEPERIODLASTYEAR, TOTALYTD) to return BLANK -- Excessive columns per table (>30 suggests denormalization issues) -- Many-to-many relationships without bridging tables -- Multiple fact tables relating to the same dimension via different keys without a shared conformed dimension (causes slicers on one fact to not filter the other) -- Inactive relationships without corresponding USERELATIONSHIP in measures (orphaned relationships that suggest incomplete modeling) - -**Direct Lake (if applicable)** -- Delta table health (parquet file count, V-Order, row group sizes) -- DirectQuery fallback risk (RLS definitions, SQL endpoint views) - -**AI and Copilot Readiness** (see `references/ai-readiness.md`) -- Duplicate field names across tables (confuses Copilot/data agents) -- Missing AI instructions -- Missing or inadequate descriptions for AI consumption -- Complex patterns (disconnected tables, many-to-many, inactive relationships) are valid model design but AI may struggle with them - -### Step 3: Performance Analysis - -For performance-specific analysis, see `references/performance.md`. - -### Step 4: Report Findings - -Produce a structured markdown report with: - -- Summary table of finding counts by severity -- Detailed findings with file locations and line numbers where possible -- Specific remediation recommendations for each finding -- Prioritized action list (critical first) - -## Using the Semantic Model Reviewer Agent - -Dispatch the `semantic-model-auditor` agent to perform the structural audit. The agent handles export, analysis, and reporting autonomously. - -## Notes - -- The structural audit analyzes model metadata -- it does not execute DAX queries or check data quality -- For DAX query performance testing, see `references/performance.md` -- For DAX optimization, use the [`dax` skill](../dax/) -- For companion report review, use the `review-report` skill in the reports plugin - -## References - -- **`references/ai-readiness.md`** -- Copilot/Data Agent preparation: AI instructions, descriptions, schema, verified answers -- **`references/performance.md`** -- Performance testing methodology, unused column detection, memory analysis -- **`scripts/get_model_info.py`** -- Quick model metadata overview (storage mode, size, reports, pipeline, endorsement, data sources) - -## Related Skills - -- **[`dax`](../dax/)** -- DAX performance optimization -- **`review-report`** (reports plugin) -- Companion skill for report-level review -- **`standardize-naming-conventions`** -- Naming audit and remediation -- **`lineage-analysis`** -- Downstream report discovery -- **`refreshing-semantic-model`** -- Refresh monitoring and troubleshooting diff --git a/plugins/semantic-models/skills/review-semantic-model/references/ai-readiness.md b/plugins/semantic-models/skills/review-semantic-model/references/ai-readiness.md deleted file mode 100644 index f43b9286..00000000 --- a/plugins/semantic-models/skills/review-semantic-model/references/ai-readiness.md +++ /dev/null @@ -1,145 +0,0 @@ -# AI and Copilot Readiness - -Preparing a semantic model for conversational BI experiences (Copilot, data agents) and evaluating whether a model is ready for AI consumption. - -## Before Investing in AI Readiness - -AI readiness preparation can easily double the development time for a semantic model. Before investing that effort, confirm with the user: - -1. **Will users actually use conversational BI?** If users will only consume reports, AI readiness work is low priority. Good naming and descriptions are always beneficial, but the AI-specific work (instructions, verified answers, AI schema) is only valuable if someone will use Copilot or a data agent. -2. **Is the model stable enough?** AI readiness is iterative and requires testing. If the model is still undergoing significant structural changes, defer AI prep until it stabilizes. -3. **Is Copilot/Data Agent enabled in the tenant?** Check tenant settings before investing effort. - -## The Decision: Reports, Conversational BI, or Both? - -Reports and conversational BI address different problems and suit different scenarios. The model design choices change depending on the consumption method: - -| Consumption | Model Impact | -|---|---| -| Reports only | Standard design: star schema, good measures, appropriate grain | -| Conversational BI | All of the above PLUS: AI instructions, AI schema, verified answers, linguistic schema, synonyms, English naming, descriptions optimized for AI | -| Both | Full investment in both report design and AI readiness | - -Ask the user which consumption methods are planned before scoping the AI readiness review. - -## AI Readiness Checklist - -### 1. Model Architecture (Foundation) - -These are prerequisites: without them, Copilot and data agents will produce poor results regardless of other configuration. - -- [ ] Star schema with clear fact and dimension tables (no flat, denormalized, or pivoted structures) -- [ ] Correct data types on all columns -- [ ] Unnecessary columns and tables removed -- [ ] Explicit DAX measures for all key metrics (implicit measures are not accessible to data agents) -- [ ] Report-scoped extension measures moved to the model (extension measures are invisible to data agents) -- [ ] Duplicate or overlapping measures consolidated or clearly differentiated - -### 2. Naming and Metadata - -Business-friendly naming is one of the highest-impact changes for AI readiness. AI interprets field names literally. - -- [ ] Human-readable names on all visible objects (no CamelCase, snake_case, UPPER_CASE, or technical abbreviations like TR_AMT, CustName) -- [ ] Synonyms configured for alternative terminology users employ (e.g. "Revenue" and "Sales" and "Turnover" for the same measure) -- [ ] Row labels set on dimension tables (helps AI identify the "name" column) -- [ ] Default summarization set correctly on numeric columns (prevents accidentally summing IDs) - -### 3. Descriptions - -Descriptions are critical for AI -- they provide context that field names alone cannot convey. However, good descriptions for AI are *different* from good descriptions for human users. - -**For AI/agents:** Descriptions should disambiguate and direct. Example: -> "When a user asks for Margin, they are referring to Standard Margin (this measure). Use this field, not Gross Margin or Contribution Margin." - -**For human users:** Descriptions should explain calculation and source. Example: -> "Standard margin = revenue minus standard cost of goods sold, using the valuated production cost from the manufacturing plant" - -**Guidance:** -- [ ] Descriptions present on all visible tables, columns, and measures -- [ ] Descriptions make implicit or hidden information explicit (not restating the field name) -- [ ] Descriptions for AI focus on disambiguation, preferred usage, and relationships between fields -- [ ] Descriptions for humans focus on calculation logic, data sources, and business context -- [ ] If both audiences need descriptions, consider using AI instructions for the AI-specific guidance and descriptions for the human-readable content - -**Do not rely solely on AI to generate descriptions.** AI-generated descriptions without additional business context tend to be verbose and unhelpful -- they restate what the AI can already infer from the model structure. If using AI to draft descriptions, always review and enhance with business-specific knowledge. Ask the user or domain experts for implicit knowledge that is not obvious from the model alone. - -### 4. AI Instructions - -AI instructions are freeform text that Copilot and data agents read automatically. They are one of the most impactful controls for conversational BI quality. - -**What to include:** -- Business terminology definitions with examples ("TMS is total media spend, calculated using the measure total_media_spend") -- Time period definitions (fiscal year start, peak season, reporting cadence) -- Metric preferences for common questions ("when users ask about margin, use Standard Margin") -- Disambiguation of date fields ("Order Date is the primary date field; Ship Date is only for logistics analysis") -- Default groupings and analysis preferences -- Example DAX queries for complex scenarios to guide AI patterns -- Instructions for Calculation Groups, DAX UDFs, or Field Parameters if present -- Visualization preferences for Copilot (which chart types to prefer/avoid) -- Output style guidance (conciseness, question-asking behavior, possibly specific tools or scenarios to avoid if copilot) - -**Where AI instructions live:** -- Eventually: saved as a `.md` file in the `/Copilot` folder of a PBIP project (might not be live yet but was on the docs...?) -- Editable in Power BI Desktop ("Prep your data for AI"), VS Code, or Tabular Editor (via C# macro) -- AI instructions are NOT automatically read by coding agents (Claude Code, Codex, GitHub Copilot) -- they must be explicitly referenced in agent context - -**Important:** AI instructions are a *writing skill*, not an engineering task. Inform instructions from observing users interact with Copilot, not from guessing. Iterate incrementally based on testing and user feedback. - -- [ ] AI instructions present and non-empty -- [ ] Business terminology documented -- [ ] Common question patterns addressed -- [ ] Instructions refer to fields by their proper names -- [ ] Instructions tested with Copilot or data agent -- [ ] Instructions kept concise; non-contradictory (also not contradictory with i.e. descriptions) - -### 5. AI Data Schema - -The AI data schema controls which tables, columns, and measures are visible to Copilot and data agents. Scoping this correctly prevents confusion and improves response quality. - -- [ ] Only relevant tables, columns, and measures selected (not the entire model) -- [ ] All dependent objects for selected measures are included (use `get_measure_dependencies()` from semantic-link-labs for complex models) -- [ ] Helper measures and intermediate calculation objects excluded -- [ ] Duplicate or overlapping measures excluded from schema -- [ ] All fields required for verified answers are visible (not hidden) -- [ ] Schema selection matches between "Prep for AI" and data agent configuration - -### 6. Verified Answers - -Verified answers are pre-built report visuals that Copilot returns for specific questions, ensuring accuracy for the most common and important queries. - -**Note:** It's probably not worth investing in Verified Answers. Power Bi visuals are just too poor for this. - -- [ ] Most common user questions identified (ask the user or their team) -- [ ] Verified answers created with appropriate visuals -- [ ] Complete, robust trigger questions per verified answer (not partial phrases) -- [ ] Both formal and conversational phrasings included -- [ ] Relevant filters configured per verified answer for flexible slicing -- [ ] All fields used in verified answers are visible in the model -- [ ] Trigger questions tested for exact and semantic matching - -### 7. Data Agent Configuration (if applicable) - -- [ ] Same tables selected in data agent as in Prep for AI > AI Data Schema -- [ ] Model-specific instructions in AI instructions, NOT at data agent level -- [ ] Data agent instructions limited to: response formatting, cross-source routing, common abbreviations, tone -- [ ] Routing instructions added if multiple data sources ("For revenue questions use the semantic model; for real-time delivery use the KQL database") - -## Performance Analysis for AI - -AI-generated DAX queries can be unpredictable in their patterns. Measure performance with: -- **Tabular Editor 3:** VertiPaq Analyzer for memory footprint, Best Practice Analyzer for structural issues -- **DAX Studio:** Server timings for query performance, VertiPaq Analyzer (alternative to Tabular Editor) - -Test AI-generated queries specifically -- the DAX patterns Copilot produces may differ from hand-written measures and may expose performance issues that don't surface in reports. - -## Gathering Context for Descriptions and Instructions - -When the user asks to improve AI readiness, gather business context before writing descriptions or instructions: - -1. **Ask the user** about the business process, key metrics, common questions, and areas of ambiguity -2. **Review existing reports** connected to the model -- what questions do they answer? What fields do they use? -3. **Check the model's measures** -- are there naming patterns that suggest business logic (YTD, MTD, vs. Plan, vs. LY)? -4. **Look for contentious metrics** -- margins, budgets, forecasts often have multiple definitions across teams -5. **Understand the organizational vocabulary** -- different departments may use different terms for the same concept - -Do not assume you know what good descriptions or instructions look like for a specific business context. Always validate with the user before applying changes. diff --git a/plugins/semantic-models/skills/semantic-model/SKILL.md b/plugins/semantic-models/skills/semantic-model/SKILL.md new file mode 100644 index 00000000..b56752eb --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/SKILL.md @@ -0,0 +1,104 @@ +--- +name: semantic-model +version: 26.24 +description: This skill should be used whenever the user mentions a "semantic model", "data model", or "dataset", or asks to "build", "model", "design", "optimize", "review", or "audit" one, or to "add a measure", "add a relationship", "create a role" / "set up RLS", "add a calculation group", "set up incremental refresh", "fix a star schema", "reduce model size", "prepare a model for Copilot / AI", or "check model quality". Covers the full lifecycle (design, build, refresh, review) and drives every operation through the `te` CLI first, then a model MCP or TOM (connect-pbid), then TMDL authoring (the tmdl skill). Not for report visuals (use pbir-cli) or isolated DAX query tuning (use the dax skill). +--- + +# Semantic models: design, build, refresh, review + +Guidance for designing, building, refreshing, and reviewing Power BI / Analysis Services tabular models from the terminal. It consolidates model review with modeling best practices, and routes every change to the narrowest capable tool. Depth lives in `references/`; this file is the operating loop and the routing. + +## When to use + +- Designing or building a model: star schema, relationships, measures, calculation groups, roles, parameters, storage modes, incremental refresh +- Optimizing a model: size / VertiPaq, DAX correctness, Direct Lake, refresh cost +- Reviewing or auditing a model against quality, performance, and best-practice standards +- Preparing a model for Copilot / AI consumption + +## When NOT to use + +- Editing report visuals, pages, or formatting: use the `pbir-cli` skill (reports plugin) +- Isolated DAX query performance tuning: use the `dax` skill +- TMDL file syntax mechanics: use the `tmdl` skill (this skill routes to it for the file-edit fallback) +- The `te` command surface itself: the `te-cli` skill (tabular-editor plugin) is the command reference; this skill is the modeling judgment layered on top + +## Tool cascade (the core operating rule) + +Reach for the narrowest capable tool, in order. Most edits never leave step 1. + +1. **`te` CLI first.** One verb per operation, staged in memory until `--save`, with a save-time DAX + referential-integrity gate. Covers add/set/rm/mv for measures, columns, relationships (`Sales[K]->Dim[K]` shorthand on `te add`), roles + RLS filters, calculation groups / items, incremental-refresh policy, `te format`, `te bpa`, `te vertipaq`, `te query`. Each Bash call is a fresh shell, so pass `-m <model>` (and `-s`/`-d` for remote) on every command, or set `TE_SESSION`. Read the real object's settable surface first with `te get <obj>` and `te set <obj> -q <prop>` (no value). The `te-cli` skill is the full command reference. +2. **A model MCP, or TOM, when `te` cannot reach a property.** Some properties are absent from `te set -q` (for example `alternateOf`, `securityFilteringBehavior`, `crossFilteringBehavior`, KPI sub-objects, linguistic-schema content, calendar objects). Drive these through the Power BI Modeling MCP server when available, a `te script` C# pass (in-process TOM), or the `connect-pbid` skill (PowerShell + TOM/ADOMD against a live local Desktop instance, and the only route to traces: `EVALUATEANDLOG`, aggregation-hit events, storage DMVs). The local Desktop proxy cannot reach Direct Lake; use a remote XMLA endpoint there. +3. **`fab` + direct TMDL last, with the `tmdl` skill.** Service- and file-shape operations with no model-edit verb: assigning Entra principals to roles (workspace-side, not in `.tmdl`), report-to-model binding, Copilot-folder features (AI instructions, AI data schema, verified answers), Lakehouse / Delta reshaping behind Direct Lake, and bulk structural surgery that is cleaner as one TMDL diff than N `te` calls. Author the TMDL with the `tmdl` skill, then run `te validate`. + +Ordering gate: add relationships before any measure that uses `RELATED()` or a cross-table `CALCULATE()`, or the save gate fails with `DAX0002` (no relationship in context). + +## Lifecycle + +### Design +Model dimensionally: a star of fact plus conformed dimensions beats snowflakes and fact-to-fact joins. Decide storage mode and refresh strategy before building; both are near one-way doors once published. See `references/dimensional-modeling.md`, `references/storage-modes.md`, `references/composite-models.md`, and `references/direct-lake.md`. + +### Build +Make each change through the cascade above. Author measures with full metadata (DisplayFolder, FormatString, Description) in one pass. Validate after every mutation (`te validate`) and gate on BPA (`te bpa run --fail-on error`). Renaming or moving any object can silently break downstream reports and models; run the lineage check first, then propagate with `pbir-cli` / `fabric-cli` (see `references/refactoring-renaming.md`). Deep guidance per area: `references/relationships.md`, `references/time-intelligence.md`, `references/calculation-groups.md`, `references/parameters.md`, `references/security.md`, `references/dax-authoring.md`. + +### Refresh +Configure incremental refresh from the terminal (`te incremental-refresh`); for Direct Lake, the refresh is the framing. See `references/incremental-refresh.md`, and the `refresh-semantic-model` skill for monitoring and troubleshooting. + +### Review +Audit against the categories below and produce prioritized findings with file locations. Gather context first with `scripts/get_model_info.py` (storage mode, size, connected reports, endorsement, data sources, refresh schedule). Full checklist in `references/review-checklist.md`; performance method in `references/performance.md`. + +## Review categories (by severity) + +- **Critical**: bidirectional ambiguity, circular dependencies, missing data types, orphaned tables, fail-open RLS, limited relationships that silently drop rows +- **Memory & size**: high-cardinality dictionaries, auto attribute hierarchies (`isAvailableInMDX` on hidden / high-cardinality columns), unsplit DateTime, auto date/time tables, wrong data types, calc columns that should be measures, unused objects +- **Data reduction**: unfiltered fact history (no incremental refresh), unnecessary columns, detail grain not needed for reporting, logic better pushed upstream +- **DAX correctness**: filtering tables not columns in CALCULATE, unguarded division, context-blind calc columns, variable time-shift bugs (`references/dax-authoring.md`; for query tuning use the `dax` skill) +- **Measure hygiene**: implicit measures, report-scoped measures that belong in the model, ambiguous duplicates +- **Documentation & AI**: missing descriptions (Copilot truncates after 200 characters), missing display folders, missing synonyms, inconsistent naming (use `standardize-naming-conventions`) +- **Design**: star-schema violations, mis-marked date table, many-to-many without a bridge, dead inactive relationships +- **Direct Lake**: non-unique one-side keys (queries fail at runtime), DirectQuery fallback, calc columns on Direct Lake, Delta guardrail breaches + +## Related skills + +- `tmdl`: TMDL file authoring (the cascade's step-3 fallback) +- `dax`: DAX query performance optimization +- `connect-pbid`: TOM / ADOMD via PowerShell against a live Desktop instance; traces; the MCP / TOM tier +- `te-cli`: the `te` command reference +- `c-sharp-scripting`: TOM C# scripts and macros (`te script`) for properties `te` cannot reach +- `standardize-naming-conventions`: naming audit and remediation +- `refresh-semantic-model`: refresh monitoring and troubleshooting +- `lineage-analysis`: artifact lineage (downstream reports and models that consume this model, across workspaces); distinct from intra-model object dependencies +- `bpa-rules` (tabular-editor): authoring BPA rules; `fabric-cli`: service / workspace operations + +## Reference map + +```yaml +references/dimensional-modeling.md: star schema, SCD2, junk / degenerate dimensions, header-detail, bridges +references/relationships.md: cardinality, limited relationships, ambiguity, active / inactive, USERELATIONSHIP +references/time-intelligence.md: classic vs calendar TI, mark-as-date traps, week-based / 4-4-5 +references/calculation-groups.md: precedence, sideways recursion, selection expressions, the variant trap +references/parameters.md: field parameters, what-if parameters, dynamic titles +references/security.md: RLS validation + defensive filters, bidirectional + RLS, OLS restrictions +references/query-semantic-model.md: querying a model with DAX, INFO functions, output formats, probing +references/storage-modes.md: Import / DirectQuery / Dual / Hybrid decision matrix +references/composite-models.md: source groups, regular vs limited, Direct-Lake-plus-Import +references/aggregations.md: user-defined aggregations, grain, te-script AlternateOf +references/direct-lake.md: OneLake vs SQL, framing, DirectQuery fallback, guardrails +references/incremental-refresh.md: IR policy, detect data changes, hybrid / real-time, refresh strategy +references/vertipaq-optimization.md: VPA metrics, value vs hash encoding, splitting high-cardinality keys +references/dax-authoring.md: variable semantics, DIVIDE, measure vs calc column (gaps vs the dax skill) +references/ai-copilot-readiness.md: the Copilot grounding contract, synonyms / linguistic schema, descriptions, Q&A retirement +references/metadata-and-organization.md: descriptions, display folders, naming, measure tables, perspectives +references/hierarchies-cultures.md: user hierarchies, parent-child, KPIs, perspectives, cultures / translations +references/documentation-and-bpa.md: data dictionary, BPA documentation gate, metadata diffs, intra-model impact analysis (artifact lineage = the lineage-analysis skill) +references/refactoring-renaming.md: safe rename workflow (lineage check first, then propagate via pbir-cli / fabric-cli) +references/review-checklist.md: full audit checklist with remediation +references/performance.md: performance testing, unused-column detection, memory analysis +scripts/get_model_info.py: model metadata overview (mode, size, reports, endorsement, sources, refresh) +``` + +## What this skill deliberately leaves out + +- GUI-tool walkthroughs and the old review skill's "use whatever tool is available" framing: superseded by the te-cli-first cascade +- Hand-authored Q&A phrasings: Q&A is being retired and Copilot does not read phrasings; invest in synonyms and descriptions +- Perspectives or display folders presented as security: both are queryable by anyone with model access; use RLS / OLS +- The PBIT format: out of scope diff --git a/plugins/semantic-models/skills/semantic-model/references/aggregations.md b/plugins/semantic-models/skills/semantic-model/references/aggregations.md new file mode 100644 index 00000000..1be0f76a --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/aggregations.md @@ -0,0 +1,13 @@ +# User-defined aggregations + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** build the agg table with `te`-native verbs (`te add table`, hide columns), then map `AlternateOf` via `te script` (TOM) since `te set -q` does not expose it. Audit the >= 10x grain ratio with `te query`; confirm cache hits via a trace (connect-pbid / DAX Studio), not `te query` alone. + +## The cascade decision rule and the user-defined-aggregation example + +A triage rule before any modeling edit, so the agent reaches for the narrowest capable tool instead of defaulting to TMDL text-editing (re-implementing what `te` does atomically and losing the save-time DAX/RI gate). The full mapping is the **Tool cascade** in SKILL.md; the operational anchors: most ops have one `te` verb; ops with no `te -q` (user-defined aggregations / `AlternateOf`, OLS `metadataPermission` if rejected, relationship `crossFilteringBehavior`/`securityFilteringBehavior`, KPI sub-objects, calendar objects, linguistic metadata) drop to `te script` TOM; service/file-shape ops (Entra role membership, report binding, Copilot folder, Lakehouse reshaping, bulk TMDL surgery) go to fab + TMDL. + +User-defined aggregations are the textbook `te script` escape hatch: build the agg table and hide it with `te`-native verbs, then map columns via TOM (`new TOM.AlternateOf { Summarization = TOM.SummarizationType.Sum, BaseColumn = ... }`), then `te validate`. The agg is a hidden coarser-grain pre-aggregate of a large DQ detail table that the engine transparently rewrites qualifying subqueries to; keep it at least ~10x fewer rows than its detail or maintenance outweighs the speedup, and strip high-cardinality attributes from the GroupBy set. Relationship-based aggs (the agg relates to the same dimensions) make GroupBy entries optional except DISTINCTCOUNT, which needs an explicit GroupBy on the key; GroupBy-based aggs (denormalized, no relationships) require them or the agg never hits. RLS must filter both agg and detail or the engine refuses to answer from the agg; `detailTable` must be DirectQuery and must point at the real detail (chained aggregations are illegal); hybrid tables and Direct Lake (either flavor) do not support user-defined aggregations (pre-aggregate in the Lakehouse instead). + +Sources: repo te-cli command-reference / gotchas / SKILL / semantic-modeling-practices; learn.microsoft.com aggregations-advanced / SummarizationType; learn.microsoft.com composite-model-guidance; learn.microsoft.com aggregations-auto diff --git a/plugins/semantic-models/skills/semantic-model/references/ai-copilot-readiness.md b/plugins/semantic-models/skills/semantic-model/references/ai-copilot-readiness.md new file mode 100644 index 00000000..62d14376 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/ai-copilot-readiness.md @@ -0,0 +1,39 @@ +# AI and Copilot readiness + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** set descriptions with `te set <obj> -q description -i "..." --save`, kept under 200 characters (Copilot truncates there). Bulk-audit missing or over-long descriptions with a `te script` C# pass over `Model.AllMeasures` / `AllColumns`; synonyms and linguistic-schema content go through `te script` (TOM) or TMDL. + +## The Copilot grounding contract (what metadata actually reaches the LLM) + +When a user asks Copilot or the DAX query view a natural-language question, a fixed documented set of metadata is serialized as grounding context. What gets sent: the full schema (tables, columns, measures, relationships, calculation groups ; hidden objects included **except** on a live connection to a shared model); synonyms from the linguistic schema; per object the DAX expressions, descriptions (**truncated after the first 200 characters**), data types, format strings (including format-string expressions), and data category; min/max values of columns likely used in the query (actual data points); sometimes the query result echoed back to explain the answer. Excluded: conversation history on Retry. + +Implications: the 200-char truncation is a hard budget ; front-load disambiguating keywords, anything past char 200 is human-tooltip only. `isHidden` does **not** remove an object from Copilot grounding on import/composite models (only a live connection to a shared model hides them) ; hiding declutters the human field list but does not shrink the AI surface (use an AI data schema for that). Format strings and data category steer answer rendering and intent, not just display (a `WebUrl` category or currency format changes how Copilot frames the result). Min/max of candidate columns are real data points ; sensitive extremes are a data-exposure consideration. Audit description length and fix truncation-prone descriptions in one model load with a `te script` C# pass over `Model.AllMeasures`/`AllColumns` where `Description.Length > 200`; set a single description with `te set <obj> -q description -i "..." --save`. Do not assume hidden = invisible to AI; a measure with a keyworded description grounds far better than one relying on a clear name alone (Copilot leans on description keywords for discovery). + +Sources: learn.microsoft.com copilot-semantic-models; learn.microsoft.com tutorial-copilot-power-bi-prepare-model; repo te-cli SKILL property table + +## Synonyms and the linguistic schema (where they live and how to write them) + +Synonyms are alternate words a user might use ("revenue" for `Sales Amount`, "client" for `Customer`). They are not a plain property like `displayFolder`; they live in the model's LinguisticMetadata, attached to a culture, serialized as LSDL (`.lsdl.yaml`), and in a PBIP project also land in `<Name>.SemanticModel/Copilot/schema.json`. The classic Q&A linguistic schema additionally carries phrasings (relationship verbs, measurement/dynamic adjectives, noun phrasings). Synonyms are explicitly in the Copilot grounding contract and survive the Q&A deprecation, so they are high-return; phrasings are Q&A-engine constructs largely irrelevant to Copilot, so hand-authoring them is low-return. + +`te` has no first-class synonym verb (no `te set -q synonyms`); the supported path is a TOM C# script through `te script` writing to the culture's `LinguisticMetadata.Content` (a JSON document ; ensure a culture exists, round-trip the content through a JSON parser, inject under Entities -> entity -> Terms, validate, then save ; edit as structured JSON, not string concatenation, or you corrupt it and the save gate may not catch a semantically-broken-but-syntactically-valid doc). connect-pbid reaches the same `Culture.LinguisticMetadata` object on a live Desktop instance. Direct LSDL/Copilot-folder editing is last: export the `.lsdl.yaml` (Modeling > Linguistic schema > Export), edit, re-import; or edit `Copilot/schema.json`. Generated entries carry `State: Generated` ; on re-import anything Generated is ignored and regenerated, so to keep an edit strip the tag, and to suppress a generated entry set `State: Deleted`. Synonyms are culture-scoped, so a model with no culture has nowhere to put them; after deploying LSDL/Copilot edits the service needs a refresh to sync (import on deploy; DirectQuery/Direct Lake once per day). + +Sources: learn.microsoft.com q-and-a-tooling-advanced / -intro; repo pbip copilot-folder; repo c-sharp-scripting translations + +## Prep-data-for-AI: four distinct features, where each persists, and deploy gotchas + +"Prep data for AI" is an umbrella over four non-interchangeable features; conflating them is the common mistake: +- **descriptions** ; natural-language context per object, used in DAX queries + Copilot search/discovery; stored as a standard property; only the first 200 chars feed AI +- **AI data schema** ; selects a subset of fields Copilot prioritizes, the only feature that genuinely shrinks the AI surface (hiding a field does not exclude it from Copilot); stored in `Copilot/schema.json`; ignored by report-page summaries, "create a report page," search, and DAX-query Copilot (those pull the whole model); requires Q&A enabled or the Prep-data tabs grey out +- **verified answers** ; map trigger phrases to a specific pinned visual as the canonical answer; stored in `Copilot/VerifiedAnswers/...` (PBIR) +- **AI instructions** ; free-text business context steering Copilot (not a guaranteed instruction-follow); stored in `Copilot/Instructions/instructions.md`; 10,000-char limit; Copilot-only + +All four save to the semantic model, never the report. The AI data schema is the only feature that narrows what Copilot reasons over and is frequently confused with hiding; you cannot fix "Copilot uses the wrong column" by hiding it (still grounded) or an AI instruction alone (non-binding) ; the deterministic fix is the AI data schema plus a verified answer. None has a `te` verb today, so the cascade inverts here: **fab + direct folder editing** is the practical path (edit the markdown/JSON in the PBIP project, deploy with `fab`); te-cli still owns the prerequisites (description quality, star-schema/dedup cleanup, unique field names, removing unused objects, resolving ambiguous relationships ; do those first). Deploy gotchas: after a Git/pipeline deploy a service refresh is required to sync (import on deploy; DirectQuery/Direct Lake once per day max); editing the AI data schema needs the Copilot pane closed and reopened to see the effect; "Approved for Copilot" is a separate final toggle that removes the friction-warning banner for the model and reports on it (propagation usually under an hour, up to 24h on models with many reports; force it by saving a trivial report change). Only the model can be approved, never a report/dashboard/app. + +Sources: learn.microsoft.com copilot-prepare-data-ai-faq / -data-schema / copilot-prepare-data-ai / tutorial-copilot-power-bi-prepare-model; repo pbip copilot-folder + +## Q&A is retiring: what to stop investing in vs keep + +Microsoft is retiring the Power BI Q&A experience and steering users to Copilot. This re-prioritizes the AI-readiness backlog. Stop or deprioritize: hand-authored Q&A phrasings (Attribute, Name, measurement/dynamic adjectives, Noun), adjective tuning for "long rivers"/"red products" questions, and the Q&A setup-menu and "review questions users asked" loop ; these are Q&A-engine constructs Copilot does not consume. Keep high-value: synonyms (in the Copilot grounding contract, survive deprecation); descriptions (<= 200 char, keyworded ; universal, feed Copilot + DAX-query + search); format strings, data category, data types (part of the contract); the AI data schema, verified answers, AI instructions (the forward-looking Copilot toolset). Caveat: the AI data schema currently requires Q&A enabled on the model, so do not pre-emptively disable Q&A while you still depend on AI-data-schema authoring. Investing in the full Q&A linguistic schema for AI readiness targets a feature being retired; direct those hours at the keep list. + +Sources: learn.microsoft.com q-and-a-tooling-intro / -advanced; powerbi.microsoft.com deprecating-power-bi-qa; learn.microsoft.com copilot-prepare-data-ai-data-schema diff --git a/plugins/semantic-models/skills/semantic-model/references/calculation-groups.md b/plugins/semantic-models/skills/semantic-model/references/calculation-groups.md new file mode 100644 index 00000000..68f6fab4 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/calculation-groups.md @@ -0,0 +1,37 @@ +# Calculation groups + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** `te add calculationGroup "Time Intelligence" --save`, then `te add calculationItem ...`. Read / set group precedence with `te set <group> -q precedence`; selection expressions, `selectionExpressionBehavior`, and variant guards that `te set` does not expose go through `te script` (TOM). + +## Calculation group precedence: how items actually combine + +When two calc groups have an item in filter context at once, the engine nests them: each item's `SELECTEDMEASURE()` token is textually replaced by the next-lower-precedence item's DAX, down to the base measure. The group with the **highest** `precedence` integer is the **outermost** wrapper. The output is order-dependent and rarely commutative ; a high-precedence `SELECTEDMEASURE()*2` over a lower `SELECTEDMEASURE()+2` on measure=10 gives `((10)+2)*2 = 24`, not 14. When a higher-precedence item uses `CALCULATE`/context transition, it rewrites the filter context the inner item sees (Time Intelligence at higher precedence makes YTD wrap both numerator and day-count denominator of an average). Precedence also decides whose dynamic format string wins (only the highest group's applies; a measure's own dynamic format is always lower than any calc group). + +Precedence lives on the **group** (not per-item `ordinal`, which is only within-group sort order ; do not confuse them). Inspect with `te set <group> -q precedence` before setting; assign distinct integers when groups co-occur in one visual, or apply order is undefined. If `te set` cannot reach it, fall to a `te script` C# pass (`CalculationGroupPrecedence`) or edit the `precedence:` line in TMDL last. A calc item only modifies an expression containing a measure reference; with no `SELECTEDMEASURE()` in scope it is a no-op. + +Sources: learn.microsoft.com calculation-groups (precedence); repo SpaceParts Z04CG1 Time Intelligence.tmdl; repo te-cli semantic-modeling-practices + +## Calculation groups: sideways recursion (the only supported recursion) + +A calc item can reference another item in the same group by overriding that group's column inside `CALCULATE` ; the one recursion form the engine permits. `YOY%` is built from the `YOY` and `PY` items (`DIVIDE(CALCULATE(SELECTEDMEASURE(), 'Time Intelligence'[Time Calculation]="YOY"), CALCULATE(SELECTEDMEASURE(), ...="PY"))`) rather than re-deriving them; a `PY YTD` item layers PY on top of the already-defined YTD item. This is composition for calc items: define `YTD`/`PY`/`YOY` once, then build derived items from them, and changing the base propagates. Each `DIVIDE` branch is a separate `CALCULATE` that re-enters the group cleanly. + +Use the column's quoted full name inside the item DAX with the item name as a string literal ; this is the one place the literal is unavoidable, and the rename-safe `ISSELECTEDMEASURE` guidance does not apply (it is about measure references). So renaming a base item still requires a find-replace across dependent items. Only sideways recursion works; an item recursing into itself, or applying the same group twice in one `CALCULATE`, errors or is silently ignored (one item per group in filter context at a time). Nesting two overrides of the same group column in one `CALCULATE` collapses to the last filter. + +Sources: learn.microsoft.com calculation-groups (sideways recursion, single-item-in-filter-context) + +## Calculation groups: selection expressions and selectionExpressionBehavior + +Two optional group-level DAX properties handle non-clean selections: `multipleOrEmptySelectionExpression` fires on multi-select, a nonexistent item, or a conflict; `noSelectionExpression` fires when the group is unfiltered. Each carries its own `formatStringDefinition`. Default when undefined: the group does not filter (base measure passes through); a single valid selection never triggers either. A model-level `selectionExpressionBehavior` (`Automatic` = today's `nonvisual`, or `visual`) tunes the default; above a future compat level `Automatic` resolves to `visual`. + +Without `multipleOrEmptySelectionExpression`, a user multi-selecting calc-group items gets the unfiltered base measure with no signal ; a common "why is the number wrong" ticket. Define it to return a deliberate value (`BLANK()` with a `--`-style format) or block ambiguous selections; `noSelectionExpression` makes a "Current" default. These are group properties (confirm casing first); if `te set` does not expose them, use `te script` (`MultipleOrEmptySelectionExpression`, `SelectionExpressionBehavior`) or TMDL. Never put normal logic in these (they never fire on a single valid selection); each needs its own `formatStringDefinition` or it inherits a mismatching default (returning `BLANK()` but inheriting currency shows a stray symbol); `selectionExpressionBehavior=visual` changes existing report numbers, so baseline key measures before and after. + +Sources: learn.microsoft.com calculation-groups (selection expressions) + +## Calculation groups: hard limitations and the variant-data-type trap + +Adding any calc group flips a model-wide switch. Unsupported alongside calc groups: OLS on the calc-group table; RLS on the calc group itself (use the data tables); Detail Rows Expressions; Smart Narrative; implicit column aggregations (the sigma options appear but cannot apply unless `discourageImplicitMeasures=true`, which removes them cleanly). In Live Connection, dynamic format strings are not applied to report-level measures. + +The variant trap: the moment a calc group exists, Power BI treats **every** measure as the variant type. This silently breaks any dynamic format string that reuses another measure's value, and breaks visuals when an item runs arithmetic on a non-numeric measure (dynamic titles, text measures): `Cannot convert value ... of type Text to type Numeric`. Removing all calc groups reverts measures to their real types. This is the most surprising side effect in a review because the failure surfaces in report visuals, not the model. Two fixes, in order: guard arithmetic items with `IF(ISNUMERIC(SELECTEDMEASURE()), ...)`, or coerce a reused format-string measure with `FORMAT([Dynamic format string], "")`; the preferred long-term fix moves shared format-string logic into a DAX UDF, sidestepping the coercion (dovetailing with the UDF-over-calc-group guidance). When reviewing a model with calc groups, proactively check text/title measures and measure-reusing dynamic format strings. + +Sources: learn.microsoft.com calculation-groups (limitations, considerations); learn.microsoft.com desktop-dynamic-format-strings; repo te-cli semantic-modeling-practices diff --git a/plugins/semantic-models/skills/semantic-model/references/composite-models.md b/plugins/semantic-models/skills/semantic-model/references/composite-models.md new file mode 100644 index 00000000..36b9200b --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/composite-models.md @@ -0,0 +1,15 @@ +# Composite models + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** inspect each table mode and source group with `te script` (TOM `Partitions[0].Mode`); promote shared dimensions to Dual to keep cross-table joins regular. Confirm no path went limited with `te query -q "EVALUATE INFO.VIEW.RELATIONSHIPS()"` after edits. + +## Composite-model tuning: source groups, regular vs limited, Direct-Lake-plus-Import + +A composite partitions tables into source groups (the Import/Direct-Lake cache is one; each DQ source is its own). Performance and even correctness depend on whether a query stays inside one group. The runtime branches four ways, only the first three fast: (1) Import/Dual only -> cache; (2) Dual + DQ from the same source -> few native queries, regular relationships; (3) Dual/Hybrid + DQ same source -> cache for import partitions, native for the rest; (4) **anything cross-source-group** -> limited relationships everywhere in that path, groupings shipped as materialized subqueries, and **rows dropped when keys don't match across groups** (an RI gap becomes a silent wrong total). + +The single-source regular-relationship rule: many-side Dual needs one-side Dual; many-side Import needs Import or Dual; many-side DirectQuery needs DirectQuery or Dual. Cross-source regular only when both tables are Import; m2m is always limited. The exception worth knowing: a true composite of Direct Lake **on OneLake** + Import supports **regular** relationships across the two, unlike classic DQ+Import (limited only) ; so a billion-row Delta fact in Direct Lake plus small Import dimensions avoids the cross-group penalty. Direct Lake on SQL does not support this; convert to OneLake first. + +Tuning workflow: enumerate each table's mode + bound source (`te script` over `Partitions[0]`), promote dimensions shared with a DQ fact to Dual, re-check relationship types and flag any non-`Regular`. For DQ source tuning the source side matters more ; ensure warehouse indexes support the emitted joins/filters/groupings (a source-DBA task, not te-cli). Keep caches in sync (the engine will not mask a Dual/import-vs-DQ gap; fix data integrity at the source). Direct Lake on SQL silently falls back to DQ for view-based tables and granular RLS, quietly turning a "Direct Lake" model into a slow DQ one ; OneLake web-modeling composites avoid this. + +Sources: learn.microsoft.com composite-model-guidance; learn.microsoft.com aggregations-advanced (regular vs limited); learn.microsoft.com direct-lake-web-modeling; learn.microsoft.com direct-lake-develop; local forums DB (composite-sluggish, Direct-Lake-edit threads) diff --git a/plugins/semantic-models/skills/semantic-model/references/dax-authoring.md b/plugins/semantic-models/skills/semantic-model/references/dax-authoring.md new file mode 100644 index 00000000..5ffc565f --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/dax-authoring.md @@ -0,0 +1,31 @@ +# DAX authoring correctness + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** prove a rewrite by running both versions with `te query` across contexts and diffing scalars. Find risky patterns with `te find "/" --in expressions --paths-only` and `te find "CALCULATE" --in expressions`. Gate every change with `te validate` and `te bpa run --fail-on warning`. + +## DAX: variable evaluation semantics (evaluated once, where defined, lazily) + +A `VAR` is not a macro or a cell reference. It is a named value computed at most once, in the filter+row context at the point of the `VAR` line, and only if a reachable branch consumes it. Three properties held simultaneously: **context-at-definition, not at use** (a later `CALCULATE` wrapping a reference reuses the already-computed scalar, the opposite of a measure which re-evaluates at invocation); **computed once** (materialized once, reused for every reference and context); **lazy** (a variable no reachable branch consumes is never evaluated). + +This is the most common source of silently-wrong refactors, not slow ones. Two failure modes: (1) **time-shift bug** ; an agent hoists a measure call into a variable then wraps the reference in `SAMEPERIODLASTYEAR`/`DATEADD`, and the shift is a no-op because the value froze at definition. The fix is to keep the measure reference inside the `CALCULATE` (SQLBI: `SalesPY` must compute `CALCULATE([Sales Amount], SAMEPERIODLASTYEAR(...))`, not `CALCULATE(SalesCY, ...)`). (2) **branch-selection bug** ; an agent assumes putting two heavy expressions in variables and selecting with `IF`/`SWITCH` runs only the chosen one, but variables declared before the `IF` are outer-scope and both evaluate regardless. For strict per-branch evaluation, declare the `VAR` inside the branch expression. + +Validate by execution, not static inspection: read the body, stage a rewrite, prove equivalence by querying both versions across contexts and diffing scalars, gate with `te validate`, then save. For branch-vs-scope inspection of intermediate values, drop to connect-pbid and `EVALUATEANDLOG`; `te query` returns only the final scalar/table, the one case the TOM/ADOMD trace path beats it. Pitfalls: a variable colliding with a column name shadows the column ; prefix with `_`; variables are immutable (no running-total-in-a-loop mental model ; restructure as an iterator); "wrap it in a VAR" is not a free optimization when the value is context-dependent and the wrapping `CALCULATE` meant to change that context. + +Sources: SQLBI variables-in-dax / when-are-variables-evaluated / eager-vs-strict-evaluation; dax.guide var; repo dax-patterns (DAX003) + +## DAX: DIVIDE is the correctness default, bare `/` is the narrow exception + +`DIVIDE(n, d)` returns `BLANK()` (or an optional third-arg alternate) when `d` is zero or blank, instead of the division error bare `n / d` raises. `DIVIDE` is the default in a measure body; the bare operator is the deliberate exception used only when the denominator is provably never zero/blank and you are inside a hot iterator. The performance guidance (DAX018: replace `DIVIDE` with `/` inside iterators to avoid the FE callback the zero-guard forces) over-generalizes in isolation into "prefer `/` everywhere," reintroducing divide-by-zero in ordinary scalar measures ; the anti-pattern the review skill flags as critical. The two rules reconcile once the precondition is explicit: outside iterators default to `DIVIDE` (a scalar runs once, the callback is irrelevant, blank-on-zero is what you want in a visual); inside an iterator switch to `/` **only after** guaranteeing a non-zero denominator by pre-filtering the iterated table (`CALCULATETABLE('Items', 'Items'[Rate] <> 0)`). The guarantee is not optional. + +Decide by where the division sits, then validate the zero-path. Find bare-operator divisions with `te find "/" --in expressions --paths-only`, classify each (scalar vs inside `SUMX`/`AVERAGEX`), ensure scalar cases use `DIVIDE`, and for the iterator case prove the zero row is gone (`FILTER('Items', 'Items'[Rate] = 0)` returns empty). A BPA rule is the durable enforcement ; gate with `te bpa run --fail-on warning` so any future bare `/` in a non-iterator measure surfaces. Pitfalls: `DIVIDE(n, 0, alternate)` fires the alternate on blank denominators too, not just literal zero; `n / 0` returns Infinity/NaN in some surfaces rather than a hard error, poisoning downstream aggregations silently (worse than an error); the blank `DIVIDE` returns is indistinguishable from a no-data blank if a downstream `ISBLANK` drives logic, so supply an explicit alternate to disambiguate. + +Sources: dax.guide divide; repo dax-patterns (DAX018) + +## DAX: measure vs calculated column, the context-blindness decision rule + +A calculated column is computed at refresh, materialized into VertiPaq, evaluated in row context only ; it cannot see report filters, slicers, or the visual. A measure is computed at query time in the filter context the visual provides. This is a correctness decision first, cost second (the cost framing ; calc columns ~4x larger ; is already covered). The deciding question: does the value need to respond to what the user filters? If yes it must be a measure ; no calc-column cleverness can read filter context. A row-fixed `[Quantity] * [Unit Price]` is a legitimate calc column (better, a Power Query or source column); a "% of total" / running total / "rank within selection" is inherently a measure because the denominator/window depends on what is filtered, and as a calc column it bakes in the whole-table answer and never changes as the user slices. + +The frequent mistake is using `CALCULATE` inside a calc column to "fix" the missing context ; that gives a context transition over the table (wrong, frozen at refresh) and is the direct cause of the **circular-dependency trap**: `CALCULATE` in a calc column makes that column depend on every column in its table, so a second such column depends on the first and the model refuses to validate. This is a top forum error. The fix restricts the dependency surface to the key: wrap the filter removal in `ALLEXCEPT('Table', 'Table'[PrimaryKey])`, or on a keyless table use `ALLNOBLANKROW` (a bare `ALL` still creates a blank-row dependency). Inventory mis-modeled columns with `te find "CALCULATE" --in expressions --paths-only | grep column`; convert by adding the measure, validating, checking downstream dependents (`te find "Sales[RowRevenue]" --in expressions`), then removing the column. Pitfalls: "it works in my test visual" hides the bug (a calc-column percent looks right unfiltered then stays constant when sliced ; always test against a slicer selection); calc columns/tables are not materialized under DirectQuery and (mid-2026) Direct Lake (default to measures or upstream columns for any model that might move to Direct Lake); the circular-dependency error names the second column you created, not the design flaw at the first. + +Sources: SQLBI understanding-circular-dependencies / avoiding-circular-dependency-errors; Fabric forums (circular-dependency threads); repo te-cli SKILL; repo te-cli semantic-modeling-practices diff --git a/plugins/semantic-models/skills/semantic-model/references/dimensional-modeling.md b/plugins/semantic-models/skills/semantic-model/references/dimensional-modeling.md new file mode 100644 index 00000000..c7954676 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/dimensional-modeling.md @@ -0,0 +1,47 @@ +# Dimensional modeling + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** build a dimension with `te add table "Dim" --columns "Key:Int64,Attr:String" --save` and relate it via `te add relationship "Fact[Key]->Dim[Key]" --save`. Build junk / bridge / SCD calc tables with `te script` (TOM `CROSSJOIN` / `ADDCOLUMNS`) when there is no source, then hide keys with `te set <col> -q isHidden -i true --save`. + +## Slowly changing dimensions (SCD2): durable keys and version-safe counting + +A type-2 dimension keeps history by inserting a new row per tracked-attribute change. Each row carries a surrogate key (unique per version), a durable/natural key (stable per business entity), and validity bounds (`ValidFrom`/`ValidTo`, often `IsCurrent`). The fact joins on the surrogate, so each fact row points at the version correct at event time. This is the only shape where "sales by the customer's region at time of sale" and "by current region" coexist. + +Two silent bugs appear the moment an SCD2 dimension exists, neither throwing an error: +- `DISTINCTCOUNT('Sales'[CustomerKey])` counts versions, not customers ; a customer who moved twice counts as three. Count the durable key instead, evaluated through the fact so VertiPaq solves it in the storage engine: `COUNTROWS(SUMMARIZE('Sales', 'Customer'[Customer Code]))` +- Slicing by a dimension attribute gives point-in-time history by default (the fact froze the version). Authors expecting current-state grouping get wrong totals. For current-state, do not put the changing attribute on the SCD2 table at all; split it into a separate type-1 (overwrite) dimension keyed on the durable key with its own 1:M to the fact. Then "current region" and "region at time of sale" live on different tables and the author picks. A `LASTNONBLANK`/`ValidTo IS BLANK` re-derivation works but pushes cost to the formula engine + +There is no model property that says "this is SCD2"; infer it. A `te query` probe comparing `COUNTROWS(VALUES(Customer[CustomerKey]))` against `COUNTROWS(VALUES(Customer[Customer Code]))` plus a check for `Valid*` columns tells you: surrogate rows greater than durable entities means history is present and every `DISTINCTCOUNT` over the surrogate is suspect. Enumerate offenders with a `te script` C# pass over `Model.AllMeasures`, add corrected durable-key measures, and set the surrogate `IsHidden`, `SummarizeBy=None`, `isAvailableInMDX=false` (high-cardinality VertiPaq hog) ; never delete it, it is the relationship key. Drop to TOM only to read `ValidFrom`/`ValidTo` contents for non-overlap validation. Note `RELATED('Customer'[Region])` in a fact calc column returns the historical region ; usually correct, but it bites agents porting "current attribute" logic. + +Sources: SQLBI distinct-count-of-customers-in-SCD2; SQLBI slowly-changing-dimensions-in-powerpivot; learn.microsoft.com star-schema (slowly changing dimensions) + +## Junk dimensions: collapsing low-cardinality flag columns + +A junk dimension folds several small, independent, low-cardinality attributes (order status, ship method, yes/no flags) into one table whose rows are the Cartesian product of distinct values plus a surrogate. Three status flags with 3, 2, and 4 states collapse from three relationships and three fact FKs into one dimension of at most 24 rows and one FK ; the fact shrinks, the relationship graph simplifies, and the AI schema surface reads cleaner. The payoff is bounded by the product of distinct counts, so it only works for genuinely low-cardinality attributes ; cross two 50-value columns and you have a 2,500-row table that is no longer junk. + +Build the Cartesian product upstream (warehouse view, or Power Query full-outer-joins of the distinct lists plus an index surrogate merged back onto the fact). When the source is fixed, build it as a DAX calculated table via `te script` (`CROSSJOIN`/`ADDCOLUMNS` of the distinct flag values, with a concatenated `StatusKey`), hide every column, set `SummarizeBy=None`, and relate on a matching computed `StatusKey` on the fact (1:M, single direction). A calculated-table junk dim avoids ETL but will not materialize in DirectQuery or (mid-2026) Direct Lake; push the build to the Lakehouse/warehouse for those modes. Prefer building from observed tuples over the full Cartesian when many combinations never occur, so slicers do not surface impossible pairs. + +Sources: learn.microsoft.com star-schema (junk dimensions); learn.microsoft.com fabric dimensional-modeling-dimension-tables (junk dimensions) + +## Header-detail: carrying header-grain measures on a flattened model + +Transactional sources arrive as a header (order: date, customer, store, freight, order-level discount) plus detail lines. Denormalize header attributes onto the line fact ; never relate two facts on the order number (SQLBI's benchmark: a 94M-distinct order-number join ran one query at 15x the CPU of the star, and a product filter forcing bidirectional propagation pushed a query from 6 to 17 minutes). The part that bites after flattening: header-grain measures (freight, shipping cost, a flat order fee) double-count when summed off the line fact, because the header value repeats per line. + +Two correct shapes, picked by how the header measure is sliced: +1. **Header value additive only over header-grain dimensions.** Keep one line-grain fact and de-dup the header value per order: `SUMX(VALUES('Sales'[Order Number]), CALCULATE(MAX('Sales'[Freight])))`. Correct sliced by date/customer/store, and intentionally won't break down by product (freight has no product grain). Hide the raw `Freight` so nobody drags the `SUM` version onto a visual +2. **Many header measures, heavily used.** Keep two facts at their natural grains (a `Sales Order` header fact, a `Sales` line fact), each related to the shared conformed dimensions, never to each other. Header measures live on the header fact, line measures on the line fact; both filter correctly by shared dimensions and neither double-counts. This is "two facts, conformed dimensions," distinct from the forbidden "two facts related to each other" + +Validate the de-dup with a `te query` comparing naive `SUM(Sales[Freight])` against the `SUMX` form ; equal means either one line per order or freight was already allocated upstream. For the two-fact shape, confirm both facts relate only to shared dimensions via `INFO.VIEW.RELATIONSHIPS()`, and validate RI on both (a header customer the lines lack diverges totals). Allocating freight down to the line is a different decision that changes the number; only do it if the business wants freight attributable per product. + +Sources: SQLBI header-detail-vs-star-schema-models; learn.microsoft.com relationships-one-to-one + +## Bridge tables as factless facts, and degenerate-dim as-table vs as-column + +Two nuances the simple framing misses: +- The recommended way to relate two dimensions many-to-many is a **factless-fact bridge** (only the two keys, duplicates allowed) with both dimensions on the one-side and the bridge on the many-side, preferred over a native many-to-many relationship. Both produce the same filtering, but the bridge is a real table you can put RLS on, hang a weighting/allocation measure on, and read in lineage; the native m2m hides that. A native m2m is `Limited`, so RI violations group silently under a blank; two 1:M `Regular` legs surface the same blanks but are inspectable +- A degenerate dimension is not always "a hidden fact column." One degenerate attribute means a hidden fact column. Two or more correlated ones (order number and order line number) mean a separate 1:1 dimension built from a composite surrogate (`OrderNumber * 1000 + OrderLineNumber`), giving clean `Sales Order` / `Sales Order Line` fields while keeping the fact narrow + +Build the bridge via `te script` as a calculated table of distinct key pairs (or load from source), hide its columns, add two 1:M relationships, and make exactly one leg bidirectional. m2m grand totals are non-additive (a salesperson in two regions contributes to both, so "All Regions" is less than the sum of parts) ; bake that into the measure `Description` so Copilot and authors do not read it as a bug. The degenerate-as-table only works at exactly one row per fact line with matching surrogate values both sides ; build it at fact grain, never `DISTINCT`. Do not reach for a bridge to fix a snowflake ; that is a normalization issue, flatten upstream. + +Sources: learn.microsoft.com star-schema (factless fact tables); learn.microsoft.com relationships-many-to-many; learn.microsoft.com relationships-one-to-one (degenerate dimensions) diff --git a/plugins/semantic-models/skills/semantic-model/references/direct-lake.md b/plugins/semantic-models/skills/semantic-model/references/direct-lake.md new file mode 100644 index 00000000..dc6aecf6 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/direct-lake.md @@ -0,0 +1,40 @@ +# Direct Lake (OneLake vs SQL, framing, fallback, guardrails) + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** read the partition mode + shared expression with `te script` to tell OneLake from SQL. Frame after any deploy / add with `te refresh --type automatic`. Surface hidden DirectQuery fallback by setting `te set "Model.DirectLakeBehavior" DirectLakeOnly --save` in dev, then run representative report DAX. + +## Direct Lake on OneLake vs on SQL (not interchangeable) + +Both share the VertiPaq engine and a `directLake` partition mode but differ in schema discovery, security, and recovery. On SQL the shared expression points at the lakehouse/warehouse SQL analytics endpoint; on OneLake it points directly at the OneLake storage path. Three behaviors an agent will otherwise get wrong: +- **Fallback**: on SQL a query that can't serve in-memory silently falls back to DirectQuery (slower); on OneLake there is **no fallback** ; it returns an error and visuals fail to render +- **Composite**: OneLake supports composite models (mix Direct Lake with Import/DQ/Dual); SQL does not support mixing storage modes in one model (extend it only by building a composite on top of the published model in Desktop) +- **Security path**: SQL checks run through the SQL endpoint (SELECT is enough); OneLake uses OneLake Security (identity needs Read+ReadAll or a OneLake role). SQL-based RLS is simply not applied on OneLake + +Other deltas: SQL-only tables may be based on a SQL view (forces fallback); OneLake cannot bind a non-materialized SQL view (use a materialized view or Import); deployment pipelines rebind the data source on SQL but not directly on OneLake (use a parameter expression in the connection string); neither flavor works through any gateway. Inspect the flavor before touching: read the partition mode and the shared expression (SQL endpoint URL means on SQL; OneLake abfss path means on OneLake). Choosing OneLake "because no fallback" means hard query failure under guardrail breach or unprocessed tables, not graceful degradation ; size capacity and optimize Delta so in-memory always wins. Adding multiple model tables from the same source Delta table is unsupported via Desktop/web in both flavors (XMLA only, and "Edit tables" + refresh then errors). + +Sources: learn.microsoft.com direct-lake-overview (key concepts, comparison, limitations); learn.microsoft.com direct-lake-develop (model tables); learn.microsoft.com direct-lake-how-it-works; repo te-cli command-reference + +## Framing, reframing, and why a fresh Direct Lake table answers nothing + +Framing points the model at the latest committed Delta version by reading the Delta log and current Parquet; it is triggered by a refresh, usually completes in seconds, and is mostly metadata. After framing, queries see the Delta state as of that last framing, not necessarily the latest writes. Framing evicts changed column segments so they reload on next access; unchanged columns stay resident (incremental framing). The trap: a Direct Lake table created/added via XMLA/automation (or just deployed with `te deploy`) is **unprocessed** until you send a refresh ; until framed, on SQL every query falls back to DirectQuery, on OneLake queries error. So "I deployed and it's slow / returns errors" is almost always "you never framed it." Tabular Editor 3 frames on first deploy for this reason; the cross-platform `te` CLI does not implicitly, so the agent must trigger it. + +A refresh is the frame ; after deploying or adding Direct Lake tables, run `te refresh --type automatic` (or per-table). For a full cold rebind the XMLA path is processClear then processFull; with `te` the nearest is `--type clearvalues` then `--type full`, and `--dry-run`/`--trace` inspect what TMSL it sends and confirm framing. Control automatic framing (the default that reflects OneLake changes without manual refresh) ; disable it for point-in-time control and frame deliberately (set via manage APIs/TMSL if `te set` lacks the toggle). Pitfalls: `te refresh` on Direct Lake reframes metadata pointers, it does not copy data; framing fails if a Delta table breaks a guardrail (e.g. >10,000 Parquet files) ; optimize first; fresh lakehouse rows do not appear when automatic updates are off and you never reframed; do not `--type full` a hot production model expecting "safer," it forces a colder reload that hurts the next queries. + +Sources: learn.microsoft.com direct-lake-how-it-works (framing, automatic updates); learn.microsoft.com direct-lake-understand-storage; learn.microsoft.com direct-lake-overview; docs.tabulareditor.com direct-lake-guidance; repo te-cli command-reference + +## DirectQuery fallback: exact triggers and how to forbid it + +On Direct Lake **on SQL** only, a query that can't serve in-memory transparently switches that table to DirectQuery against the SQL endpoint ; slower, but always returns the latest source data. On OneLake there is no fallback. Fallback is the biggest hidden Direct Lake regression; worse, the docs note fallback uses hybrid query plans that carry a tradeoff **even when no fallback is needed**, so leaving it enabled taxes every query. The evaluation order for a Direct Lake on SQL query: (1) semantic-model OLS on a restricted object -> error; (2) SQL-endpoint CLS denial -> error; (3) SQL-endpoint RLS, or any table on a SQL view -> fall back to DirectQuery; (4) exceeds a capacity guardrail -> fall back; (5) otherwise in-memory. Sharp edge: if a lakehouse SQL endpoint switches from fixed-identity to SSO, OneLake roles become SQL granular rules and Direct Lake on SQL then falls back 100% of the time. + +Control with the model-level `directLakeBehavior`: `Automatic` (default, falls back), `DirectLakeOnly` (never fall back ; a query that can't run in Direct Lake **fails** instead of going slow), `DirectQueryOnly` (A/B comparison). Setting `DirectLakeOnly` in dev is the best way to surface hidden fallback ; failures become loud instead of slow. It applies only to Direct Lake on SQL. Set via `te set "Model.DirectLakeBehavior" DirectLakeOnly --save`, or TOM/TMDL if rejected, then run representative report DAX and watch for visual errors (a real fallback path to fix). Do not set `DirectLakeOnly` in production where some queries legitimately need RLS/view paths ; those visuals hard-fail. To detect whether fallback happened, capture an XMLA query trace and look for DirectQuery storage-engine events. Any table on a non-materialized view forces fallback on SQL and is not creatable on OneLake ; materialize it or use Import for that one table. + +Sources: learn.microsoft.com direct-lake-how-it-works (DirectQuery fallback); learn.microsoft.com direct-lake-security-integration; learn.microsoft.com direct-lake-understand-storage; learn.microsoft.com DirectLakeBehavior; fabric.guru controlling-direct-lake-fallback + +## Direct Lake guardrails and modeling-rule deltas vs Import + +Direct Lake imposes constraints Import doesn't, and breach behavior differs by flavor. The model-level Max model size guardrail is evaluated once; the rest are **per query**. Max Memory is a paging ceiling, not a guardrail (it pages and degrades, won't fail). Breach: on SQL the refresh warns and queries still return via fallback; on OneLake the refresh **fails** like Import until the Delta tables are optimized back under the limits. Because guardrails are mostly per-query, you cannot audit by model size alone ; the same model serves small queries in-memory and falls back (or fails) on a wide scan. The fix is upstream Delta optimization (V-Order, large segments roughly 1-16M rows, Parquet files well under 10,000, reduced cardinality), not a model edit. + +Modeling rules that differ: no calculated columns/tables referencing Direct Lake columns on SQL (unsupported); on OneLake calculated columns are preview and unmaterialized (review before relying). Push row-level logic upstream into the Delta table or an Import table in an OneLake composite. Calc groups, what-if/field parameters implicitly create calculated tables but are allowed (they don't reference Direct Lake columns), which is why a SQL-flavor model can host calc groups while banning calc columns. MDX clients (Analyze in Excel) treat Direct Lake tables like DirectQuery: no session-scoped MDX, no Direct-Lake-table user hierarchies (Import-table hierarchies still work in a composite). Audit before deploy: list calculated columns/tables (illegal on a SQL-flavor model) and partition modes, and run `te bpa` with org rules flagging calc columns on Direct Lake partitions and tables bound to SQL views. te-cli detects and edits the model; it cannot V-Order or compact Parquet (that is `fab` + a lakehouse/Spark OPTIMIZE job). + +Sources: learn.microsoft.com direct-lake-overview (capacity, limitations); learn.microsoft.com direct-lake-understand-storage; learn.microsoft.com direct-lake-how-it-works; repo te-cli command-reference diff --git a/plugins/semantic-models/skills/semantic-model/references/documentation-and-bpa.md b/plugins/semantic-models/skills/semantic-model/references/documentation-and-bpa.md new file mode 100644 index 00000000..b343a811 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/documentation-and-bpa.md @@ -0,0 +1,41 @@ +# Documentation and BPA + +Companion to the `semantic-model` skill (SKILL.md). Documentation (data dictionary, change diffs), BPA as a documentation gate, and intra-model impact analysis for safe edits. + +**Scope note.** This reference is **intra-model only**. For lineage of **artifacts** (which reports, dataflows, and lakehouses feed and consume this model across Fabric workspaces), use the `lineage-analysis` skill and `fab`. That artifact lineage is a different question from the object-to-object dependencies covered here; do not confuse the two. + +**Working with `te`:** `te bpa run --fail-on error` (rules via `--rules` / `TE_BPA_RULES`); build a data dictionary or coverage audit with `te query -q "EVALUATE INFO.VIEW.MEASURES()"` and the `INFO.VIEW.*()` family. For impact analysis before an edit, `te deps "<obj>"` and `INFO.CALCDEPENDENCY` (intra-model object dependencies). Artifact lineage across workspaces is the `lineage-analysis` skill, not `te`. + +## Generate a model data dictionary from INFO functions + +A reproducible data dictionary (every table, column, measure, relationship with description, format, folder, type, hidden flag, source) built by querying the model's own metadata through DAX INFO functions instead of parsing TMDL by hand. INFO functions are DAX wrappers over the AS schema DMVs, returning tables you reshape with SELECTCOLUMNS/ADDCOLUMNS/FILTER. TMDL is the source of truth but spread across dozens of files and does not resolve inherited/inferred state (effective data type, summarizeBy, post-perspective hidden status); one DAX query returns the **evaluated** catalog, which is what a dictionary, a coverage audit, or a version diff needs. The first ~200 chars of each description is what Copilot ingests, so a description-coverage report doubles as an AI-readiness gate. + +The `INFO.VIEW.*` family (`TABLES()`, `COLUMNS()`, `MEASURES()`, `RELATIONSHIPS()`) returns friendly pre-joined human-named columns and is the only INFO family usable inside calculations and calculated tables (so you can even bake a self-documenting hidden table into the model). Raw rowsets (`INFO.MEASURES()`, `INFO.COLUMNS()`, etc.) expose ID/TableID/LineageTag for joins and matching against TMDL lineage tags; `INFO.MODEL()` gives the dictionary header. `te` has no `document` verb but `te query` runs the EVALUATE and redirects to CSV/JSON ; that is the generator. A description-coverage audit is `FILTER(INFO.VIEW.MEASURES(), NOT [IsHidden] && LEN([Description]) = 0)`. Batch multiple queries in one `te script` pass (each `te` call carries ~1-2s startup). Pitfalls: INFO functions need write/admin permission and can't run over a Desktop live connection (a non-issue against your own .pbip via `te query`); `INFO.MEASURES()` returns `TableID` not a name (join to `INFO.TABLES()` or use `INFO.VIEW.MEASURES()` which pre-resolves it; the two families don't share column names, don't mix blind); `[Expression]`/`[Description]` carry literal newlines (emit JSON, not CSV, for naive parsers); `INFO.VIEW.*` returns the default culture only (pull translated captions from `INFO.CULTURES()` for localized models). If `te` can't reach the model, run the identical EVALUATE through `executeQueries` REST or XMLA (INFO is server-side); direct-TMDL parsing is a strictly worse last resort (declared, not effective, metadata). + +Sources: learn.microsoft.com info-functions-dax / info-measures / info-model; repo te-cli command-reference + +## Intra-model impact analysis with INFO.CALCDEPENDENCY (object dependencies, not artifact lineage) + +This is the **object lineage inside the model** (measure -> measure -> column), used to know what breaks before an edit. It is not the cross-workspace artifact lineage of the `lineage-analysis` skill; pair the two, since neither alone sees the full blast radius. + +Before renaming or deleting a column/measure you need the full transitive set of referencing objects: other measures, calc columns, calc items, format-string expressions, detail-rows, RLS filters, relationships. The two tools see different things and you want both. `te deps "Sales/Revenue" --upstream --downstream` walks Tabular Editor's static reference graph (fast local refactors; powers `te deps --unused`, objects with no DAX refs and not used in relationships/hierarchies/sort-by/variations/time roles). `INFO.CALCDEPENDENCY` (alias `INFO.DEPENDENCIES`) is the engine's own dependency graph, including dependencies that only materialize in a query context, so it surfaces references `te deps` may not classify the same way: a column pulled in only via a measure's RLS context, a calc item's `SELECTEDMEASURE` chain, or dependencies of an arbitrary ad-hoc DAX query. Crucially it answers "given this report visual's DAX, which model objects does it need" via the `"Query"` restriction ; query-context impact `te deps` cannot do. + +Run it through `te query` (full graph `EVALUATE INFO.CALCDEPENDENCY()`; upstream of a measure via the `"Query"` restriction; reverse/downstream by filtering on `[REFERENCED_TABLE]`/`[REFERENCED_OBJECT]`). Safe-rename sequence: `te deps --downstream` for the fast view, the reverse `INFO.CALCDEPENDENCY` filter for context-only refs, the edit with `te mv`/`te set`, `te validate`, then `te bpa run --fail-on error`. `te mv`/rename does not rewrite references inside report PBIR, so combine with the **`lineage-analysis` skill** (artifact lineage) and `pbir-cli` for the external blast radius ; see `references/refactoring-renaming.md`. Pitfalls: needs write permission, won't run over a Desktop live connection; each row is one direct edge (recurse yourself for N-hop, where this pass's `[REFERENCED_OBJECT]` becomes the next pass's `[OBJECT]`); double-quotes inside the `"Query"` restriction must be doubled; `te deps --unused` and `INFO.CALCDEPENDENCY` both miss a measure referenced only by a report visual (neither sees PBIR), so don't auto-delete on `--unused` alone, cross-check report (artifact) lineage. + +Sources: learn.microsoft.com info-calcdependency-function-dax / info-dependencies; SQLBI understanding-data-lineage-in-dax; repo te-cli command-reference + +## BPA as the documentation and metadata-completeness gate + +Use the Best Practice Analyzer not for performance/DAX antipatterns but as an automated documentation enforcer: fail the build when visible measures lack a description, columns lack a format string, or objects sit outside a display folder. The rules are tiny Dynamic LINQ expressions scoped to an object type. Documentation rots silently; a CI gate keeps it honest far cheaper than human review, and since `te bpa run` already gates deploys, folding metadata rules in costs nothing extra. This complements the INFO dictionary: INFO reports coverage, BPA enforces it and can auto-fix trivial cases. + +Author metadata rules (e.g. `not IsHidden and string.IsNullOrWhitespace(Description)` scoped to Measure; a numeric-column-needs-format-string rule with a `FixExpression = FormatString = "#,0"`), then run `te bpa run -r ./metadata-rules.json --no-defaults --fail-on error --ci github` for a clean documentation gate, or `--fix --save` to auto-apply trivial fixes. `-r` accepts a local file or URL (repeatable); `--no-defaults` runs only your rules so the gate isn't drowned by the standard ruleset; to ship rules with the model, embed them in the `BestPracticeAnalyzer` model annotation. Pitfalls: a `FixExpression` writing a format string is fine, one "fixing" a missing description is not (no sensible default text ; leave description rules fix-less and let them fail loudly); BPA evaluates the model graph so it can't enforce description coverage that only matters downstream in reports (pair with report-side review); `--no-defaults` (drop built-in rules) differs from `--no-model-rules` (drop model-annotation rules); severity is advisory until mapped to a gate (`--fail-on error` only trips on error-severity rules, so set documentation rules' severity to match). + +Sources: github.com TabularEditor/BestPracticeRules; docs.tabulareditor.com Best-Practice-Analyzer-Improvements; repo te-cli command-reference; repo bpa-rules skill + +## Snapshot and diff model metadata for change documentation + +Capture the evaluated catalog (from the INFO queries) as a versioned text artifact, then diff two snapshots to auto-generate a human-readable changelog. TMDL git diffs show file changes but are noisy (lineage tags, reordering, whitespace) and don't summarize semantically ("3 measures added, 1 description removed, format changed on Sales[Amount]"). A metadata snapshot is a stable sorted projection of only the fields you track, so its diff is the changelog ; the documentation counterpart to the pbi-desktop `refresh-cache` hook that snapshots metadata to `tmp/model-metadata.json`. + +Emit a deterministic sorted projection (`SELECTCOLUMNS(INFO.VIEW.MEASURES(), ...) ORDER BY [Table], [Name]`) to CSV per object type (tables, columns, relationships in sibling files so each diffs independently), commit it, and `git diff --no-index` between releases. The `ORDER BY` is what makes the diff stable across exports. Pitfalls: don't snapshot `INFO.MEASURES()` raw (its `ModifiedTime`/`StructureModifiedTime` change on every save, producing noise; project only semantic fields via `INFO.VIEW.*`); embedded newlines in `[Expression]` make CSV line-diffs awkward (format DAX consistently first with `te format --save` so only real logic changes surface); a renamed object shows as delete+add in a text diff (key the projection on `[LineageTag]`, stable across rename, for rename-aware changelogs). + +Sources: learn.microsoft.com info-functions-dax; repo te-cli command-reference; pbi-desktop refresh-cache hook precedent diff --git a/plugins/semantic-models/skills/semantic-model/references/hierarchies-cultures.md b/plugins/semantic-models/skills/semantic-model/references/hierarchies-cultures.md new file mode 100644 index 00000000..2994184d --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/hierarchies-cultures.md @@ -0,0 +1,29 @@ +# Hierarchies, KPIs, perspectives, cultures + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** `te add hierarchy ...` and set levels; KPIs, cultures, and metadata translations that `te` does not expose go through `te script` (TOM) or TMDL. Perspectives: `te add perspective` plus membership. + +## User hierarchies (creation, levels, ordering, anti-patterns) + +A user hierarchy is a navigation path on a single table (Category -> Subcategory -> Product); metadata only, no storage, no DAX. In TOM/TMDL it is a `Hierarchy` owning ordered `Level` objects, each pointing at one column on the same table. It is a usability and AI-readiness signal (Copilot, Q&A, and the field list read it); a model with logical drill paths reads as curated. The hard constraint authors miss: a hierarchy **cannot span tables** (one reason to flatten snowflakes into one dimension upstream); if the levels live on two tables, fix it upstream in Power Query, not with a relationship. + +Create the hierarchy then add levels top-to-bottom (level order = creation order); bind each level to its feeding column. Confirm the `Level` child-property names (`Column`, `Ordinal`) with `te set <level> -q` before scripting many; reordering an existing hierarchy means setting `Ordinal` explicitly. For bulk work (the same hierarchy across role-playing date dimensions, or fixing ordinals across dozens), use `te script` appending `Level` objects to a `Hierarchy.Levels`. Pitfalls: a level needs its column's Sort By Column set for non-alphabetical order (the hierarchy honors the column's sort, it defines none of its own); set `summarizeBy: none` on every level column and key (an auto-summing level shows a sigma and produces nonsense aggregations dropped in alone); TOM/TMDL does not auto-hide level columns (decide deliberately ; hide raw columns to force navigation, or leave visible for ad-hoc, but both visible doubles the field list); `te deps <col> --unused` treats a hierarchy-level column as used, so an unused sweep won't remove it (remove the level first). + +Sources: learn.microsoft.com tmdl-reference-tabular-object; repo te-cli command-reference / semantic-modeling-practices; SQLBI parent-child-hierarchies-in-tabular + +## Parent-child hierarchies via PATH (ragged trees) + +When depth is variable or unknown (org charts, chart-of-accounts, BOM) you cannot pre-build N named level columns. The tabular pattern flattens a self-referencing parent/child table into fixed level columns with the `PATH` family in DAX calculated columns, then builds a normal user hierarchy over them. This is the one legitimate documented use of DAX calculated columns the modeling-practices ref calls out by name (reserve calc columns for `RELATED`, `PATH`, or `COMBINEVALUES` keys) ; the recursion is naturally a DAX path operation and the ragged depth defeats a regular relationship. + +Author the calc columns with `te add -t CalculatedColumn`, validating each: `PATH(Employee[EmployeeId], Employee[ManagerId])` for the root-to-node path string, `PATHLENGTH` for node depth, and one `LOOKUPVALUE(..., PATHITEM(path, n, INTEGER))` column per fixed level to a chosen max depth >= the deepest branch; then build the hierarchy over the level columns. For depth > 5 or several parent-child tables, write the loop in `te script` after reading max depth from a `te query`. The actual hard part is ragged depth: a shallow branch leaves trailing `Level` columns blank, rendering as repeated leaf labels under deeper siblings. Suppress with a browse-depth-vs-node-depth measure using `ISINSCOPE` per level that blanks the value when the visual has drilled past where a node exists; **every** measure surfaced against the hierarchy needs the blanking guard (the maintenance cost a reviewer should flag ; parent-child couples every measure to the depth logic). Pitfalls: `PATHITEM(..., position)` without the `INTEGER` type arg returns text and a `LOOKUPVALUE` against an integer key silently misses; multiple roots are valid but each starts its own path (add a synthetic root upstream for a single tree); `PATH` requires the parent column to reference the same table's key with no orphans (an orphan `ManagerId` errors at refresh, not author time ; the `check-ri` hook covers cross-table keys not self-references); these are calc columns, so unsupported in DirectQuery and not materialized in Direct Lake (forces Import, or push the flattened columns to the Lakehouse). + +Sources: learn.microsoft.com path-function-dax; SQLBI parent-child-hierarchies-in-tabular; repo te-cli semantic-modeling-practices; Fabric forums (PATH multiple-root thread) + +## KPI objects on measures (status / goal / trend) + +A KPI is a sub-object on a single measure adding a goal (target), a status expression (band logic mapping a value to good/neutral/bad), and optional trend, so clients render a traffic light without the author rebuilding the band DAX. In TOM it is the `KPI` object at `Measures/<name>/KPI` (one of the three single-object TOM children, with Hierarchies and LinguisticMetadata, rather than collections). Defining status once on the measure means every consumer and Copilot's notion of "on target" agree; the tradeoff is the goal/status are baked into the model, so a per-report target override is impossible (if targets vary by audience, leave it as report-level conditional formatting). + +A KPI attaches to an existing measure: create the wrapper (`te add "<table>/<measure>/KPI" -t KPI`), discover settable properties with `te set <kpi> -q` (`TargetExpression`/target-measure, `StatusExpression`, `StatusGraphic`, `TrendExpression`), then set them. The status expression must return a small integer band (-1 bad / 0 / 1 good) referencing the owning measure, e.g. `SWITCH(TRUE(), [Margin %] >= 0.4, 1, [Margin %] >= 0.25, 0, -1)` ; returning a boolean or the raw value renders no indicator. `StatusGraphic` is a string from a fixed client-recognized set ("Three Circles Colored", "Traffic Light", "Five Bars Colored") ; a typo serializes fine but renders nothing, so pin the exact spelling from a model Power BI already authored (`te get <measure>/KPI --output-format tmdl`). For anything beyond one KPI, drive it through `te script` (`measure.KPI = new KPI()` then assign). A measure carries one KPI (different bands for different audiences = different measures, or report-level formatting). These are legacy KPI objects, distinct from Fabric Metrics/Scorecards (a workspace item type) ; do not conflate them in a review. + +Sources: learn.microsoft.com introduction-to-tabular-object-model (KPI single child); repo te-cli command-reference; repo tmdl object-properties diff --git a/plugins/semantic-models/skills/semantic-model/references/incremental-refresh.md b/plugins/semantic-models/skills/semantic-model/references/incremental-refresh.md new file mode 100644 index 00000000..fc99d08e --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/incremental-refresh.md @@ -0,0 +1,46 @@ +# Incremental refresh and refresh strategy + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** read the (preview, unstable) flags with `te incremental-refresh set --help`, then `te incremental-refresh set ...`, `te incremental-refresh apply`, and `te incremental-refresh show`. Targeted refresh: `te refresh --table <t> --partition "<t>.<p>" --type full`. Properties not exposed drop to TOM / TMSL or the `refreshPolicy` TMDL block. + +## Configure an incremental refresh policy from a terminal + +A `refreshPolicy` on one table tells the service how to auto-partition by date and which partitions to re-process. Two windows: archive (rolling) = history kept; incremental = recent slice re-queried per refresh. The service rolls both forward, merges aged partitions, drops out-of-archive ones. None are visible in Desktop or the service UI until the first service refresh applies the policy. It is the biggest lever on large-fact refresh cost (a 10k-rows/day fact with a 3-day window re-queries ~30k rows, not all history) and a one-way door: once published you cannot re-publish from Desktop (it wipes partitions) or download the `.pbix`, so the policy must be right before first service refresh, and edits after that go only through XMLA/te-cli. + +Prerequisites in order: (1) two `Date/Time` model parameters named exactly `RangeStart`/`RangeEnd` (reserved, case-sensitive); (2) the partition M filters its date column half-open `>= RangeStart and < RangeEnd` (upper-exclusive so boundary rows aren't double-counted); (3) the filter and column are `Date/Time` and **query-fold** (if the source keys on an integer like `OrderDateKey`, convert the params inside the filter via `Int32.From(DateTime.ToText(RangeStart,[Format="yyyyMMdd"]))` rather than abandoning folding ; a non-folding filter is the dominant cause of initial-refresh timeouts). + +`te incremental-refresh` is the operable entry point, but exact flag names are not stable across preview builds, so read them from the binary first (`te incremental-refresh set --help`). `set` writes the policy (rolling-window granularity/periods, incremental granularity/periods, `incrementalPeriodsOffset` for complete-periods-only, `policyType=basic`, `mode=import|hybrid`); `apply` materializes/expands the partition set (equivalent to TOM `ApplyRefreshPolicies` = RequestRefresh + SaveChanges). If a flag is missing (e.g. `pollingExpression`), fall to TOM/TMSL or hand-edit the `refreshPolicy` block. Compat level must be >= 1550 (>= 1565 hybrid). Pitfalls: every table reuses the same `RangeStart`/`RangeEnd` (no per-table pairs); size the incremental window to the late-arrival margin, not wider; enable Large model storage before the first refresh if over ~1 GB; a backdated update to the partition date column itself breaks IR (engine reads delete+insert, the delete is never picked up) ; treat transaction dates as immutable and selectively refresh from the change point; the first service refresh loads the whole store window (bootstrap on Premium to dodge the 5h/2h ceilings). + +Sources: learn.microsoft.com incremental-refresh-overview / -configure / -xmla / -troubleshoot; repo te-cli command-reference, workflows; repo tmdl object-properties; repo c-sharp-scripting partitions + +## Detect data changes and custom polling queries + +An optional refinement: instead of unconditionally re-querying every period in the incremental window, the service tracks the max of a dedicated audit date/time column (e.g. `ModifiedDate`) per period and skips periods whose max hasn't moved. This can collapse a 3-day refresh to 1 day or fewer when most days are quiet, cutting work without shrinking the window (so you keep a late-arrival safety margin). Hard constraints: the audit column must differ from the `RangeStart`/`RangeEnd` partition column (same column = no signal); default behavior caches that column into memory for comparison, costing RAM proportional to cardinality (reduce it first, or use a polling query to avoid materializing it); it detects soft deletes only ; a hard delete (row physically gone) is invisible. + +A custom polling query (Premium, TOM/TMSL only) sets `pollingExpression` to a lightweight M scalar run once per partition; a changed scalar flags that partition for full processing. This avoids caching the audit column and lets an ETL process drive refresh by writing a control table the polling expression reads, so a backdated change to one month reprocesses one month cheaply. No Desktop UI ; set via te-cli (if exposed), TOM, or TMSL. Microsoft's `120 months` granularity example is deliberate: a month-grain rolling window over 10 years lets a backdated change reprocess a single month but sacrifices some compression vs coarser yearly partitions ; surface that RAM-vs-refresh tradeoff, don't silently pick. + +Sources: learn.microsoft.com incremental-refresh-overview (real-time data); learn.microsoft.com incremental-refresh-xmla (custom queries for detect data changes); repo tmdl object-properties + +## Hybrid (real-time) tables: the DirectQuery partition and its blast radius + +Setting the policy `mode` to `hybrid` (Premium) appends one DirectQuery partition covering the slice newer than the incremental window; the table then serves imported history and live source rows in one query. Compat >= 1565, AS client libs >= 19.27.1.8. "Add one DQ partition" is misleadingly small: it converts the table to hybrid storage and propagates to related tables and report caching. Two consequences an agent must address: +1. **Related dimensions must move to Dual.** A hybrid table is queried in both Import and DQ contexts; any related table must be Dual or the relationship degrades to limited (over-fetch, slow). Desktop reminds on toggle but does not auto-fix import dims (a DQ dim flips to Dual trivially; an import dim must be recreated in DQ then switched by hand). Through TOM/te-cli there is no reminder, so check every related table's mode after enabling hybrid +2. **Report visuals cache and won't show the live partition by default.** Power BI caches visual results, defeating the DQ partition unless reports use Automatic Page Refresh (fixed-interval, or change-detection ; the latter Premium-only). This is a report-side setting; the model change alone doesn't deliver real-time + +"Only refresh complete days" is mandatory under hybrid (auto-enabled) ; with partial periods allowed, the boundary between the live DQ partition and the newest import partition can double-count or drop rows mid-day. It is also useful standalone when partial-day metrics are meaningless or upstream data finalizes late (set incremental period = 1 month, schedule for the close date). Service refreshes run in UTC unless you set a refresh time zone, which shifts what counts as a complete day. + +Sources: learn.microsoft.com incremental-refresh-xmla (partitions); learn.microsoft.com incremental-refresh-troubleshoot (hybrid in the service); learn.microsoft.com incremental-refresh-overview + +## Refresh-strategy decision guide for large fact tables + +A decision path before reaching for configuration, since agents tend to jump to "enable incremental refresh" when the real constraint is folding, freshness, or capacity tier ; picking the wrong layer wastes a one-way-door publish: +- **Does the source query-fold on the date filter?** If not, incremental refresh is off the table until folding is fixed (the per-partition queries won't filter at the source and the initial refresh times out). Verify with a tracing tool that one folded query carries the `RangeStart`/`RangeEnd` filter +- **Large but static history?** Plain incremental refresh (import); archive window to reporting need, incremental window to the late-arrival margin +- **Most of the window quiet day-to-day?** Add detect-data-changes (or a polling expression for ETL-driven control); watch the audit-column RAM cost +- **Sub-hour freshness on the newest slice?** Hybrid (Premium) + Dual dimensions + report Automatic Page Refresh; accept the storage-mode and caching complexity +- **Initial load can't finish?** Bootstrap the first refresh on Premium (create partitions empty, backfill via XMLA) and enable Large model storage beforehand; for small per-external-query sources (ADX, Log Analytics, App Insights) shrink store/refresh granularity to avoid truncation + +After publish you never touch Desktop again for that model: inspect with `te incremental-refresh show`, trigger targeted refresh with `te refresh --table <t> --partition "<t>.<part>" --type full` (note `--apply-refresh-policy true` is the default and re-evaluates the rolling window; pass `false` to refresh data without rolling it). Fix a backdated-data conflict by refreshing every partition from the change point to current to keep the one-side key unique. Metadata-only changes deploy through XMLA (ALM Toolkit, TMSL, `te deploy --skip-refresh-policy`), never a re-publish. + +Sources: learn.microsoft.com incremental-refresh-troubleshoot / -overview / -xmla; repo te-cli command-reference; repo refresh-semantic-model SKILL diff --git a/plugins/semantic-models/skills/semantic-model/references/metadata-and-organization.md b/plugins/semantic-models/skills/semantic-model/references/metadata-and-organization.md new file mode 100644 index 00000000..95744c47 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/metadata-and-organization.md @@ -0,0 +1,45 @@ +# Metadata, naming, and organization + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** `te set <obj> -q displayFolder -i "Sales" --save`, `te set <obj> -q isHidden -i true --save`, `te set <obj> -q formatString -i "..." --save`. To rename, `te mv` or `te set <obj> -q name`, but renaming breaks downstream references: follow `references/refactoring-renaming.md` first (lineage check, then propagate with pbir-cli / fabric-cli). + +## Measure tables: how many, where they sort, and the DirectQuery gotcha + +A measure table hosts measures so they don't clutter fact/dimension tables. Three decisions the one-liner hides: (1) **one vs many** ; a single `_Measures` is the default, but a model with hundreds of measures across disjoint subject areas benefits from several (`_Sales Measures`, `_Finance Measures`) giving a coarse first-level grouping before display folders, at the cost of more top-level field-list entries ; do not split a 20-measure model. (2) **sort position** ; measure tables sort alphabetically with everything, so a leading `_` pins them to the top (underscore sorts above letters and survives TMDL round-trips; avoid leading characters TMDL must quote). (3) **the placeholder column must stay but be invisible** ; the repo's `add-measure-table.csx` builds a single-row table and sets the lone column `IsHidden=true` and `IsAvailableInMDX=false` (hiding removes it from the field list; `isAvailableInMDX=false` stops MDX clients showing the stub and stops an unnecessary attribute hierarchy). + +Run the existing `add-measure-table.csx` and `move-measures-to-table.csx` scripts against a live/local model via `te ... --file --stage`, then `te validate` and `te save`. Pitfalls: an empty measure table from `Table.FromRows` is an **Import** partition ; in a pure DirectQuery or Direct Lake model that single-row Import table forces composite/mixed storage (build from a constant-folding source or accept the flag, do not assume it's free); give it no relationship (a stray one makes its hidden column filterable); a measure's home table is cosmetic (it doesn't change filter context) but moving a measure changes its qualified name's table part, breaking any visual that qualified it as `'OldTable'[Measure]` ; run the rename-cascade check after moving measures. + +Sources: repo c-sharp-scripting add-measure-table.csx / move-measures-to-table.csx; repo SpaceParts __Measures.tmdl; docs.tabulareditor.com creating-and-testing-dax + +## Display-folder structure: path syntax, nesting, and per-table scoping + +`DisplayFolder` groups columns/measures into a virtual tree. Non-obvious rules: (1) **nesting uses a forward slash** ; `"Columns/Keys"` creates `Columns` with child `Keys` (confirmed by the repo's `organize_folders.csx`); a backslash does not nest, only `/` does, so a wrong separator yields one literal folder `Columns\Keys`. (2) **folders are scoped to the home table** ; `"Metrics"` on a column in `Invoices` and `"Metrics"` in `Customers` are two unrelated folders that never merge (for one cross-table folder use perspectives or a measure table). (3) **measures and columns share the namespace within a table** ; on a measure table a shallow tree (`Time Intelligence`, `Ratios`, `Counts`) is right, and the scalable move is driving folders from naming convention (`YTD`/`MTD`/`QTD` -> Time Intelligence, `%`/`Rate` -> Ratios), as `organize_measures_by_type.csx` does. (4) empty/whitespace segments and leading/trailing slashes produce ghost folders ; trim first. + +Fold an entire model's measures by naming pattern in one pass with the shipped scripts, wiping inconsistent folders first (`clear_all_display_folders.csx`) so you don't leave orphans from a prior scheme, then organize. Pitfalls: a folder on a hidden object is invisible work (hidden objects don't appear in the field list at all); folders are not security or a perspective (everything is still queryable and visible to Copilot, folders only tidy the human browse); TMDL stores `displayFolder` as a plain property line with `/` separators, no quoting unless it contains a reserved character. + +Sources: repo c-sharp-scripting organize_folders.csx / organize_measures_by_type.csx / clear_all_display_folders.csx; repo tmdl object-properties + +## Perspectives as an AI/Copilot scope, not security + +A perspective is a named subset of tables/columns/measures/hierarchies. The forward-looking reason to build one: it is a meaningful scoping surface for Copilot/Q&A (a focused perspective constrains what the experience reasons over, reducing the "too many fields, ambiguous names" failure) ; a different lever than hiding (hiding is global, a perspective is a named view you point a specific consumption surface or audience at). Hard rule: perspectives are usability, not security ; anyone who reaches one can still query every table with hand-written DAX ("perspectives are not security") ; use OLS/RLS to actually deny access, and never present a perspective to a stakeholder as access control. Maintenance cost is real: a perspective must be re-synced whenever you add a measure/column or new fields silently fall out. + +Use the shipped perspective scripts; the key operational pattern defines membership from the model's own visible/hidden state (the "Sync Perspective with Hidden Status" pattern: loop `Model.Tables.Where(t => !t.IsHidden)` and set `InPerspective[name]=true` for non-hidden children) so the perspective tracks hide decisions and stays in sync with one re-run after model changes. Audit contents before trusting one. Editing perspectives requires unlocking "Allow unsupported Power BI features" in the GUI; via `te`/TOM/TMDL there is no such gate (the property is directly settable, another reason to drive this from the CLI). A perspective should be additive from empty (add only what the audience needs), because new tables default to **excluded** from existing perspectives, so a subtractive "everything minus a few" mental model drifts as the model grows. Test in the actual consumption surface (Excel honors perspectives; some web experiences historically ignored them); include columns a visible measure depends on, or the perspective confuses a human/Copilot browsing it (the measure still computes). + +Sources: repo c-sharp-scripting perspectives examples / object-types; repo SpaceParts Measure Selection.tmdl; learn.microsoft.com copilot-semantic-models; mssqltips.com perspectives-in-power-bi; docs.tabulareditor.com + +## The "No Measure Selection" perspective pattern for calc-group defaults + +The SpaceParts model ships two near-empty perspectives, `Measure Selection` and `No Measure Selection`, with nothing explaining them ; a specific Tabular Editor idiom worth documenting so an agent encountering them knows they're load-bearing. When a model has a calculation group and a report uses `SELECTEDMEASURE()`-driven items, a visual with no explicit measure falls into the calc group's multiple-or-empty branch and returns `BLANK()`, leaving cards empty until the user picks a measure. The paired perspectives are the convention for wiring a **default measure** so something renders before selection: one represents the chosen state, the other the empty state, and the report (or a field-parameter/default-member setup) switches between them. The perspectives carry no DAX ; they are markers the report layer keys off. + +Recognize, don't delete (run the rename-cascade/downstream-report check before removing either). Diagnose blank visuals at the calc group's empty/multiple-selection branch (give it a sensible default measure), not in the perspectives ; the perspectives organize which measures are offered, the BLANK comes from the calc-group expression. This is a convention, not an engine feature (there is no "default measure" property), so it only works if the report and any field-parameter wiring agree with the perspective names ; renaming a perspective without updating the report silently breaks the default-measure UX. Don't confuse it with security or AI scoping. + +Sources: repo SpaceParts Measure Selection.tmdl / No Measure Selection.tmdl; community.fabric.microsoft.com (Tabular Editor calculation groups thread); learn.microsoft.com calculation-groups + +## Cultures, metadata translations, and perspectives (full-coverage workflow) + +A culture (locale, e.g. `fr-FR`) carries metadata translations: per-object translated caption, description, and displayFolder for tables, columns, measures, hierarchies. The client picks the culture via the connection (`LocaleIdentifier=1036`, or the Analyze-in-Excel language drop-down) and the engine returns translated **names only**. Translations and perspectives are commonly half-done (a culture with three measures translated and forty not, so a user sees a jarring mix); there is no per-object fallback beyond "fall back to the default culture string," so treat partial translation as a finding, not a feature. Constraints: all translations share one collation (you cannot natively sort French and Japanese in the same model); translations are metadata only, never data values (a `Color` column's "Red" stays "Red" ; translated data is an upstream ETL problem). + +Create the culture, then translate ; properties are bracket-indexed by culture (`TranslatedNames[<culture>]`, `TranslatedDescriptions[<culture>]`, the display-folder equivalent ; confirm its exact name first). Hand-setting per object does not scale and silently leaves gaps; drive completeness from `te script` iterating every visible Table/Column/Measure/Hierarchy, looking up each translation, and reporting or filling blanks (the only reliable way to **audit** coverage ; there is no `te` subcommand for a translation-coverage report). For perspectives, go beyond bare membership and build additive from empty (new tables default to excluded, so a subtractive model drifts as the model grows). Pitfalls: report visuals bind to the object name not the translated caption, so translating does not break reports but also does not localize hard-coded report titles; the Power BI service does not expose a culture picker like Excel/AS clients, so metadata translations land mainly for Excel/Analyze-in-Excel and XMLA clients (a report will not auto-switch language from these alone); perspectives are not a security boundary; adding a table to the model does not add it to existing perspectives or translate it (both are maintenance surfaces that silently rot ; re-check coverage after any model growth). + +Sources: learn.microsoft.com translations-in-tabular-models; learn.microsoft.com translation-support-in-analysis-services; learn.microsoft.com tmdl-reference-tabular-object (translations); repo te-cli workflows / command-reference diff --git a/plugins/semantic-models/skills/semantic-model/references/parameters.md b/plugins/semantic-models/skills/semantic-model/references/parameters.md new file mode 100644 index 00000000..b2300a02 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/parameters.md @@ -0,0 +1,34 @@ +# Field and what-if parameters + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** build field parameters with the `c-sharp-scripting` macro via `te script` (do not hand-author the DAX + annotations through `te add`). A what-if is `te add table` with a `GENERATESERIES` partition plus a `SELECTEDVALUE` measure. Verify with `te get "<FP>" --output-format tmdl`. + +## Field parameter structure, ordering, and the sort column + +A field parameter is a calculated table whose import partition is a DAX constructor of 3-tuples `("Label", NAMEOF([Measure or Column]), <sortIndex>)`, projected into three columns with the second tagged `extendedProperty ParameterMetadata = { "version": 3, "kind": 2 }`. Three things break silently if wrong: +- The label column (`Value1`) must carry `sortByColumn` pointing at the order column (`Value3`), or the slicer sorts labels alphabetically instead of business order +- `ParameterMetadata` goes on the hidden `NAMEOF` column (`Value2`), not the label or table; omit it and the slicer's field-swap never activates +- `relatedColumnDetails/groupByColumn` on the label binds it back to the hidden column so a selection swaps the field; lose it and the slicer filters rows without swapping + +Ordering has two independent mechanisms that fight: the `Value3` index controls slicer order, but the order fields appear **inside** the target visual is driven by selection order at runtime (regular slicer) or hierarchy order (hierarchy slicer). So `Value3` is a slicer-presentation concern only; do not assume it controls matrix column order. + +Do not hand-author the DAX + annotations through `te add`/`te set` (the repo's te-cli workflow flags it as error-prone). Run the field-parameter macro from `c-sharp-scripting` via `te script`, which builds the table, constructor, extended property, and sort binding in one pass; verify with `te get 'FP - MTD' --output-format tmdl` that `ParameterMetadata` sits on the hidden column and `sortByColumn` on the label. Fall back to `create-field-parameter.ps1` (connect-pbid) for a live local instance, then mirror the `FP - MTD.tmdl` example as a last resort. Review findings: a field parameter references only explicit measures/columns by `NAMEOF` (no implicit aggregation); it is not valid as a drillthrough/tooltip linked field (link the underlying fields); selecting zero items equals selecting all (no empty state); it needs a local model on live-connect (composite); it is unsupported in Q&A/AI visuals; keep `Value3` a dense integer (0,1,2,...) or order is nondeterministic. + +Sources: learn.microsoft.com power-bi-field-parameters; repo SpaceParts FP - MTD.tmdl; repo te-cli workflows; repo create-field-parameter.ps1 + +## What-if (numeric range) parameters + +A what-if parameter produces two objects from one Desktop dialog: a calculated table of evenly spaced values via `GENERATESERIES(min, max, increment)`, and a measure `SELECTEDVALUE([col], default)` returning the picked value. It is a scenario input (discount rate, FX, threshold), distinct from a field parameter (swaps fields) and a dynamic M query parameter (folds a value into the source). The decision rule: swap which measure/dimension a visual shows means field parameter; let the user feed a scalar into a calculation means what-if; push a value into the source query for server-side folding means dynamic M query parameter (they are not interchangeable). A downstream measure consumes the Value measure, not the table. + +There is no special metadata flag, so standard `te` object commands fully build it: a calculated table with the `GENERATESERIES` source plus the value measure; set `formatString` (e.g. `0%`) and `summarizeBy: none` on the column so it stays a slicer dimension. Validate the series row count is `(max-min)/increment + 1`. Review findings: the table holds at most **1,000 unique values** ; beyond that Power BI evenly samples and silently drops granularity, so pick an increment keeping cardinality at or under 1,000, and flag any `GENERATESERIES` over it. Use the value in a measure, not a dimension/row-context calculation (the selection is not in scope there ; a `SELECTEDVALUE`-of-parameter inside a calculated column or grouping is a red flag). Always set a meaningful default (the `SELECTEDVALUE` second arg fires on multi-select and no-select; an omitted default blanks dependent measures when the slicer clears). `GENERATESERIES` is unsupported in DirectQuery for calculated columns/RLS; the what-if table is an Import calculated table (fine for Import/composite, materialized, non-foldable). + +Sources: learn.microsoft.com desktop-what-if; learn.microsoft.com generateseries-function-dax; learn.microsoft.com power-bi-visualization-troubleshoot; learn.microsoft.com desktop-slicer-numeric-range + +## Dynamic visual titles tied to a parameter (model-side measure) + +Field and what-if parameters pair with an expression-based title: a measure reflecting the current selection so the visual header narrates what is on screen. The model piece is a `SELECTEDVALUE`-based measure; the report binds it to the title via conditional formatting. The value of a parameter usually surfaces to the user through this title, and the fix lives in the model (a measure), reusable across pages, so it belongs in the semantic-model skill even though the binding is a report step. + +Add a measure that reads the parameter's visible label column (`"Showing: " & SELECTEDVALUE('FP - MTD'[FP - MTD], "All metrics")`), or for what-if surfaces the scalar (`"Discount: " & FORMAT([Discount percentage Value], "0%")`), then validate the string with `te query`. `SELECTEDVALUE` returns the default on both multi-select and no-select, so pick a default reading correctly in both ("All metrics", not ""). Reference the visible label column, not the hidden `NAMEOF` column. Keep the measure model-level (not a report-scoped extension measure) so every report reusing the model gets it. `USERCULTURE()` returns the user's culture only inside a measure (in a calculated column/table it returns the model default at load time), so keep dynamic-title logic in measures. + +Sources: learn.microsoft.com desktop-conditional-format-visual-titles; learn.microsoft.com power-bi-field-parameters diff --git a/plugins/semantic-models/skills/review-semantic-model/references/performance.md b/plugins/semantic-models/skills/semantic-model/references/performance.md similarity index 94% rename from plugins/semantic-models/skills/review-semantic-model/references/performance.md rename to plugins/semantic-models/skills/semantic-model/references/performance.md index ab04b76a..1171ba18 100644 --- a/plugins/semantic-models/skills/review-semantic-model/references/performance.md +++ b/plugins/semantic-models/skills/semantic-model/references/performance.md @@ -2,6 +2,8 @@ Guidance for evaluating semantic model performance: memory analysis, query optimization, unused column detection, and benchmarking. +**Working with `te`:** time a query with `te query -q "..." --trace --cold --runs 10` and compare medians; find unused objects with `te deps --unused` (confirm with `te get` before removing, keys can read as unused); read the model-size split with `te vertipaq --columns --detail`. Formula-engine vs storage-engine timings need a trace (connect-pbid / DAX Studio). + ## Performance Analysis Tools | Tool | What It Provides | When to Use | diff --git a/plugins/semantic-models/skills/semantic-model/references/query-semantic-model.md b/plugins/semantic-models/skills/semantic-model/references/query-semantic-model.md new file mode 100644 index 00000000..5223675e --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/query-semantic-model.md @@ -0,0 +1,39 @@ +# Querying a semantic model + +Companion to the `semantic-model` skill (SKILL.md). How to read data and metadata out of a model from the terminal with DAX, for validation, review, and probing. For query *performance* tuning, use the `dax` skill. + +**Working with `te`:** `te query -q "EVALUATE SUMMARIZECOLUMNS(...)"` (inline) or `te query -f query.dax` (from a file); add `--output-format json` for parseable results and `--output-file out.csv` to save. Target a local file with `-m ./model`, a remote model with `-s <workspace> -d <model>`. + +## Inline vs file + +Short probes go inline with `-q`; multi-line or reused queries go in a `.dax` file with `-f`. A file avoids shell-escaping the double quotes DAX uses for strings and column aliases, which is the main inline foot-gun. Always wrap a table expression in `EVALUATE`; wrap a scalar in `EVALUATE ROW("x", <scalar>)`. + +## Output formats + +`--output-format json` is the default to use when driving `te` programmatically (the text/table output mangles in transcripts). `csv`/`tsv` for tabular results, `--output-file <path>` to write to disk (format inferred from the extension). Errors and warnings go to stderr in JSON mode, so they never contaminate the parseable stdout. + +## Metadata via INFO functions + +The model's own schema is queryable as data, which is how you inspect a model that `te ls` cannot fully enumerate: + +- `EVALUATE INFO.VIEW.RELATIONSHIPS()` ; relationships with friendly from/to, cardinality, cross-filter, active flag (`te ls` cannot list relationships) +- `EVALUATE INFO.VIEW.MEASURES()` / `INFO.MEASURES()` ; measures, expressions, format strings, display folders +- `EVALUATE INFO.VIEW.COLUMNS()` / `INFO.TABLES()` ; columns and tables with data types and properties +- `EVALUATE INFO.STORAGETABLECOLUMNS()` / `INFO.DICTIONARYSTORAGES()` ; live cardinality and dictionary sizes when a VPAX export is not available + +## Probing patterns + +- Test a measure in isolation: `EVALUATE ROW("Result", [Total Revenue])` +- Visual-shaped query (what a report sends): `EVALUATE SUMMARIZECOLUMNS('Date'[Year], "Revenue", [Total Revenue])` +- Check column values / cardinality: `EVALUATE TOPN(20, VALUES('Geography'[Region]))`, `EVALUATE ROW("n", DISTINCTCOUNT('Sales'[CustomerKey]))` +- Emulate an RLS filter offline: `EVALUATE CALCULATETABLE(<query>, TREATAS({"alice@contoso.com"}, 'UserMap'[UserEmail]))` (proves the predicate; does not exercise the security context, see `security.md`) + +## Performance-aware querying + +For timing, `te query ... --trace --cold --runs 10` and compare medians, not means; discard the first cold run as warm-up. `--cold` clears the cache for a true cold measurement. Single runs are noise. Formula-engine vs storage-engine split needs a trace via `connect-pbid` or DAX Studio; `te query` returns the result, not the timings breakdown. + +## Pitfalls + +- Each Claude Bash call is a fresh shell, so a `te connect` from a prior call is gone; pass `-m` (and `-s`/`-d`) on every `te query`, or set `TE_SESSION`. +- `pbir model -q` (in the reports plugin) runs `EVALUATE` DAX only; `INFO.*` and DMV queries return 400 there. Use `te query` against the model endpoint for metadata. +- DMV-style `SELECT * FROM $SYSTEM.TMSCHEMA_*` works through ADOMD against a live local instance (`connect-pbid`), not through `te query`'s DAX path. diff --git a/plugins/semantic-models/skills/semantic-model/references/refactoring-renaming.md b/plugins/semantic-models/skills/semantic-model/references/refactoring-renaming.md new file mode 100644 index 00000000..996dcb80 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/refactoring-renaming.md @@ -0,0 +1,36 @@ +# Refactoring and renaming objects safely + +Companion to the `semantic-model` skill (SKILL.md). Renaming or moving a model object is never a local edit; the name is a contract that downstream consumers depend on. Run the lineage check first, then rename, then propagate. + +**Working with `te`:** rename with `te mv <old> <new> --save` or `te set <obj> -q name -i "<new>" --save`, but ONLY after the lineage check below. Never rename a model object in isolation. + +## Why renaming is dangerous + +A measure, column, or table name is referenced far beyond the model. Renaming breaks, often silently: + +- **Report visuals** bound to the old name ; the visual drops the field or errors, and Power BI does not auto-repair code-edited PBIR +- **Other model objects** (measures, calculated columns, calc items) that reference it ; `te validate` catches these, but only these +- **Downstream models** in a composite, and every report in any workspace that binds to this model +- **Bookmarks, report-level filters, conditional formatting, and field parameters** that name the old object + +`te`'s save-time validation sees model-internal breaks only. It cannot see reports or downstream models, so a structurally valid rename can still break production dashboards. + +## The safe rename workflow + +1. **Lineage check FIRST.** Find every consumer before touching the name: + - the `lineage-analysis` skill, or `fab` (fabric-cli), to list reports and downstream models bound to this model across workspaces + - `te deps "<obj>"` and `te find "<obj>" --in expressions --paths-only` for model-internal references +2. **Rename in the model:** `te mv` or `te set <obj> -q name`, then `te validate`. +3. **Propagate to reports:** rebind every affected visual, filter, and bookmark with the `pbir-cli` skill (`pbir` locates and updates the references); for service-side items use `fabric-cli` (`fab`). +4. **Re-validate:** `te validate` the model and `pbir validate` each report. + +## Coordinated rename (the te + pbir tandem) + +The canonical pattern: rename in the model with `te`, capture the old -> new map, then run the matching rename in `pbir-cli` against each downstream report so visual bindings, filters, and bookmarks follow. Treat the model rename and the report rename as one change set so the model and its reports never diverge in source control. + +## Pitfalls + +- Renaming a measure's **home table** also moves the `'OldTable'[Measure]` form that reports may use; check both the measure name and its table. +- Internal name vs display name can diverge; confirm what actually changed with `te get <obj> --output-format tmdl`. +- A **field parameter** references fields by `NAMEOF`; renaming a referenced field requires rebuilding the parameter, not just renaming. +- Renaming is the one model edit where "it validates" is not "it is safe"; the lineage step is mandatory, not optional. diff --git a/plugins/semantic-models/skills/semantic-model/references/relationships.md b/plugins/semantic-models/skills/semantic-model/references/relationships.md new file mode 100644 index 00000000..ccafca0f --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/relationships.md @@ -0,0 +1,46 @@ +# Relationships and cardinality + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** enumerate with `te query -q "EVALUATE INFO.VIEW.RELATIONSHIPS()"` (`te ls` cannot list relationships). Create with `te add relationship "Sales[K]->Date[K]" --save`; set cross-filter / active / security behavior with `te set Relationships/<name> -q <prop>`, or `te script` (TOM) when the property is not exposed. + +## Detecting limited relationships and the silent drops they cause + +Regular vs limited is inferred at evaluation time, not set. A relationship is limited when its cardinality is many-to-many (even if both columns hold unique values) or it crosses a source group (import-to-DQ, or DQ-to-different-DQ). Everything else 1:M or 1:1 inside one source group is regular. Direct Lake + import composites keep regular relationships across the two modes, unlike classic import + DirectQuery. + +The difference changes numbers silently: +- Regular 1:M joins LEFT OUTER and synthesizes a blank "unknown member" for unmatched many-side keys, so RI violations still appear in totals under (Blank) +- Limited joins INNER, adds no blank row, and drops unmatched rows entirely ; an orphan fact row just vanishes from every aggregate. No error, a quietly understated total +- `RELATED()` cannot traverse a limited relationship (it errors); table expansion never happens, so the join resolves per query in multiple passes and degrades fast above low-cardinality keys + +From a terminal there is no diagram, so infer and probe. `INFO.VIEW.RELATIONSHIPS()` lists cardinality, cross-filter behavior, and active flag (`te ls` cannot enumerate relationships). Flag any many-to-many row as limited; for composites cross-check storage modes and treat any cross-mode relationship as limited. Probe orphans with an anti-join (`EXCEPT`/`NATURALLEFTOUTERJOIN` on the keys) ; a non-zero orphan count on a regular relationship lands under (Blank), on a limited one those rows are silently gone (the dangerous case). If the m2m was an accident (transient duplicates at create time), fix the cardinality once the one-side is genuinely unique; confirm the property name on a live object first or fall to `te script`. Do not force cardinality on a real cross-source-group limited relationship ; no cardinality change makes it regular, so reduce cost instead (Dual on the shared dimension, or pull it into the Vertipaq group). + +Sources: learn.microsoft.com desktop-relationships-understand; learn.microsoft.com direct-lake-web-modeling (DL+import keep regular); SQLBI strong-and-weak-relationships; repo te-cli command-reference + +## Cardinality and uniqueness go unvalidated in Direct Lake and web modeling + +Import and DirectQuery profile columns when you create a relationship and auto-populate cardinality and direction. Direct Lake and web modeling do not. Direct Lake guesses the many side from a row-count DAX query, pre-sets single direction, and never checks one-side uniqueness; web modeling issues no validation queries at all, including for the marked date column. + +The hard rule: a Direct Lake relationship's one-side column must be unique, and if duplicates exist the **query fails at runtime**, not at refresh or edit time. So an agent can author a structurally valid model, pass every static check, deploy it, and have every visual touching that relationship error the first time a user opens the report because the Delta table had a duplicate key. The row-count heuristic also picks the wrong many side whenever the dimension is larger than the fact (a wide SCD against a sparse fact), producing a backwards relationship. + +Prove it yourself: `te query` `COUNTROWS(Product)` vs `DISTINCTCOUNT(Product[ProductKey])` (rows greater than distinct means duplicates ; de-dup upstream in the Delta table before the model is usable) and a non-zero blank-key count (a second uniqueness hazard). Confirm the heuristic assigned the right many side via `INFO.VIEW.RELATIONSHIPS()`; correct mis-assignment explicitly. After any Direct Lake relationship change, re-run a real visual-shaped `SUMMARIZECOLUMNS` query to actually exercise the join, since nothing else does until a user does. Direct Lake requires exact data-type match across the relationship (string vs int is rejected; Binary/GUID must be cast to string upstream), and Direct Lake on SQL needs an explicitly marked date table joined on a real date column ; no auto date-part leniency. + +Sources: learn.microsoft.com direct-lake-edit-tables (no preview, row-count heuristic, no validation); learn.microsoft.com direct-lake-overview (one-side unique or queries fail; type match); learn.microsoft.com desktop-create-and-manage-relationships + +## Resolving ambiguous filter paths deterministically (priority tiers and weight) + +Ambiguity has two causes, only one being the expected bidirectional case: (1) a bidirectional cross-filter opening a second route, (2) a diamond schema with two paths to the same target and no bidirectional filter at all (two bridges reaching one dimension). Power BI resolves with a fixed priority-tier sequence, first matching tier wins: 1:M-only paths, then 1:M-or-M:M, then M:1, then 1:M-to-intermediate-then-M:1, then the same allowing M:M legs, then anything else. A relationship in every candidate path is dropped from consideration. Within one tier, path weight is the maximum weight of its relationships (count is irrelevant) and the higher-weight path wins; the engine never crosses tiers to chase weight. A same-tier same-weight tie is a hard ambiguous-path error, not a silent pick. + +Two failure modes that look nothing alike: sometimes Desktop refuses a bidirectional change at edit time (safe), other times it accepts the topology and silently routes filters down the tier-1 path, so a measure returns plausible-but-wrong numbers. An agent that "just adds the relationship the measure needs" can flip the chosen path and change every cross-table total with no error. `USERELATIONSHIP` is the deliberate lever ; it raises a relationship's weight (innermost nested call gets the highest), activating an inactive relationship and breaking a same-tier weight tie toward the path you want. + +Map the topology before and after any edit with `INFO.VIEW.RELATIONSHIPS()` plus `te deps`; any two tables reachable by more than one active path (including a bidirectional return route) are an ambiguity candidate. For a genuine diamond, keep one path active and others inactive, then select per measure with `USERELATIONSHIP` rather than leaving the engine to pick. If activating one path still raises ambiguity, nest a second `USERELATIONSHIP` to force ordering. Prefer the documented bidirectional alternatives (a visual-level "is not blank" filter, or `CROSSFILTER(..., BOTH)` scoped to one measure) over a model-level bidirectional flag, which is the usual source of accidental ambiguity. `USERELATIONSHIP` in a calculated column does nothing (row context) ; use `LOOKUPVALUE`. + +Sources: learn.microsoft.com desktop-relationships-understand (resolve path ambiguity); SQLBI bidirectional-relationships-and-ambiguity-in-dax; tabulareditor.com ambiguous-paths-in-power-bi + +## Active/inactive relationships and the dead-inactive-relationship defect + +A model holds at most one active path between two tables; extras must be inactive. Inactive relationships still participate in table expansion (so a regular one still costs refresh-time index build) but propagate no filter until a calculation wraps the query in `USERELATIONSHIP`. The common real-world bug, confirmed by heavy forum traffic on inactive-relationship date filtering, is an inactive relationship that no measure ever activates: it sits there, the author wires a date or visual to it expecting filtering, and gets either an unfiltered result or the active relationship's behavior. Treat an inactive relationship with zero `USERELATIONSHIP` references as a defect, the inverse of "missing relationship." + +Cross-reference in one pass: list inactive relationships with `FILTER(INFO.VIEW.RELATIONSHIPS(), NOT [IsActive])`, then `te find "USERELATIONSHIP" --in expressions --paths-only`. Any inactive relationship whose columns never appear in a `USERELATIONSHIP` call is dead weight or a latent bug ; either a measure is missing, the relationship should be deleted, or (role-playing case) the dimension should have been duplicated per role with its own active relationship. Duplicate the physical dimension per role unless simultaneous multi-role filtering is genuinely needed; the shared-table-plus-inactive pattern forces a `USERELATIONSHIP` wrapper into every measure and breaks Q&A/Copilot, which cannot inject one. `USERELATIONSHIP` on an RLS-bearing relationship is blocked; relocate the filter instead. + +Sources: learn.microsoft.com desktop-relationships-understand (make-this-relationship-active); learn.microsoft.com relationships-active-inactive; local forums DB (inactive-relationship date-filter threads); repo te-cli semantic-modeling-practices diff --git a/plugins/semantic-models/skills/semantic-model/references/review-checklist.md b/plugins/semantic-models/skills/semantic-model/references/review-checklist.md new file mode 100644 index 00000000..e423bfb0 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/review-checklist.md @@ -0,0 +1,63 @@ +# Review checklist + +Companion to the `semantic-model` skill (SKILL.md). The full audit workflow and per-category checks. Drives inspection through the te-cli-first cascade; produces prioritized, actionable findings rather than a pass/fail. + +**Working with `te`:** gather context with `scripts/get_model_info.py`; inspect with `te load`, `te ls Measures`, `te query -q "EVALUATE INFO.VIEW.RELATIONSHIPS()"`, and `te vertipaq --columns --detail`; gate findings with `te validate` and `te bpa run --fail-on error`. + +## Workflow + +### Step 0: gather context +Run `scripts/get_model_info.py -w <workspace-id> -m <model-id>` for storage mode, model size, connected reports, deployment pipeline, endorsement, sensitivity label, data sources, refresh schedule, last refresh, capacity SKU. Then ask the user: what business process the model serves; who consumes it (report developers, analysts, executives, Copilot/AI); whether they own the model, the reports, or both; whether it is in dev, test, or production; and where findings should be documented. Severity shifts with context: a model for three analysts is judged differently from one Copilot queries org-wide. + +### Step 1: inspect structure +Read the model with the cascade. `te load ./model` for a summary, `te ls Measures` / `te ls Tables`, `te query -q "EVALUATE INFO.VIEW.RELATIONSHIPS()"` for relationships (`te ls` cannot enumerate them), `te vertipaq --columns --detail` for size. Drop to `connect-pbid` for traces and storage DMVs when the endpoint is unreachable from `te`. + +### Step 2: audit by category +Walk the categories below; each links to the topic reference with the mechanics and the fix. + +### Step 3: performance +Query-level timing, unused-column detection, and memory analysis in `performance.md`; model-size diagnosis in `vertipaq-optimization.md`. + +### Step 4: report findings +Produce a markdown report: a count-by-severity summary, detailed findings with object paths and line numbers, a specific remediation per finding, and a prioritized action list (critical first). + +## Categories + +### Critical +- Bidirectional relationships creating ambiguous filter paths, and circular dependencies between tables (`relationships.md`) +- Missing data types on columns; orphaned tables with no relationship +- Fail-open RLS (`IF(..., TRUE())` fall-through) and limited relationships that silently drop unmatched rows (`security.md`, `relationships.md`) + +### Memory and size (`vertipaq-optimization.md`) +- High-cardinality dictionaries (GUIDs, transaction IDs, composite keys); unsplit DateTime columns +- `isAvailableInMDX` left on hidden or high-cardinality columns (wasted attribute-hierarchy memory) +- Auto date/time tables (hidden `LocalDateTable_*`); wrong data types (Double for currency, String for numeric) +- Calculated columns that should be measures (`dax-authoring.md`); unused columns or tables + +### Data reduction +- Fact history with no date-range filter or incremental refresh (`incremental-refresh.md`) +- Columns not needed for reporting or calculations; detail grain finer than any report needs; logic better done upstream + +### DAX correctness (`dax-authoring.md`; for query tuning use the `dax` skill) +- Filtering whole tables instead of columns in CALCULATE; unguarded division (`DIVIDE` vs bare `/`) +- Context-blind calculated columns using CALCULATE; variable time-shift bugs + +### Measure hygiene +- Implicit measures where explicit measures should exist; report-scoped extension measures that belong in the model; ambiguous duplicate measure names + +### Documentation and AI (`ai-copilot-readiness.md`, `metadata-and-organization.md`) +- Tables/columns/measures missing descriptions (Copilot truncates after 200 characters; front-load keywords) +- Missing display folders; missing synonyms; inconsistent naming (use `standardize-naming-conventions`) + +### Design (`dimensional-modeling.md`, `relationships.md`, `time-intelligence.md`) +- Star-schema violations (fact-to-fact joins, snowflakes); many-to-many without a bridge +- Date table not correctly marked for the relationship column type; dead inactive relationships (no `USERELATIONSHIP`) +- Multiple facts on the same dimension via different keys without a conformed dimension + +### Direct Lake, if applicable (`direct-lake.md`) +- Non-unique one-side relationship keys (queries fail at runtime, not refresh); DirectQuery fallback risk (RLS, SQL views) +- Calculated columns on Direct Lake tables; Delta health (Parquet file count, V-Order, guardrails) + +## Notes +- The structural audit reads metadata; it does not execute report DAX or check data quality +- For companion report review, use the `review-report` skill (reports plugin) diff --git a/plugins/semantic-models/skills/semantic-model/references/security.md b/plugins/semantic-models/skills/semantic-model/references/security.md new file mode 100644 index 00000000..f2769322 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/security.md @@ -0,0 +1,55 @@ +# Row-level and object-level security + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** `te add role "Region" --save`, then the table filter via `te set` / `te script`. `te` has no impersonation flag: validate with `te test` (wrap filters in `TREATAS`) or connect-pbid (`Roles=` / `EffectiveUserName=`). Audit fail-open filters with `te find "TRUE" --in expressions --paths-only`. + +## Validating RLS roles before deploy (te-cli has no impersonation flag) + +A wrong RLS filter is a data leak, not a noticeable bug. The trap for agents: `te query`/`te test` expose no `--as-role`/effective-identity option (only `-q/-f/--limit/--trace/--cold/--plan/--runs`), so you cannot ask the engine "render as Alice." Validate by other means and know which actually exercises the security layer: +1. **Emulate the filter as DAX (offline).** Substitute a literal for the identity function: `CALCULATETABLE(SUMMARIZECOLUMNS(...), TREATAS({"alice@contoso.com"}, 'UserRegionMap'[UserEmail]))`. This proves the predicate is well-formed and returns the intended rows, but does not exercise the security context, so it misses ambiguous-path/propagation surprises. Always assert the adversarial case: an unmapped UPN must return zero rows +2. **Bake these into `te test`** (`te test init --from-model`) wrapping each role's filter in `CALCULATETABLE(..., TREATAS(...))`, asserting counts for a known and an unmapped identity, run as a gate (`te test run --fail-on error`) +3. **True security-context validation needs a deployed model and connection-string impersonation, not te-cli.** Tabular Editor appends `Roles=` + `EffectiveUserName=` (+ `CustomData=` since TE3 3.25) to the XMLA string (the impersonated user needs Build + role assignment); if `te` exposes no flag, fall to TOM/ADOMD via connect-pbid opening a connection with `Roles=...;EffectiveUserName=...`, or test in the service with View as / Test as role + +Test as role uses your identity for the DAX functions ; `USERPRINCIPALNAME()` returns the tester's UPN unless you supply "Other user" + role, and you cannot see what a B2B guest or service principal sees without signing in as them (docs admit it "doesn't fully replicate the authentication context"). View as roles does not work for DirectQuery with SSO ; validate by signing in as a real Viewer. Test 3-10 distinct identities per role, including out-of-band values. + +Sources: learn.microsoft.com service-admin-row-level-security (validating role, Test as role); learn.microsoft.com rls-guidance (validate roles); docs.tabulareditor.com data-security-testing; repo te-cli command-reference + +## Defensive dynamic-RLS filter expressions (fail closed, not open) + +Write the role filter so unexpected or malicious identity values return zero rows. The common `IF(USERNAME()="Worker", [Type]="Internal", TRUE())` fails **open** ; any value that is not "Worker" (a typo, or an injected value in an embedded "embed for your customers" app that can pass arbitrary effective usernames) hits the `TRUE()` branch and returns everything. Exploitable, not theoretical. Match every expected value explicitly and make the fall-through `FALSE()`. The mapping-table pattern (`[Region] = USERPRINCIPALNAME()` against a table with no row for an unmapped UPN) is fail-closed automatically; the danger is only the `IF(..., TRUE())` style. Audit with `te find "TRUE" --in expressions --paths-only`. + +`USERNAME()` returns `DOMAIN\user` in Desktop but the UPN in the service; standardize on `USERPRINCIPALNAME()` and store that exact sign-in UPN (not the Entra `mail` alias) in the mapping table. B2B guest UPN format is not guaranteed (`user@partner.com` vs `user_partner.com#EXT#@tenant.onmicrosoft.com`); a format mismatch silently shows the guest no data ; verify with a `[WhoAmI]=USERPRINCIPALNAME()` card. A user in no role sees no rows once any role exists; a user seeing everything is usually a workspace Admin/Member/Contributor (Edit bypasses RLS) or in a fail-open role. In embedded service-principal flows the DAX identity functions return the app ID or empty string, so per-user RLS there comes from the REST `EffectiveIdentity` (and `CUSTOMDATA()`), never the DAX functions. + +Sources: learn.microsoft.com rls-guidance (defensive filters); learn.microsoft.com service-admin-row-level-security (dynamic RLS considerations); learn.microsoft.com embedded-row-level-security + +## Bidirectional cross-filtering with RLS (mechanics and the one-relationship constraint) + +By default RLS propagation is single-directional regardless of the relationship's cross-filter setting; a bidirectional relationship does not push the RLS filter both ways. You opt in per relationship via "apply security filter in both directions" (`securityFilteringBehavior = BothDirections`). This is distinct from the modeling rule "avoid model-level bidirectional" ; here the toggle is sometimes required, not a smell. Canonical case: dynamic RLS where the secured user-mapping table on the one-side must filter through a bridge to reach the fact; without the opt-in the filter does not reach across and users see wrong rows. + +The property is not in the documented `te set -q` list, so confirm it on a real object (`te get 'Relationships/<name>'`), then set it or drop to TOM (`SecurityFilteringBehavior.BothDirections`). Hard constraint: if a table participates in multiple bidirectional relationships, the security filter can be enabled on only **one** of them (the others stay bidi for normal cross-filtering); pick the path the RLS filter must travel. `securityFilteringBehavior` is independent of `crossFilteringBehavior` (two separate properties; one does not imply the other). It degrades query performance (extra propagation passes), so use it only where the RLS path needs it, and it only changes RLS propagation ; it does not fix ambiguous-path numbers in normal cross-filtering. + +Sources: learn.microsoft.com service-admin-row-level-security (define roles and rules); learn.microsoft.com desktop-bidirectional-filtering + +## OLS hard restrictions (relationship chains, RLS+OLS cross-role error, measure propagation, composite leakage) + +Object-level security secures tables/columns (data + name + metadata) via per-role `metadataPermission = None`. Most OLS failures are not "the column is still visible" but design-time errors, query-time errors for specific role combinations, or silent over/under-exposure through composite chaining: +- **Securing a table cannot break a relationship chain** (design-time error): with A->B->C you cannot secure B; fix with a direct A->C relationship or secure a leaf. `te validate` catches it +- **RLS and OLS from different roles cannot combine for one user** (query-time error): keep both kinds of restriction for a user inside one combined role +- **Measures are auto-restricted by reference, never explicitly** (no measure-level OLS): a measure touching a secured object becomes restricted (same for KPIs, DetailRows). To deliberately hide a measure, rewrite it to touch a secured object (the sentinel workaround); to avoid accidentally killing a measure, check `te deps` before securing a column +- **A relationship may reference a secured column only if the column's table is not itself secured** +- **Composite-model OLS leakage**: OLS is enforced only on the source model; the composite author copies whatever schema they can see, so a downstream consumer may see metadata hidden from them (or miss entitled objects) because the composite froze the author's view. Actual data never leaks (DirectQuery re-evaluates OLS at the source and errors for unauthorized columns), but the metadata experience is wrong. Treat OLS as non-composable + +OLS applies only to Viewers (Admin/Member/Contributor bypass). Power BI features unsupported with any OLS: Q&A, Quick Insights, Smart Narrative, Excel Data Types ; if a model needs Copilot/Q&A, OLS may be the wrong tool (consider RLS-only plus separate models). Perspectives are not security. There is no Desktop UI for OLS, so the `te`/TOM/TMDL path is the only route. + +Sources: learn.microsoft.com object-level-security; learn.microsoft.com service-admin-object-level-security; learn.microsoft.com desktop-composite-models; learn.microsoft.com powerbi-implementation-planning-security-report-consumer-planning + +## User-defined aggregations: grain design, agg-awareness, precedence, RLS + +A user-defined aggregation is a hidden Import (or DQ) table holding a coarser-grain pre-aggregate of a large DQ detail table; the engine transparently rewrites qualifying DAX subqueries to it. Encoded as `alternateOf` on each agg column (`baseColumn`+`summarization`, or `baseTable`+`groupByColumns`) plus the table mode ; no special object. Two techniques with different rules: **relationship-based** (star sources, agg relates to the same dimensions) where GroupBy entries are optional except DISTINCTCOUNT needs an explicit GroupBy on the key; **GroupBy-based** (denormalized, no relationships) where GroupBy entries are mandatory or the agg never hits. + +Agg-awareness works through DAX the engine doesn't see literally: COUNTROWS hits a count agg; AVERAGE folds to SUM/COUNT and hits if both exist; time-intelligence hits only if the agg covers the grain it expands to (DATESYTD expands to day grain, so a month-grain agg misses); DISTINCTCOUNT can hit if the key is a GroupBy column. The load-bearing grain rule: an agg should be at least ~10x fewer rows than its detail or maintenance outweighs speedup ; strip high-cardinality attributes (lat/long, timestamps) from the GroupBy set. **Precedence** lets one subquery try a smaller/coarser agg first (higher `precedence` integer) and fall to a larger agg then the detail table; chained aggregations are illegal (`detailTable` must point at the real detail, never another agg). + +`te` v0.2.0 has no `-q AlternateOf`, so this is the canonical `te script` fallback: build the scaffolding (agg table, hide it, set columns) with `te`-native verbs, then map via TOM (`new TOM.AlternateOf { Summarization = ..., BaseColumn = ... }`). Audit existing mappings and check the 10x ratio with `te query`. Verify hits at runtime via a trace (connect-pbid or DAX Studio) watching `Aggregate Table Rewrite Query` ; `te query` alone won't show cache-hit vs source. Rules `te` won't warn about: RLS must filter both agg and detail or the engine refuses to answer from the agg; inactive-relationship + USERELATIONSHIP for an agg hit is unsupported (use TREATAS); `detailTable` must be DirectQuery; max 3-table chains; `AlternateOf` needs compat >= 1460; Import aggs are ignored on SSO-enabled sources since Aug 2022; hybrid tables and Direct Lake (either flavor) do not support user-defined aggregations (pre-aggregate in the Lakehouse). + +Sources: learn.microsoft.com aggregations-advanced (+ precedence); learn.microsoft.com composite-model-guidance; learn.microsoft.com SummarizationType / AlternateOf; learn.microsoft.com aggregations-auto diff --git a/plugins/semantic-models/skills/semantic-model/references/storage-modes.md b/plugins/semantic-models/skills/semantic-model/references/storage-modes.md new file mode 100644 index 00000000..c64870a0 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/storage-modes.md @@ -0,0 +1,15 @@ +# Storage modes (Import / DirectQuery / Dual / Hybrid) + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** read each table mode / source with `te script` over `Partitions[0]`; set the mode via `te script` (TOM `ModeType` = `import|directQuery|dual|directLake`). After any change, re-list relationships with `te query -q "EVALUATE INFO.VIEW.RELATIONSHIPS()"` to catch newly limited ones. + +## Storage-mode decision matrix (Import / DirectQuery / Dual / Hybrid / Direct Lake) + +Storage mode is a per-table partition property deciding where a DAX subquery is answered: VertiPaq cache (Import, Direct Lake after transcoding) or the source via native query (DirectQuery). Dual is both, decided per query. Hybrid is import partitions plus one DQ partition. Mixed modes = composite model. Direct Lake has two flavors: on OneLake (composites with Import/DQ/Dual) and on SQL (single-source, no mixing, falls back to DQ on views/granular RLS). + +The biggest non-obvious lever: a dimension left **Import** on the one-side of a **DirectQuery** fact forces a cross-source-group path ; its groupings ship to the source as materialized subqueries and the relationship goes **limited**. Setting it **Dual** keeps the join intra-source-group and **regular**, so one native query handles slicer+fact, with slicers served from cache. You only get both behaviors from one table if it is Dual, not Import. Hybrid is the move for "latest live, history fast" (import bulk + one DQ tail), but hybrid tables do not support aggregations. Direct Lake on SQL cannot mix with DQ/Dual; on OneLake it can. + +Mode-to-when: DirectQuery for large facts / near-real-time / unimportable volume; Import for tables not filtering a DQ/Hybrid fact, unreachable sources, all calculated tables; Dual for dimensions queried with a DQ/Hybrid fact from the same source; Hybrid for one fact needing live latest + fast history; Direct Lake on OneLake for very large Delta facts you do not refresh (composites with Import dims); Direct Lake on SQL for single-source Delta. There is no headless storage-mode dropdown ; mode is `partition.Mode` (`import|directQuery|dual|directLake`), set via `te script` (TOM `ModeType`). After any change, list relationship types (`INFO.VIEW.RELATIONSHIPS()`) to confirm you did not create limited relationships. Calculated tables are always Import regardless of what they reference; "Mixed" in Desktop is a UI label, not a fourth mode. + +Sources: learn.microsoft.com service-dataset-modes-understand; learn.microsoft.com composite-model-guidance; learn.microsoft.com direct-lake-overview; learn.microsoft.com directquery-model-guidance diff --git a/plugins/semantic-models/skills/semantic-model/references/time-intelligence.md b/plugins/semantic-models/skills/semantic-model/references/time-intelligence.md new file mode 100644 index 00000000..e0a2cdec --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/time-intelligence.md @@ -0,0 +1,40 @@ +# Date tables and time intelligence + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** mark the date table via `te script` (TOM `DataCategory="Time"`, key column `IsKey`); add TI measures with `te add measure ... --save` and validate by executing them with `te query`. Calendar (Enhanced TI) objects TOM cannot reach drop to TMDL (the `tmdl` skill). + +## Classic vs calendar-based time intelligence (which engine the model commits to) + +Power BI ships two time-intelligence engines and a model effectively commits to one. Classic uses a marked date table plus functions like `SAMEPERIODLASTYEAR('Date'[Date])`. Calendar-based (the Enhanced DAX TI preview) adds a `calendar` object onto a table; functions reference it by name, e.g. `TOTALYTD([Sales], 'Fiscal Calendar')`, and the engine reads the underlying period columns as-is rather than assuming Gregorian. + +The choice changes the measure DAX and the metadata, and the two are not interchangeable mid-model without rewriting every TI measure: +- Classic assumes Gregorian or shifted-Gregorian (a July-start fiscal year with Gregorian month boundaries), cannot do week math, and throws on any gap between first and last date. Calendar-based handles 4-4-5/4-5-4/5-4-4, 13-month, lunar, ISO-week, and tolerates sparse dates +- Calendar-based adds context-clearing semantics with no classic equivalent: `DATEADD` and `SAMEPERIODLASTYEAR` are "fixed" (lateral shift, keep all context); every other TI function is "flexible" (clears context on time-related columns). So the same `PARALLELPERIOD` returns different results depending only on whether a column was tagged time-related ; reason about it explicitly +- Calendar-based cannot be used with live-connected or composite models, and (mid-2026) calendars are authored only in Desktop or external tools/TMDL, not the service. Auto-date/time must not coexist; RLS on a calendar-defining fact can surprise + +`te` edits whatever the running engine supports via TOM ; set the classic marking (`DataCategory="Time"`, key column `IsKey`/`DateTime`) in one scripted pass. The `calendar` object is newer than most TOM surfaces expose; if `te`/TOM cannot create it, that is the case to drop to TMDL (`calendarColumnGroup` blocks under the table, an uncategorized group tagging a column as time-related). Validate by executing a TI function that exercises the calendar ; a missing-category error surfaces at query time, not save time. Do not mix reference styles in one calculation; calendar-based functions cannot be nested (rewrite `PREVIOUSDAY(PREVIOUSMONTH(...))` as `CALCULATETABLE(PREVIOUSDAY(...), PREVIOUSMONTH(...))`); `DATEADD` gains extension/truncation params under calendars that IntelliSense does not surface. + +Sources: learn.microsoft.com desktop-time-intelligence; learn.microsoft.com time-intelligence-functions-dax; repo connect-pbid calendar-column-groups + +## DataCategory="Time" is not "Mark as Date Table" (the surrogate-key trap) + +Two settings get conflated. `DataCategory="Time"` plus `IsKey=true` on the date column (what the in-repo `mark-as-date-table.csx` sets) is not the same as Desktop's "Mark as date table," which records a separate table-level marking that classic TI uses to resolve a bare `'Date'[Date]` reference. A table can carry `DataCategory="Time"` and still not be a marked date table. + +This bites when the date dimension joins the fact on an **integer surrogate key** (`yyyymmdd`), the norm for warehouse star schemas. The relationship column is then not the Date-typed column, so classic TI has no Date-typed relationship to traverse, and Microsoft is explicit: you must mark the date table here or TI silently misbehaves or returns blanks. Auto-detection does not save you. Other forced-mark triggers: any classic TI use, and Excel PivotTable date filters over the published model. + +Setting `DataCategory="Time"` and walking away looks correct in the model tree and passes `te validate`, but TI still returns wrong numbers (blank or unshifted, never an error) when the join is on the integer key. Verify which column carries the marking and the relationship column's type via `INFO.VIEW.RELATIONSHIPS()`. If the relationship is on `Date[DateKey]` (Int64) not `Date[Date]` (DateTime): either relate on `Date[Date]` and keep `DateKey` as a hidden degenerate, then mark; or switch the relevant measures to calendar-based TI, which reads the calendar's date-category column directly and does not depend on the relationship column type. The marked date key must be unique, blank-free, contiguous, span full years (a fiscal year counts), and Date/Time columns need an identical timestamp on every row. + +Sources: learn.microsoft.com desktop-date-tables (when-you-must-mark); learn.microsoft.com specify-mark-as-date-table; learn.microsoft.com model-date-tables; repo mark-as-date-table.csx + +## Week-based and 4-4-5 fiscal time intelligence (why classic functions return wrong answers) + +Retail/manufacturing calendars (4-4-5, 4-5-4, 5-4-4, ISO weeks) define a year as a whole number of weeks so quarters hold equal working days and compare directly; boundaries do not align with Gregorian months. Shifted-Gregorian fiscal years (July start, Gregorian months) are fine for classic TI; this is the genuinely non-Gregorian case. Classic functions do not error here, they return wrong numbers, which is worse. `SAMEPERIODLASTYEAR` assumes a given day maps to the same period across years; in a week calendar it does not (Jan 1-2 2011 belong to ISO week 52 of 2010), so YoY built on the date axis silently misaligns and drifts year over year. + +Two approaches in preference order: +1. **Calendar-based TI** (preferred where available): tag week categories on a `calendar` object and let `TOTALWTD`/`TOTALYTD([Sales], 'ISO-454')` operate on the real period columns. The engine needs enough categories to walk up to a year (e.g. {Week}, or {Week of Year, Year}, or {Week of Quarter, Quarter}); too few errors at query time +2. **Custom DAX with offset columns** (any engine, including pre-preview, composite, live-connected): the date table carries integer columns linearizing the fiscal structure (`Fiscal Day Of Year`, fiscal Year/Week/Quarter). YoY matches ordinal position, not calendar date, via `TREATAS(VALUES('Date'[Fiscal Day Of Year]), 'Date'[Fiscal Day Of Year])` under a prior-fiscal-year filter; fiscal-YTD is a running filter on the within-year offset (`'Date'[Fiscal Day Of Year] <= MAX(...)`), not `DATESYTD` + +Compute offset columns upstream (source or Power Query) per the push-row-work-upstream rule; Microsoft cautions they bloat models if overused, so add only the few you query. Add the measures with full metadata (DisplayFolder, FormatString, Description) in one `te script` pass, then `te validate` and execute before saving. A 53-week vs 52-week year leaves the last offsets of a long year with no prior-year counterpart (blank is correct); a primary period column must sort correctly across years (`yyyy-mm`, or Sort By Column); a wrong week-TI number looks plausible, so validate against a hand-computed period. + +Sources: SQLBI week-based-time-intelligence-in-dax; learn.microsoft.com desktop-time-intelligence (calendar-based preview); learn.microsoft.com model-date-tables; repo SpaceParts Date.tmdl diff --git a/plugins/semantic-models/skills/semantic-model/references/vertipaq-optimization.md b/plugins/semantic-models/skills/semantic-model/references/vertipaq-optimization.md new file mode 100644 index 00000000..ff005370 --- /dev/null +++ b/plugins/semantic-models/skills/semantic-model/references/vertipaq-optimization.md @@ -0,0 +1,29 @@ +# VertiPaq and model-size optimization + +Companion to the `semantic-model` skill (SKILL.md). Original guidance; each section cites its sources. + +**Working with `te`:** `te vertipaq --columns --detail --top 25` ranks columns and reads the dictionary / data / hierarchy byte split; scope to a table or column to confirm a fix, and `--relationships` / `--all --detail` for hierarchy and relationship line items. Export VPAX for before / after diffs; sort by percent of DB, not absolute size. + +## Reading VertiPaq Analyzer metrics (not just "fix the biggest column") + +VPA is the column-storage profile of an Import or Direct Lake model: per-column cardinality, the dictionary/data/hierarchy/relationship byte split, encoding type, share of the database. The byte split is itself the diagnosis. Two columns of identical total size demand different fixes: mostly **dictionary** bytes is a cardinality problem (the distinct-value store dominates; high-cardinality dictionaries can exceed 90% of column cost) ; attack via splitting, precision reduction, or higher fact grain. Mostly **data segment** bytes is a row-count/compressibility problem (RLE found few runs) ; attack with sort order, grain, or removal. A large **Columns Hierarchies Size** is the auto attribute hierarchy (per-column MDX/Excel structure), pure overhead on hidden high-cardinality columns (one measured case: 1.1 GB of hierarchy alone). User Hierarchies and Relationships are separate actionable line items. + +`te vertipaq --columns --detail --top 25` ranks columns and reads the split in one pass; `--detail` turns the ranking into a diagnosis. Scope to a table/column to confirm a fix; pull relationship/hierarchy line items with `--relationships`/`--all --detail` when total size is high but no single column stands out. Export/import VPAX for offline before/after diffing in CI. For live cardinality and row counts (a TMDL-only export gives structure but not measured cardinality), point at the XMLA endpoint with `--stats`. Pitfalls: profiling an unrefreshed export tells you nothing about size (numbers only become real against a refreshed live model); `te deps --unused` can flag a relationship key as "unused" though it is load-bearing for filter propagation (confirm with `te get` before removing); sort by per-column % of DB, not absolute size, or a 38%-of-DB DateTime column hides behind a physically larger but already-optimal measure. When te-cli can't reach the endpoint, the same numbers come from storage DMVs (`DISCOVER_STORAGE_TABLE_COLUMN_SEGMENTS`, `DISCOVER_STORAGE_TABLE_COLUMNS`) via connect-pbid, or `INFO.STORAGETABLECOLUMNS()`/`INFO.DICTIONARYSTORAGES()` in DAX. + +Sources: SQLBI data-model-size-with-vertipaq-analyzer; SQLBI optimizing-high-cardinality-columns; learn.microsoft.com info-dictionarystorages-function-dax; repo te-cli command-reference; repo connect-pbid vertipaq-stats + +## Value vs hash encoding and the encodingHint knob + +VertiPaq encodes each column one of two ways. **Value encoding** stores the value as a bit-packed integer offset from a base, no dictionary ; only for integer-family columns whose range is narrow enough to bit-pack. **Hash (dictionary) encoding** builds a dictionary of distinct values and stores per-row integer pointers ; for text, dates, decimals, and any numeric the engine decides is better hashed. The engine picks per column at refresh by sampling the first segment, and can re-encode mid-refresh if a later segment violates the choice (the re-encode pass is the cost). `encodingHint` (`default|hash|value`) biases the decision but is a hint, not a guarantee. + +Value encoding skips the dictionary, so a genuinely low-range integer measure is smaller and faster to scan. But two common beliefs are wrong: integer relationship/key columns are always hash-encoded regardless of type (the win from a narrow key is a smaller dictionary, not an encoding switch); forcing `value` on a wide-range or non-integer column does nothing or backfires (value encoding needs the range to bit-pack). The realistic use is narrow: `value` on a high-volume additive integer measure where the first-segment sample mis-picked hash, or `hash` to stop a re-encode pass on a column you know is high-range. Most columns stay `default`. Confirm the property name on a real object, set the hint, re-profile to confirm the engine honored it and size moved, revert if VPA still shows HASH. The hint takes effect only on the next full refresh of that column. This is an Import lever; Direct Lake column encoding is driven by the Delta/Parquet write and V-Order, not `encodingHint`. + +Sources: learn.microsoft.com import-modeling-data-reduction; learn.microsoft.com direct-lake-understand-storage; repo tmdl SKILL / object-properties; repo connect-pbid tom-object-types; repo te-cli semantic-modeling-practices + +## Splitting high-cardinality integer keys (the general div/modulo case) + +Beyond the DateTime-into-Date+Time split, the general technique applies to any near-unique integer column (transaction ID, order-line key, a surrogate with tens of millions of distinct values): split into columns whose distinct counts multiply back to the original, using integer division and modulo, reconstructing only when a query truly needs the full value. The dictionary cost grows with distinct values per column, so two columns of ~10,000 distinct each cost far less than one of 100,000,000. SQLBI's measured case: a 100M-distinct TransactionID was ~2,811 MB as one column, ~191 MB split in two, down to ~97 MB in four ; a 90%+ reduction. Choose the divisor `K` near sqrt(cardinality) so both sub-columns carry balanced distinct counts. Do it in Power Query (or upstream SQL), **never as a calculated column** ; a calc column over the original reclaims nothing because the original still lives in the model. + +te-cli edits metadata, not source rows, so the split happens in the partition M (`Number.IntegerDivide`/`Number.Mod`, then drop the original from the load). Use `te` to inspect the offending column's share before, set the new columns to int64, `isAvailableInMDX=false`, `isHidden=true`, refresh, and confirm the reduction with `te vertipaq`. Pitfalls: splitting **breaks single-column `DISTINCTCOUNT`** on the original (the replacement `COUNTROWS(SUMMARIZE(...))` is materially slower and raises memory) ; only split a column you do not distinct-count; it adds refresh cost (the multi-column encode parallelizes but adds overhead); a lopsided split (dividing by 10 on a 100M range) saves little ; aim near sqrt and test both sides' cardinality. If the value is only ever displayed (never filtered or joined), prefer dropping it entirely over splitting (one identity column cut a real model from 1.3 GB to 490 MB). This is Import-only; for Direct Lake reduce cardinality at the Delta write. + +Sources: SQLBI optimizing-high-cardinality-columns; learn.microsoft.com import-modeling-data-reduction; repo te-cli semantic-modeling-practices diff --git a/plugins/semantic-models/skills/review-semantic-model/scripts/get_model_info.py b/plugins/semantic-models/skills/semantic-model/scripts/get_model_info.py similarity index 100% rename from plugins/semantic-models/skills/review-semantic-model/scripts/get_model_info.py rename to plugins/semantic-models/skills/semantic-model/scripts/get_model_info.py diff --git a/plugins/semantic-models/skills/standardize-naming-conventions/SKILL.md b/plugins/semantic-models/skills/standardize-naming-conventions/SKILL.md index fe9105c3..795dc584 100644 --- a/plugins/semantic-models/skills/standardize-naming-conventions/SKILL.md +++ b/plugins/semantic-models/skills/standardize-naming-conventions/SKILL.md @@ -1,6 +1,6 @@ --- name: standardize-naming-conventions -version: 26.20 +version: 26.24 description: Interactive naming convention standardization for TMDL-based Power BI semantic models. Automatically invoke when the user asks to "standardize naming conventions", "fix naming conventions", "clean up model names", "apply naming standards", "audit naming", "make names human readable", "rename fields", "fix abbreviations in model", or mentions renaming measures, columns, or tables for consistency across a model. --- diff --git a/plugins/tabular-editor/.claude-plugin/plugin.json b/plugins/tabular-editor/.claude-plugin/plugin.json index c0d9da51..7fbe096a 100644 --- a/plugins/tabular-editor/.claude-plugin/plugin.json +++ b/plugins/tabular-editor/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "tabular-editor", - "version": "26.20", + "version": "26.24", "description": "Agentic development tools for Power BI semantic models with Tabular Editor.", "author": { "name": "Kurt Buhler", diff --git a/plugins/tabular-editor/skills/bpa-rules/SKILL.md b/plugins/tabular-editor/skills/bpa-rules/SKILL.md index 092cd9ac..79d23ead 100644 --- a/plugins/tabular-editor/skills/bpa-rules/SKILL.md +++ b/plugins/tabular-editor/skills/bpa-rules/SKILL.md @@ -1,6 +1,6 @@ --- name: bpa-rules -version: 26.20 +version: 26.24 description: Interactive BPA rule generation for Power BI semantic models; guided discovery, model investigation, and expert rule authoring. Automatically invoke when the user mentions "BPA rule", "Best Practice Analyzer", or asks to "create a BPA rule", "audit BPA rules", "recommend BPA rules", "set up BPA for my team", "check model for best practices", "validate BPA rules", "improve a BPA expression". --- diff --git a/plugins/tabular-editor/skills/c-sharp-scripting/SKILL.md b/plugins/tabular-editor/skills/c-sharp-scripting/SKILL.md index 877ab7db..ca833f87 100644 --- a/plugins/tabular-editor/skills/c-sharp-scripting/SKILL.md +++ b/plugins/tabular-editor/skills/c-sharp-scripting/SKILL.md @@ -1,6 +1,6 @@ --- name: c-sharp-scripting -version: 26.20 +version: 26.24 description: Writing and executing C# scripts and macros against Power BI semantic models using Tabular Editor 2/3. Automatically invoke when the user mentions "C# script", "Tabular Editor script", "TOM scripting", "MacroActions.json", "XMLA", or asks to "automate model changes", "bulk update measures", "create calculation groups", "write a macro", "format DAX expressions", "manage model metadata". --- diff --git a/plugins/tabular-editor/skills/te-cli/SKILL.md b/plugins/tabular-editor/skills/te-cli/SKILL.md new file mode 100644 index 00000000..5a6994d9 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/SKILL.md @@ -0,0 +1,184 @@ +--- +name: te-cli +version: 26.24 +description: Expert guidance for the cross-platform Tabular Editor CLI (the `te` binary, currently in preview) that manages Power BI / Analysis Services semantic models from the terminal on macOS, Linux, and Windows. Use when the user mentions the `te` CLI or "Tabular Editor CLI" (not the "2"), or runs a `te <command>` to scaffold, inspect, edit, validate, run BPA on, query, deploy, refresh, test, or migrate a semantic model. Not for the legacy Windows-only `TabularEditor.exe` (TE2). +--- + +# Tabular Editor CLI (`te`) + +To get the `te` CLI yourself (as the agent), see [references/get-te-cli.md](references/get-te-cli.md). + +The `te` CLI is a single self-contained binary that loads, edits, validates, deploys, refreshes, and tests semantic models against TMDL/BIM files, Power BI Desktop, and cloud workspaces (Power BI, Fabric, Azure AS, SSAS). It is built on the same TOMWrapper that powers Tabular Editor 3, so model edits behave like the desktop app. + +**Always pass `--output-format json`** when driving `te` programmatically. The default text/table output uses tables and ANSI styling that mangle in agent transcripts; JSON is parseable and avoids rendering issues. + +**Limited public preview.** Preview builds stop functioning after 2026-09-30. No license is required during preview. Issues and feedback: https://github.com/TabularEditor/CLI + +**Not the TE2 CLI.** This is a different product from the legacy Windows-only `TabularEditor.exe` (TE2). If the user invokes TE2 flag syntax (`-D`, `-S`, `-A`, `-B`, `-TMDL`, `-O`, `-C`, `-V`, `-G`), route it through the compat layer or invoke `TabularEditor.exe` directly. See `references/te2-migration.md`. + +## When to use this skill + +- The user mentions "te CLI", "the new Tabular Editor CLI", or runs a `te <command>` in a terminal +- The user wants to scaffold, inspect, edit, validate, deploy, refresh, query, or test a semantic model from the terminal on any OS +- The user wants to convert TMDL, BIM, or PBIP, run BPA, or format DAX from the command line +- The user is migrating CI/CD pipelines from `TabularEditor.exe` (TE2) to `te` + +## When NOT to use this skill + +- The user explicitly wants to run `TabularEditor.exe` natively (TE2); use that product directly +- The user asks about Tabular Editor 3 desktop UI features (Preferences.json, MacroActions.json, Layouts.json); consult https://docs.tabulareditor.com/ +- The user wants help authoring a C# script body or a BPA rule expression itself rather than running it; use the `c-sharp-scripting` and `bpa-rules` skills + +## Critical general rules + +- First use in a session: run `te --version` and `te auth status`. If not authenticated, ask the user to run `te auth login`. +- Run `te --help` and `te <command> --help` the first time composing a command; flags are still evolving during preview. +- `te connect` state is per-shell-session and does NOT survive across separate Bash tool calls (each call is a fresh shell). Pass `-m <model>` (and `-s`/`-d` for remote) on every command, or set `TE_SESSION=<name>` before the first call to share state. +- MPartition path asymmetry: `te add` for an M partition uses `<Table>/<Partition>`, but every other command (`te rm`, `te get`, `te ls`, `te mv`, `te set`) uses `<Table>/Partitions/<Partition>`. +- Mutations stage in memory by default. `te set`, `te add`, `te rm`, `te mv`, `te replace`, `te format`, `te script`, `te macro run`, `te incremental-refresh set/remove` need `--save` to persist (unless `interactiveEditMode` is set to `save`). +- The BPA gate is ON by default for `te deploy` and `te save`. Bypass deliberately: `--skip-bpa`, `--fix-bpa`, or `bpa.onDeploy` / `bpa.onSave` config (keys are nested under `bpa.`, not flat). +- In CI: pass `--non-interactive` and `--force`. `te deploy` prompts with `n` as the safe default and hangs pipelines without `--force`. +- Never put secrets on the command line (visible in `ps` and shell history). Use `--auth env` with `AZURE_CLIENT_ID`/`AZURE_CLIENT_SECRET`/`AZURE_TENANT_ID`, stdin (`-`), or `--auth managed-identity`. +- Avoid destructive operations without explicit direction: `te rm`, `te mv`, `te deploy --create-only`, `te save --force`, `te connect --clear`. If a command is blocked by permissions, stop and ask. + +## Staging model (`--save` / `--stage` / `--revert`) + +Every mutating command runs through a staging dispatcher: edits (`set`, `add`, `rm`, `mv`, `replace`), DAX/M (`format`), TOM (`script`, `macro run`), refresh policy, and BPA `--fix`. + +By default edits stage in memory and are discarded on exit. Pass `--save` to persist. The default is configurable with `te config set interactiveEditMode <mode>`: + +- `stage` (default): keep changes in memory; persist with explicit `--save` +- `save`: auto-persist after each successful mutation +- `revert`: auto-roll-back after each mutation (safe audit/dry-run style) + +Inside `te interactive`, `--save`, `--stage`, and `--revert` are available per command and mutually exclusive. `--save-to <path>` writes the mutation to a different location without overwriting the source. `--force` on `te script` / `te save` lets a mutation persist even when it introduces NEW DAX validation errors; the default save gate refuses to persist if the mutation introduces new errors (pre-existing errors do not block). + +## Quickstart + +```bash +te --version && te auth status # 0. check install + auth +te auth login # 1. authenticate (browser); cached +te init ./my-model # 2. scaffold (PowerBI mode, TMDL, compat 1702) +te load ./model # 3. load + summary; then `te ls`, `te ls Sales` +te find "Revenue" --in names -m ./model # 4. search (names | expressions | descriptions | all) +te get Sales/Revenue -q expression -m ./model # 5. read a measure's DAX +te bpa run --fail-on error --ci github -m ./model # 6. BPA gate +te format --save -m ./model # 7. format all DAX +te query -q "EVALUATE TOPN(5, 'Sales')" -s ws -d model # 8. query +te save -o ./out --serialization tmdl -m ./model # 9. save / convert (tmdl|bim|pbip|te-folder) +te deploy ./model -s ws -d model --force --ci github # 10. deploy +te refresh --type full -s ws -d model # 11. refresh +``` + +`te connect <ws> <model>` sets an active connection for interactive terminals, but it does not persist across separate Bash tool calls. In agentic or scripted use, pass `-m`/`-s`/`-d` explicitly every command (or set `TE_SESSION`). + +## Common operations + +The highest-frequency tasks in their most concise form. Full flags are in `references/command-reference.md`; flags are still moving in preview, so confirm with `te <command> --help`. + +1. **Summarize a model (most concise)**: `te load ./model` prints a model summary. For a structural inventory, `te ls` (tables), `te ls Measures` (every measure across the model). Relationships do not list via `te ls` (a known gap, see `references/gotchas.md`); enumerate them with `te query -q "EVALUATE INFO.VIEW.RELATIONSHIPS()"`. Add `--output-format json` for a machine-readable dump. +2. **Search the model (fastest)**: `te find "<text>" --in names --paths-only -m ./model`. Scope `--in` to `names`, `expressions`, `descriptions`, `displayFolders`, ...; `--in expressions` walks every DAX and M expression. `--paths-only` is the fast, pipeable form. Structural lookups use wildcards (`te ls "Sales/*Amount"`). Relationships are not `te ls`-enumerable (known gap); list them with `te query -q "EVALUATE INFO.VIEW.RELATIONSHIPS()"`. +3. **Query the model**: + - Inline DAX: `te query -q "EVALUATE TOPN(10, Sales)" -m ./model` + - From a `.dax` file: `te query -f query.dax -m ./model` + - Save results (format picked by extension): `--output-file out.csv` (csv/tsv/json/dax); machine-readable stdout: `--output-format json`. +4. **Make a change** (stages in memory; `--save` persists): `te set Sales/Revenue -q expression -i "SUM(Sales[Amount])" --save`. Also `te add`, `te rm`, `te mv`. Read the current value first with `te get Sales/Revenue -q expression`. +5. **Make bulk changes**: + - Text find/replace across the whole model: `te replace "Old" "New" --in expressions --save` (previews unless `--save`). + - Arbitrary bulk logic in one pass (the model loads once, avoiding ~1-2s per-call startup): `te script -S bulk.csx --save`, or inline `echo '<C# foreach over Model.AllMeasures>' | te script -e - --save`. Predefined macros: `te macro run "<name>" --on "Sales/A,Sales/B" --save`. +6. **Validate and optimize**: + - Validate DAX, schema, and relationships: `te validate -m ./model --errors-only`. + - Best-practice gate: `te bpa run --fail-on warning -m ./model` (`--fix` auto-applies fixes); format DAX with `te format --save -m ./model`. + - Size and storage: `te vertipaq --columns --detail --top 20 -m ./model` surfaces the largest columns first; `references/semantic-modeling-practices.md` covers what to do about them. + +## Global options + +Abbreviated; the full table (including `--recent`, server and database detail) is in `references/command-reference.md`. + +| Option | Description | +|---|---| +| `-m, --model <path>` | TMDL folder, `.bim`, or TE folder | +| `-s, --server` / `-d, --database` | Workspace/endpoint and semantic model name | +| `--local` | Running Power BI Desktop (Windows only) | +| `--auth <method>` | `auto` \| `interactive` \| `spn` \| `env` \| `managed-identity` | +| `--output-format <fmt>` | `auto` \| `text` \| `json` \| `csv` \| `tmsl` (alias `bim`) \| `tmdl`; how STDOUT renders | +| `--non-interactive` | Disable prompts; fail if input missing (set in CI) | +| `--debug` | Debug logs to stderr | + +**Note:** `--output-format` (how stdout renders) and `--serialization` (how a model is written to disk on `init`/`save`) are different flags. Do not conflate them. + +## Semantic modeling checklist + +Driving the CLI correctly is not the same as building a good model. After `te add` creates an object, apply the modeling decision that makes it correct and usable. The highest-value practices, each with its `te` command: + +| Practice | Why | `te` command | +|---|---|---| +| `summarizeBy` = `none` on key/ID columns | stops Power BI silently summing keys into meaningless totals | `te set Sales/ProductKey -q summarizeBy -i none --save` | +| Hide foreign-key and surrogate-key columns | keys serve relationships, not visuals; keeps the field list clean | `te set Sales/ProductKey -q isHidden -i true --save` | +| Mark the date table | unlocks reliable time intelligence | `te set Date -q dataCategory -i Time --save` | +| Single cross-filter direction by default | avoids ambiguous filter paths and double counting | list with `te query -q "EVALUATE INFO.VIEW.RELATIONSHIPS()"` (`te ls` cannot enumerate relationships; see gotchas), read one with `te get Relationships/<name>` (the `->` shorthand is for `te add` only); enable bidirectional only for a deliberate bridge | +| Format string on every measure | unformatted measures render raw floats | `te set "_Measures/Revenue" -q formatString -i "#,0.00" --save` | +| Display folder + description on measures | a flat field pane is unusable past a few dozen measures; descriptions feed tooltips and Copilot | `te set "_Measures/Revenue" -q displayFolder -i "Revenue" --save` | +| Minimal correct data types; integer surrogate keys | high-cardinality and oversized types bloat VertiPaq | `te set Sales/CustomerKey -q dataType -i int64 --save` | +| Prefer measures over calculated columns | calculated columns cost storage and break some DirectQuery/DirectLake paths | `te add "_Measures/Margin" -t Measure -i "[Revenue]-[COGS]" --save` | +| Calculation groups over measure sprawl | turns N measures x K variants into N + K objects | see `references/semantic-modeling-practices.md` | +| Gate every batch with validate + BPA | catches broken references and antipatterns while the change is fresh | `te validate -m ./model && te bpa run --fail-on warning -m ./model` | + +Full rationale, citations, and worked workflows (RLS roles, calculation groups, date tables, VertiPaq tuning): `references/semantic-modeling-practices.md`. + +## Command index + +Ten command families. Full flags and examples in `references/command-reference.md`. + +- Model I/O: `te load`, `te save`, `te open`, `te init` +- Editing: `te set`, `te add`, `te rm`, `te mv`, `te replace` +- Inspection: `te ls`, `te get`, `te find`, `te diff`, `te deps` +- Analysis & quality: `te validate`, `te bpa run`, `te vertipaq`, `te format` +- Execution: `te query`, `te script`, `te macro` +- Deploy & refresh: `te deploy`, `te refresh`, `te incremental-refresh` +- Testing: `te test` +- Connection & auth: `te connect`, `te auth`, `te profile`, `te session` +- Configuration: `te config`, `te migrate`, `te completion` +- Shell: `te interactive` (model-aware REPL; subcommands work without the `te` prefix) + +## The authoring loop + +Run quality gates continuously, not only at deploy: + +```bash +te validate -m ./model --errors-only # after each batch of edits +te bpa run --fail-on warning -m ./model # antipattern gate during development +te format --save -m ./model # consistent DAX layout before commit +``` + +For build scripts that issue many `te` calls, set `te config set bpa.onSave false` first (skip the per-save BPA pass), run BPA once at the end, and set `te config set spinner false` for cleaner logs. Each invocation has ~1-2s of startup; prefer one `te script` with a C# loop over N `te set` calls for bulk edits. + +## Using te with other Power BI CLIs + +`te` owns the semantic model. Two sibling CLIs own the layers around it, and the highest-value workflows cross the boundary: + +- `pbir` (the Power BI report layer): renaming or moving a model object leaves the report bound to the old `Table.Field`. Rename in the model (`te mv`, then `te replace --in expressions --save`), then repair the report bindings (`pbir fields replace`, `pbir validate --fields`). See `references/pbir-cli-tandem.md`. +- `fab` (the Fabric / Power BI service): export a model from a workspace, edit and gate it locally with `te`, then deploy over XMLA (`te deploy`) or import it back (`fab import`). See `references/fabric-cli-tandem.md`. + +Gate any cross-tool refactor with `te validate` before touching the report or the service, and remember every `te` mutation stages in memory until `--save`. + +## References + +Bundled (load as needed): + +- `references/command-reference.md` - object path grammar, global options, all 10 command families, authentication, connections/profiles/sessions +- `references/semantic-modeling-practices.md` - modeling best practices tied to `te` commands, with sources +- `references/workflows.md` - multi-step recipes (table + M partition, format conversions, deploy, refresh, perspectives, translations, incremental refresh, field parameters) +- `references/gotchas.md` - path/property asymmetries, output shapes, behavior traps +- `references/config-cicd-env.md` - config keys, speed knobs, CI/CD (GitHub Actions, Azure DevOps), output formats, exit codes, environment variables +- `references/te2-migration.md` - TE2 compat activation and full flag mapping +- `references/pbir-cli-tandem.md` - using `te` with the `pbir` CLI (rename and refactor propagation, thin reports, validation pairing) +- `references/fabric-cli-tandem.md` - using `te` with the `fab` CLI (export/edit/deploy round-trip, discovery, refresh, promotion) + +Authoritative docs: + +- Command reference: https://docs.tabulareditor.com/en/features/te-cli/te-cli-commands.html +- Overview: https://docs.tabulareditor.com/en/features/te-cli/te-cli.html +- CI/CD: https://docs.tabulareditor.com/en/features/te-cli/te-cli-cicd.html +- Known limitations: https://docs.tabulareditor.com/en/features/te-cli/te-cli-limitations.html +- GitHub (issues, releases): https://github.com/TabularEditor/CLI diff --git a/plugins/tabular-editor/skills/te-cli/references/command-reference.md b/plugins/tabular-editor/skills/te-cli/references/command-reference.md new file mode 100644 index 00000000..d3d603fb --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/references/command-reference.md @@ -0,0 +1,394 @@ +# `te` command reference + +Full command surface for the `te` CLI. Companion to the te-cli skill (SKILL.md). Configuration keys, CI/CD, output formats, and environment variables live in config-cicd-env.md. + +## Installation + +Download from https://tabulareditor.com (signed in with a TE account). Single self-contained binary; no .NET / runtime install needed. + +| Platform | Archive | Install location (suggested) | +|---|---|---| +| Windows x64 / ARM64 | `te-win-x64.zip` / `te-win-arm64.zip` | `%LOCALAPPDATA%\Programs\te` | +| macOS Intel / Apple Silicon | `te-osx-x64.zip` / `te-osx-arm64.zip` | `~/.local/bin` | +| Linux x64 / ARM64 | `te-linux-x64.zip` / `te-linux-arm64.zip` | `~/.local/bin` | + +Add the install dir to `PATH`. On macOS, allow first-run network access for Gatekeeper notarization check. Update by overwriting the binary; config and credentials persist. + +**Shell completion**: +```bash +te completion bash > /etc/bash_completion.d/te +te completion zsh > "${fpath[1]}/_te" +te completion pwsh | Out-String | Invoke-Expression +``` + +**Cross-platform limits**: local SSAS connections (TCP) and Power BI Desktop connections (named pipe) are Windows-only. All cloud workflows work on every platform. + +## Authentication + +Backed by Azure Identity's full credential chain. + +| Method | Flag | When to use | +|---|---|---| +| Interactive browser | `--auth interactive` (default) | Local dev | +| Service principal (secret) | `--auth spn -u <appId> -p <secret> -t <tenant>` | Avoid; secret on cmd line | +| Service principal (cert) | `--auth spn -u <appId> -t <tenant> --certificate <path>` | Cert-based CI | +| Environment vars | `--auth env` (reads `AZURE_CLIENT_ID/SECRET/TENANT_ID`) | **Preferred for CI** | +| Managed identity | `--auth managed-identity` | Azure-hosted runners | + +```bash +te auth login # browser +te auth login --identity # managed identity +te auth status # exit 0 if authenticated, 1 otherwise +te auth logout # clear cached credentials +``` + +**Credential cache locations** (all file-mode `0600` / DPAPI on Windows): +- Windows: `%USERPROFILE%\.te-cli\` (DPAPI-encrypted) +- Linux: `~/.te-cli/` (libsecret via Azure.Identity) +- macOS: `~/.te-cli/token-cache.bin` + +## Connections and profiles + +`te connect` sets a per-terminal active connection so subsequent commands don't need `-s/-d/-m` repeated. + +```bash +te connect # show active connection (or open picker in interactive) +te connect MyWorkspace MyModel # remote workspace +te connect ./my-model # local TMDL/BIM +te connect --local # running Power BI Desktop (Windows) +te connect --clear # reset +``` + +**Workspace mirroring** (bidirectional sync between local TMDL folder ↔ remote workspace): + +```bash +te connect Finance "Revenue Model" -w ./revenue-model # remote primary, mirror to local +te connect ./revenue-model -w Finance "Revenue Model" # local primary, mirror to remote +# --workspace-format <bim|tmdl|te-folder> # on-disk format for the mirror +# --workspace-auth <method> # auth for the remote side when primary is local +``` + +**Profiles** save named connection + behavior overrides: +```bash +te profile set prod -s MyWorkspace -d MyModel --auto-format true +te profile set dev -s DevWorkspace -d MyModel --bpa-on-deploy false +te profile list +te profile show prod +te profile remove old +te connect --profile prod +``` + +## Object path syntax + +Backed by a formal grammar (`PathParser`); paths come in two flavors with subtly different rules: + +**Object paths**; used by `te get`, `te set`, `te add`, `te rm`, `te mv`. Resolve to **one** object. Wildcards rejected. + +**Filter paths**; used by `te ls`, `te find`, `te deps`, `te bpa run --path`. Resolve to a **set** of objects. Wildcards allowed. + +### Slash-form (works on both) + +- `Sales`; table +- `Sales/Revenue`; measure or column on the Sales table +- `Sales/Measures`, `Sales/Columns`, `Sales/Partitions`, `Sales/Hierarchies`; sub-containers +- `Sales/Geography/Levels`; hierarchy levels +- `Measures/<name>/KPI`; KPI sub-object on a measure (resolves through the KPI wrapper) +- `Roles/<role>/Members`, `Roles/<role>/TablePermissions`; role children +- `Perspectives/<persp>/<table>`; perspective membership (use `te add Perspectives/Default/Sales` to add a table) +- `Tables`, `Measures`, `Roles`, `Perspectives`, `Cultures`, `Hierarchies`, `Annotations`; model-level containers (pivot via `te ls Measures` for cross-table view) +- `Relationships` is **not** enumerable via `te ls`, despite `relationship` appearing in `te ls --type`'s help. The keyword falls through to a literal path match and errors with `No objects match path 'Relationships'`, even when relationships exist (recognized-but-empty containers say `No objects match 'X'` without the word `path`). List relationships with DAX `EVALUATE INFO.VIEW.RELATIONSHIPS()` (or `INFO.RELATIONSHIPS()` on older compat), or `te save` to TMDL and read `relationships.tmdl`. A single relationship is still addressable once you know its name: `te get Relationships/<name>`. + +Container-keyword table names (a table called `Tables`, `Roles`, etc.) resolve correctly via the path parser; the parser disambiguates by position. + +### DAX-form (object paths) + +DAX-style quoting and bracket-suffix follow DAX conventions; doubled quote char escapes itself (`'Bob''s'` = `Bob's`, `[foo]]bar]` = `foo]bar`): + +- `'Sales'[Amount]`; same as `Sales/Amount` +- `"Net Sales"[Sales Amount]`; same as `"Net Sales"/"Sales Amount"`, double-quoted form +- `[Total Sales]`; **model-wide measure-or-column lookup** (no table prefix; resolver searches every table) +- `"Sales[ProdKey]->Product[ProdKey]"`; relationship shorthand (used by `te add` only) +- `"Sales 2024"/Revenue`, `"_Measures/Total Revenue"`; quote any segment that contains a space, `/`, `[`, or `]` + +### Wildcards (filter paths only) + +Single `*` matches any run of characters within one segment (case-insensitive). Multi-segment globs and `?` are not supported. + +```bash +te ls Sa* # tables starting with "Sa" +te ls Sales/*Amount # any child of Sales ending in "Amount" +te ls */Amount # an "Amount" column/measure across every table +te ls Roles/Re*/Members # members of every role matching Re* +te bpa run --path "Sales/*" # run BPA only on objects under Sales +``` + +Passing a wildcard to an object-path command (`te get Sa*`, `te set Sa*`) fails fast with a parser error; wildcards on those would resolve to many objects, and the command needs exactly one. + +## Global options + +Work with every command: + +| Option | Description | +|---|---| +| `-m, --model <path>` | TMDL folder, `.bim`, or TE folder | +| `-s, --server <endpoint>` | Workspace name, `powerbi://...`, `asazure://...`, `localhost:PORT` | +| `-d, --database <name>` | Semantic model name on workspace | +| `--local` | Connect to running Power BI Desktop (Windows only) | +| `--auth <method>` | `auto` \| `interactive` \| `spn` \| `env` \| `managed-identity` | +| `--output-format <fmt>` | `auto` \| `text` \| `json` \| `csv` \| `tmsl` (alias `bim`) \| `tmdl` (default `auto`: text on TTY, JSON when piped). Controls how stdout is rendered; distinct from `--serialization` which picks the on-disk model format | +| `--recent [N]` | Use recently-used model (no value = picker, `N` = Nth most recent) | +| `--non-interactive` | Disable prompts; fail if input missing; **set in CI** | +| `--debug` | Debug logs to stderr | + +**Note:** `--output-format` (how stdout is rendered) and `--serialization` (how models are written to disk on `init`/`save`/etc.) are **two different flags**. Don't conflate them; passing one when the other was meant gives a confusing error or silent wrong output. + +## Command reference (10 families) + +### Model I/O + +| Command | Purpose | Key flags | +|---|---|---| +| `te load <path>` | Load model and show summary | global `-m/-s/-d` | +| `te save` | Save / convert / persist edits | `-o, --output-path <path>`, `--serialization tmdl\|bim\|te-folder\|pbip\|database.json`, `--force`, `--skip-bpa`, `--fix-bpa`, `--bpa-rules <file>` (repeatable, overrides config), `--skip-validation`, `--supporting-files` | +| `te open <path>` | Open in TE3 Desktop (TE3 must be installed) | n/a | +| `te init [path]` | Create new empty model. Path is optional; falls back to global `--model` when omitted | `--compatibility-mode PowerBI\|AnalysisServices` (default `PowerBI`), `--compatibility-level <int>` (alias `--compat`; defaults to 1702 for PowerBI, 1500 for AnalysisServices), `--name <model-name>`, `--serialization tmdl\|bim\|te-folder\|pbip` (default `tmdl`), `--force` | + +```bash +te load ./model # local TMDL folder +te load model.bim # local BIM file +te load -s MyWorkspace -d MyModel # remote + +te save # write back to source +te save ./model.bim -o ./tmdl-out # convert BIM → TMDL +te save -o ./project --serialization pbip --supporting-files +te save -o ./out -s ws -d model --skip-validation # fast passthrough + +te init ./my-model # PowerBI mode, TMDL, compat 1702 (default) +te init ./my-model --compatibility-mode AnalysisServices # AS mode, compat 1500 +te init ./my-model --compatibility-level 1604 # specific compat level +te init ./my-model --serialization bim # single-file .bim model +te init ./my-model --serialization pbip # full Power BI project structure +te --model ./new.bim init # path via global --model +``` + +### Model Editing + +| Command | Purpose | Key flags | +|---|---|---| +| `te set <obj>` | Set property | `-q <prop>` (e.g. `expression`, `formatString`, `description`, `isHidden`), `-i <value>` (or `-` for stdin), `--save`, `--save-to <path>` | +| `te add <obj>` | Add object | `-t <type>` (`Table`, `Measure`, `Column`, `CalculatedColumn`, `CalculatedTable`, `Hierarchy`, `Role`, `Perspective`, `Culture`, `CalculationGroup`, `CalculationItem`, `MPartition`, `Partition`, `EntityPartition`, `PolicyRangePartition`, `KPI`, `NamedExpression`, ...), `-i <value>`, `--if-not-exists` (idempotent), `--save`. Data-bound tables: `--mode import\|directquery\|directlake`, `--source sql\|lakehouse\|warehouse`, `--endpoint`, `--source-table`, `--source-database`, `--columns "Col1:Type,Col2:Type,..."`, `--partition-expression "<M>"`, `--source-type m\|query\|calculated` | +| `te rm <obj>` | Remove object | `--force`, `--if-exists`, `--dry-run`, `--save` | +| `te mv <src> <dst>` | Move/rename | `--save` | +| `te replace <find> <repl>` | Find+replace text | `--in names\|expressions\|descriptions\|displayFolders\|formatStrings\|annotations\|all`, `--regex`, `--case-sensitive`, `--save` (dry-run by default) | + +```bash +te set Sales/Amount -q expression -i "SUM(Sales[Amt])" --save +te set Sales -q isHidden -i true --save +te add Sales/Revenue -t Measure -i "SUM(Sales[Amount])" --save +te add Sales -t Table --save # empty M partition (PowerBI default) +te add "Sales[ProdKey]->Product[ProdKey]" --save # relationship shorthand +te add Sales/MarketingFlag -t CalculatedColumn -i "..." --if-not-exists --save +te rm Sales/OldMeasure --if-exists --save +te rm Sales/Revenue --dry-run # preview impact +te mv Sales/Revenue Finance/Revenue --save # cross-table move +te replace "OldTable" "NewTable" --in expressions --save +te replace "SUM" "SUMX" --regex --in expressions --save +``` + +#### Common `-q` properties + +Property names are case-insensitive and match TOM. When in doubt, run `te get <obj>` to see what's already on the object, or check `te set <obj> -q` for the settable list. The most-used ones: + +| Object | Common properties | +|---|---| +| **Measure** | `expression` (DAX), `formatString`, `displayFolder`, `description`, `isHidden` | +| **DataColumn** | `dataType` (`int64`/`string`/`double`/`decimal`/`dateTime`/`boolean`), `sourceColumn`, `summarizeBy` (`none`/`sum`/`count`/`average`/`max`/`min`/`distinctCount`/`automatic`), `isKey`, `isHidden`, `formatString`, `sortByColumn`, `dataCategory`, `displayFolder`, `description` | +| **CalculatedColumn** | `expression` (DAX) plus most DataColumn properties | +| **MPartition** | **`MExpression`** (NOT `expression`; that's what `te get` displays, but `te set` rejects it), `Mode` (`Import`/`DirectQuery`/`DirectLake`/`Default`), `description` | +| **QueryPartition** | `QueryDefinition` (alias `Query`), `Mode`, `description` | +| **Table** | `isHidden`, `dataCategory` (use `Time` to mark a date table), `description`, `name` (rename) | +| **Hierarchy** | `displayFolder`, `description`, `isHidden`; Levels take `column` (the source column name) | +| **ModelRole** | `modelPermission` (`None`/`Read`/`ReadRefresh`/`Refresh`/`Administrator`); TablePermissions take `filterExpression` (DAX) | +| **KPI** (on a Measure path `Measures/<name>/KPI`) | `statusExpression`, `trendExpression`, `targetExpression`, `statusGraphic`, `trendGraphic` | +| **CalculationItem** | `expression` (DAX), `ordinal` (int), `formatStringDefinition` | +| **Annotations / Translations** | `Annotations[<key>]`, `TranslatedNames[<culture>]`, `TranslatedDescriptions[<culture>]`; bracket-indexed property names | + +Properties not in the list are still usable; these are the most error-prone and frequently needed ones. **`te get <obj>` is always the authoritative discovery tool** for what an existing object exposes. + +### Inspection + +| Command | Purpose | Key flags | +|---|---|---| +| `te ls [filter-path]` | List objects, FS-style (filter-path: wildcards allowed) | `--type <type>`, `--paths-only`, `--no-multiline` (collapse multi-line cells; text output only) | +| `te get <obj>` | Get properties (object-path: no wildcards) | `-q <prop>` (single property), `--output-format tmdl\|tmsl\|bim` (emit object as TMDL/TMSL) | +| `te find <text>` | Search across model | `--in names\|expressions\|descriptions\|displayFolders\|formatStrings\|annotations\|all`, `--regex`, `--case-sensitive`, `--paths-only`, `--no-multiline`. **`--in expressions` walks every `IExpressionObject`**; measure DAX, calculated columns, KPI status/trend/target expressions, measure detail-rows, partition M, table-permission filters, calculation-group selection expressions | +| `te diff <m1> <m2>` | Structural diff | exit 0 identical, 2 models differ, 1 error | +| `te deps [obj]` | Dependency analysis | `--unused` (no DAX refs, not in relationships/hierarchies/sort-by/variations/time roles), `--hidden` (narrow to hidden), `--deep`, `--upstream`, `--downstream`, `--max-depth <N>` | + +```bash +te ls # tables +te ls Sales # columns + measures in Sales +te ls Sales/Measures # measures only +te ls Measures # all measures across model +te ls --type measure --paths-only # pipeable +te get Sales/Revenue -q expression +te get Model -q description +te find "CALCULATE" --in expressions # covers DAX, calc-columns, KPI exprs, partition M, role filters, calc-group selection +te find "Revenue" --in names +te find "TODO" --in descriptions --no-multiline # single-line cells, easy to grep +te find 123 --in expressions --paths-only # pipeable, e.g. for finding a KPI TargetExpression value +te diff ./model-v1 ./model-v2 +te deps "Sales/Revenue" # upstream + downstream +te deps --unused # unused everywhere +te deps --unused --hidden # hidden + unused +``` + +### Analysis & Quality + +| Command | Purpose | Key flags | +|---|---|---| +| `te validate` | Expressions + schema + TOM errors | `--ci <fmt>` (see below), `--trx <file>`, `--no-multiline`, `--no-warnings`, `--no-antipatterns`, `--errors-only` | +| `te bpa run [model]` | Run BPA (optional positional model path) | `-r/--rules <file-or-url>` (repeatable; URLs supported), `--fix`, `--save`, `--save-to <path>`, `--serialization`, `--fail-on error\|warning`, `--ci`, `--trx`, `--no-defaults`, `--no-model-rules`, `--rule <id>` (repeatable), `--path <filter>` (wildcards OK: `--path "Sales/*"`), `--vpax <file>`, `--vpa-rules`, `--allow-external-rules` (allow URL rules from model annotations), `--no-multiline` | +| `te bpa rules list` | Inspect active rules | `--all` (incl. disabled+ignored), `--ignored`, `--no-multiline` | +| `te vertipaq [path]` | VertiPaq stats (optional positional object path, e.g. `Sales` or `Sales/Amount`) | `--columns`, `--relationships`, `--partitions`, `--all`, `--detail` (encoding/segments breakdown), `--fields <csv>` (custom column set), `--export <vpax>`, `--import <vpax>` (offline), `--obfuscate` (writes `.vpax.dict` sidecar), `--top <N>`, `--stats` (DAX-queried details), `--annotate`, `--save` | +| `te format` | Format DAX or M | `-e <text>` (inline), `-p <obj>` (single), `--lang dax\|m`, `--semicolons` (Euro), `--long` (more line breaks; default is short), `--no-space-after-function`, `-t/--type <kind>` (disambiguate `-p` when path matches multiple), `--save`, `--save-to <path>` | + +```bash +te validate ./model --ci github --trx results.trx +te validate ./model --errors-only # hide warnings + anti-patterns +te bpa run --fail-on error --ci github +te bpa run --fix --save +te bpa run --rule PERF_UNUSED_HIDDEN_COLUMN +te bpa rules list --all +te vertipaq --all --export stats.vpax +te vertipaq Sales # filter to one table +te vertipaq Sales/Amount # filter to one column +te vertipaq --columns --detail # encoding/segment breakdown +te vertipaq --fields name,card,size,%tbl,%db,bar # custom column set +te vertipaq --import stats.vpax # offline analysis from VPAX +te format --save # all DAX +te format -p Sales/Amount --save # single measure +te format --lang m --save # all M +te format -e "SUM ( Sales[Amount] )" # inline preview +``` + +### Execution + +| Command | Purpose | Key flags | +|---|---|---| +| `te query` | DAX query | `-q <dax>` or `-f <file.dax>`, `--limit <N>` (default 100), `-o, --output-file <file>` (extension picks format: `.csv\|.tsv\|.json\|.dax`), `--trace`, `--cold`, `--plan`, `--runs <N>` (benchmark), `--no-validate` | +| `te script` | Run C# script (TOM) | `-S <file>` (repeatable, `.cs`/`.csx`), `-e <code>` (inline, `-` = stdin), `--save`, `--save-to`, `--serialization`, `--dry-run`, `--timeout <s>` | +| `te macro <sub>` | TE3 macros | `list`, `run <name-or-id>` (with `--on <obj-paths>`, `--save`), `add`, `set`, `rm`, `sort` | + +```bash +te query -q "EVALUATE TOPN(5, 'Sales')" -s ws -d model +te query -f query.dax --output-format json # global --output-format controls stdout format +te query -q "EVALUATE Sales" --output-file results.csv # writes CSV/TSV/JSON/DAX based on extension +te query -q "EVALUATE Sales" --runs 5 --cold --plan +te script -S fix.cs --save +te script -e "Info(Model.Tables.Count)" +echo "Info(Model.Name);" | te script -e - +te macro list +te macro run "Hide all measures" +te macro run "Format DAX" --on "Sales/Revenue,Sales/Margin" --save +``` + +### Deployment & Refresh + +| Command | Purpose | Key flags | +|---|---|---| +| `te deploy` | Deploy model | `-s/-d`, `--deploy-full` (overwrite + connections + partitions + roles + members + shared exprs), `--deploy-connections`, `--deploy-partitions`, `--skip-refresh-policy`, `--deploy-roles`, `--deploy-role-members`, `--deploy-shared-expressions`, `--create-only` (fail if exists), `--xmla <file>` (TMSL only, `-` for stdout), `--skip-bpa`, `--fix-bpa`, `--bpa-rules <file>` (repeatable), `--force` (**required for CI**), `--ci <fmt>`, `-p, --profile <name>` | +| `te refresh` | Trigger refresh | `--type full\|dataonly\|automatic\|calculate\|clearvalues\|defragment\|add` (default `automatic`), `--table <name>` (repeatable), `--partition <Tbl.Part>` (repeatable), `--apply-refresh-policy true\|false` (default true), `--effective-date yyyy-MM-dd`, `--max-parallelism <N>`, `--dry-run` (emit TMSL), `--no-progress`, `--trace [path]` (no value = stderr; with path = log file) | +| `te incremental-refresh <sub> <table>` | Manage IR policies | `show`, `set`, `remove`, `apply` (re-evaluate policy and create/expand partitions) | + +```bash +te deploy ./model -s ws -d model --force --ci github +te deploy ./model --xmla script.tmsl # generate TMSL only +te deploy ./model --xmla - # TMSL to stdout +te deploy ./model --profile staging --force +te refresh --type full +te refresh --table Sales --partition "Sales.2024" --type full +te refresh --type full --dry-run > refresh.tmsl +te refresh --type full --trace # XMLA trace events to stderr +te refresh --type full --trace refresh.log # XMLA trace events to log file +te incremental-refresh show Sales +te incremental-refresh apply Sales # re-evaluate policy, create/expand partitions +``` + +### Testing + +| Command | Purpose | Key flags | +|---|---|---| +| `te test run` | Run DAX assertion tests | `--suite <path>` (default `.te-tests/`), `--tag <tag>`, `--fail-on error\|warning`, `--ci`, `--trx <file>` | +| `te test init` | Scaffold suite | `--example`, `--from-model --model <path>` | +| `te test spec` | Print assertion format | n/a | +| `te test use <suite>` | Activate suite (session-scoped) | n/a | +| `te test list` | List test cases | n/a | +| `te test snapshot` | Capture model snapshot | n/a | +| `te test compare` | Compare snapshots | n/a | + +```bash +te test init --example +te test init --from-model --model ./my-model # generate stubs from model +te test run --ci github --trx results.trx +te test run --tag revenue +te test snapshot +te test compare +``` + +### Connection & Auth + +(Covered above under [Authentication](#authentication) and [Connections and profiles](#connections-and-profiles).) Full subcommands: + +``` +te connect [<server> <database>] [--local | -w/--workspace <path-or-server-db> | --workspace-format bim|tmdl|te-folder | --workspace-auth <method> | --force | -p/--profile <name> | --clear] +te auth login [-u <appId>] [-p <secret>|-] [-t <tenant>] [--identity|-I] [--certificate <path>] [--certificate-password <pw>] [--save] [--auth interactive|spn|env|managed-identity] +te auth status +te auth logout +te profile {set|show|list|remove} <name> [...] +te session [show | list | clear | prune [--all] [--dry-run]] +``` + +#### Sessions + +Every shell process gets its own session file under `~/.config/te/sessions/<id>.json`, holding the active connection, active profile, active test suite, and timestamps. Default session ID is derived from the parent shell PID; set `TE_SESSION=<name>` to name a session and share it across multiple shells or scripts. Sessions for dead PIDs are auto-cleaned on each invocation; `te session prune` triggers cleanup manually (or with `--all`, drop every session except the current one). + +```bash +te session # show current session (id, file, active state) +te session list # all session files on this machine +te session clear # reset active connection / profile / test suite for this shell +te session prune # delete sessions whose shell process is dead +te session prune --dry-run # preview what would be deleted +te session prune --all # delete every session except current (incl. named TE_SESSION ones) +TE_SESSION=ci-deploy te connect ws md # share session under name "ci-deploy" +``` + +Why it matters: `te connect`, `te test use`, and `--profile` all mutate the session file, not the global config. Two terminals can hold different active connections without stepping on each other. + + +> Configuration commands and the full key table: see config-cicd-env.md. + +### Shell + +| Command | Purpose | +|---|---| +| `te interactive [model]` | Model-aware REPL; prompt is `te [MyModel]>` or `te>`. All subcommands work without `te` prefix. Built-ins: `help`/`?`, `status`/`pwd`, `clear`/`cls`, `exit`/`quit`/`q` | +| `te completion <shell>` | Print completion script (`bash`, `zsh`, `pwsh`) | + +The REPL's argv splitter is bracket-aware, so DAX-style refs work without escaping the brackets; handy for paste-from-DAX-editor workflows: + +```bash +te interactive +te interactive ./model +te interactive -s MyWorkspace -d MyModel +te> ls Sales +te> ls Sa* # wildcard filter-paths +te> get "Sales/Revenue" -q expression +te> get [Total Sales] # lone-bracket: model-wide measure/column lookup +te> get 'Sales'[Amount] # DAX-quoted form +te> ls Roles/Reader/Members # role members +te> add Perspectives/Default/Sales # add Sales table to the Default perspective +te> bpa run --fail-on error +te> exit +``` + diff --git a/plugins/tabular-editor/skills/te-cli/references/config-cicd-env.md b/plugins/tabular-editor/skills/te-cli/references/config-cicd-env.md new file mode 100644 index 00000000..45e54968 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/references/config-cicd-env.md @@ -0,0 +1,162 @@ +# `te` configuration, CI/CD, and environment + +Companion to the te-cli skill (SKILL.md). + +### Configuration + +| Command | Purpose | +|---|---| +| `te config show [--output-format json]` | Show all settings | +| `te config paths` | Resolved file paths (macros, BPA rules, config) | +| `te config init [--force]` | Create default config | +| `te config set <key> <value>` | Update setting | +| `te license …` | **Hidden during preview.** Subcommands (`activate`, `status`, `deactivate`) are parseable so existing scripts don't fail at parse time, but any invocation prints *"`te license` is not available in this preview build"* and exits 1. Don't pipeline this. | +| `te migrate [-A] [--output-format text\|json]` | TE2 → new-CLI flag mapping (interactive lookup or full table) | + +**Config file**: `~/.config/te/config.json` (Windows: `%USERPROFILE%\.config\te\config.json`). Resolution order: `$TE_CONFIG` → default path → built-in defaults. + +**Configurable keys** (the keys accepted by `te config set`): + +| Key | Type | Default | Purpose | +|---|---|---|---| +| `macros` | path | _(none)_ | Override path to a `MacroActions.json` file | +| `queryLog` | path | _(none)_ | Path to the DAX query log file | +| `te3ExePath` | path | _(none)_ | Override path to the TE3 desktop executable (for `te open`) | +| `autoFormat` | bool | `false` | Apply DAX Formatter after mutations | +| `validateOnMutation` | bool | `true` | Verify `Table[Column]` references after edits | +| `vertipaqOnRefresh` | bool | `false` | Capture VertiPaq stats post-refresh | +| `bpa.rules` | string[] | _(none)_ | Path(s)/URL(s) to BPA rule file(s); repeatable; comma-separated on `te config set` | +| `bpa.onMutation` | bool | `false` | Run BPA after every mutation | +| `bpa.onDeploy` | bool | `true` | **BPA gate before deploy** (bypass: `--skip-bpa`) | +| `bpa.onSave` | bool | `true` | **BPA gate before save** (bypass: `--skip-bpa`) | +| `bpa.builtInRules` | bool | `true` | Include built-in default rules in scans | +| `bpa.disabledBuiltInRuleIds` | string[] | _(none)_ | Suppress specific built-in rule IDs | +| `formatOptions.useSemicolons` | bool | `false` | Use Euro separator (`;`) in DAX output | +| `formatOptions.shortFormat` | bool | `true` | Compact DAX layout (vs `--long`) | +| `formatOptions.skipSpaceAfterFunction` | bool | `false` | `SUM(x)` instead of `SUM (x)` | +| `formatOptions.useSqlBiDaxFormatter` | bool | `false` | Use SQLBI's online formatter instead of the in-house one | +| `interactiveEditMode` | enum | `stage` | Default for mutating commands: `stage` (in-memory only), `save` (auto-persist), `revert` (auto-roll-back). Overridden per-command by `--save`/`--stage`/`--revert` | +| `hidePreviewNotice` | bool | `false` | Suppress yellow preview banner | +| `spinner` | bool | `true` | Animated progress (disable for CI) | +| `debug` | bool | `false` | Debug logs to stderr | +| `disableTelemetry` | bool | `false` | Opt out of anonymous usage telemetry | + +**Note:** BPA keys are **nested under `bpa.`**; `te config set bpa.onDeploy false`, not `bpaOnDeploy`. Same for `formatOptions.*`. Active connection / profile / test-suite are **session-scoped** (see `te session`) and explicitly rejected by `te config set`; use `te connect`, `te profile`, `te test use` instead. + +**Speed knobs for batch / demo / CI runs**: each `te` invocation has ~1-2 s of process startup + model load. For pipelines that issue many sequential `te` calls (build scripts, live demos, mass-edit loops), set these once before the run: + +```bash +te config set bpa.onSave false # skip the BPA gate on every --save; run BPA once at the end instead +te config set spinner false # disable the animated progress widget (cleaner CI logs, slightly faster) +te config set hidePreviewNotice true # suppress the yellow preview banner +``` + +`bpa.onSave: false` is by far the biggest win; without it, BPA runs on every saved mutation, which on a typical model-build script means dozens of redundant passes. + +**Project-local BPA gate**: drop a `.te-bpa.json` in repo root (or set via `TE_BPA_CONFIG`) to override gate behavior per project. + + +## CI/CD integration + +### GitHub Actions + +```yaml +- name: Validate model + env: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + run: | + te validate ./model --ci github --trx validate.trx --non-interactive + +- name: BPA gate + run: te bpa run --rules ./rules/BPARules.json --fail-on error --ci github --non-interactive + +- name: Deploy + run: | + te deploy ./model \ + -s "${{ vars.WORKSPACE }}" \ + -d "${{ vars.SEMANTIC_MODEL }}" \ + --auth env --force --ci github --non-interactive + +- name: Run tests + run: te test run --ci github --trx test.trx --non-interactive + +- name: Publish TRX + if: always() + uses: dorny/test-reporter@v1 + with: + name: TE tests + path: '*.trx' + reporter: dotnet-trx +``` + +### Azure DevOps Pipelines + +Same commands, swap `--ci github` for `--ci azdo` (or `vsts`/`azure-devops`; all aliases). Pipeline annotations come back as native `##vso[...]` markers; `--trx` integrates with the `PublishTestResults@2` task. + +### Patterns + +- **Always pass** `--non-interactive` and `--auth env` (with `AZURE_CLIENT_*` env vars) and `--force` (on `te deploy`) +- **Stable annotations**: `--ci azdo` or `--ci github` on `validate`, `bpa run`, `deploy`, `test run`, `script` +- **Test publishing**: `--trx <file>` on `validate`, `bpa run`, `test run` for VSTEST-compatible XML +- **Promotion (dev → test → prod)**: build once, deploy with `--profile dev`, `--profile test`, `--profile prod` against the same TMDL artifact +- **Disable spinner in CI**: `te config set spinner false` in setup step + +## Output formats and exit codes + +**`--output-format`** (global stdout format): +- `auto` (default): text on TTY, JSON when stdout is piped/redirected +- `text`: forces human-readable +- `json`: always valid JSON to stdout; errors/warnings to stderr (won't contaminate) +- `csv`: tabular results (only `query`, `bpa run`, `vertipaq`) +- `tmsl` (alias `bim`): emit the resolved object(s) as TMSL/BIM JSON; supported on `te get` and `te ls` +- `tmdl`: emit the resolved object as TMDL; supported on `te get` (single named object only) and `te ls` + +```bash +te get Sales --output-format tmdl # Sales table as TMDL +te get "Sales/Revenue" --output-format bim # Single measure as TMSL fragment +te ls Tables --output-format bim # All tables as TMSL/BIM +te ls Measures --output-format tmdl # Every measure across the model, in TMDL +``` + +**`--ci` formats** (orthogonal to `--output-format`; emits CI-system logging commands to stderr on `validate`, `bpa run`, `deploy`, `test run`, `script`): + +| Value | Effect | +|---|---| +| `vsts`, `azdo`, `azure-devops` | Azure DevOps: `##vso[task.logissue type=error/warning;...]message` + `##vso[task.complete result=...]` summary | +| `github`, `gh` | GitHub Actions: `::error file=…,line=…::message` / `::warning::message` | +| anything else | No CI output | + +Errors and warnings are accumulated, so a non-zero exit code reflects total error count for the run. + +**Exit codes**: +- `0`; success +- `1`; generic failure: invalid args, validation errors, auth failure, BPA gate +- `2`; `te diff` only: models differ + +```bash +# JSON-safe pipeline +te ls --type measure --output-format json | jq -r '.[].path' + +# Bash conditional on diff +if te diff old.bim new.bim --output-format json > /dev/null; then + echo "Identical" +elif [ $? -eq 2 ]; then + echo "Models differ" +fi +``` + +## Environment variables + +| Var | Purpose | +|---|---| +| `TE_CONFIG` | Override config file path (otherwise `~/.config/te/config.json`) | +| `TE_DEBUG` | Set `1` or `true` for debug logging to stderr | +| `TE_COMPAT` | Set `te2` to force legacy compat mode | +| `TE_SESSION` | Name the current session (instead of parent-PID-derived ID). Lets multiple shells share active state; named sessions are never auto-cleaned | +| `TE_MACROS_PATH` | Override path to a `MacroActions.json` (highest priority for `te macro`) | +| `TE_BPA_RULES` | Override path to a BPA rules file (precedence: explicit `--rules` > `TE_BPA_RULES` > `bpa.rules` config > CWD `BPARules.json`) | +| `TE_BPA_CONFIG` | Override path to a `.te-bpa.json` gate-config (for deploy/save BPA gating) | +| `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_TENANT_ID` | SPN credentials (used with `--auth env`) | + diff --git a/plugins/tabular-editor/skills/te-cli/references/fabric-cli-tandem.md b/plugins/tabular-editor/skills/te-cli/references/fabric-cli-tandem.md new file mode 100644 index 00000000..f1ec8906 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/references/fabric-cli-tandem.md @@ -0,0 +1,199 @@ +# `te` + `fab` tandem workflows + +`te` (Tabular Editor CLI) and `fab` (Fabric CLI) split cleanly along a single seam. `te` owns the semantic model itself: it parses TMDL/BIM locally, edits objects (`set`, `add`, `rm`, `mv`, `format`), runs validation, BPA, and VertiPaq, generates TMSL, deploys over XMLA, and runs DAX queries, refreshes, and tests against a live XMLA endpoint. `fab` owns everything around the model in the service: discovering workspaces and items, resolving display names and GUIDs, exporting and importing item definitions over the Fabric REST API, managing permissions and capacities, driving deployment pipelines, and triggering refreshes via the Power BI REST API. Neither tool crosses into the other's half: `fab` has no DAX parser, no BPA engine, and no XMLA/TOM layer, so it cannot validate or BPA-check a model; `te` has no Fabric REST client, so it cannot enumerate workspaces, resolve item GUIDs, or trigger a REST refresh. The two share Azure AD identity but cache credentials independently, so authenticate both at the start of any tandem session. + +Before each workflow: `fab auth status` and `te auth status`. If either is unauthenticated, ask the user to run `fab auth login` / `te auth login`. During preview, run `te <command> --help` and `fab <command> --help` the first time a command is composed; both surfaces are still moving. + +## 1. Export, edit, deploy back (the common round-trip) + +Pull a production model to local TMDL with `fab`, edit and quality-gate with `te`, push it back over XMLA with `te deploy`. + +```bash +# 0. Confirm the item path is real before exporting +fab exists "Production.Workspace/Sales.SemanticModel" + +# 1. Pre-create the output dir (fab export will not create parents) +mkdir -p ./sales-export + +# 2. Download the model definition as a TMDL folder +fab export "Production.Workspace/Sales.SemanticModel" -o ./sales-export -f +# -> ./sales-export/Sales.SemanticModel/definition/ holds model.tmdl, tables/, etc. + +# 3. Confirm te parses the exported TMDL; settle on the path that loads +te load ./sales-export/Sales.SemanticModel/definition + +# 4. Baseline gates BEFORE editing (separate pre-existing issues from yours) +te validate -m ./sales-export/Sales.SemanticModel/definition --errors-only +te bpa run --fail-on error -m ./sales-export/Sales.SemanticModel/definition + +# 5. Edit (each mutation needs --save to persist) +te set "_Measures/Revenue" -q formatString -i "#,0.00" -m ./sales-export/Sales.SemanticModel/definition --save +te format --save -m ./sales-export/Sales.SemanticModel/definition + +# 6. Re-gate: no new errors or violations introduced +te validate -m ./sales-export/Sales.SemanticModel/definition --errors-only +te bpa run --fail-on error -m ./sales-export/Sales.SemanticModel/definition + +# 7. Deploy back over XMLA (BPA gate runs by default; --force required non-interactively) +te deploy ./sales-export/Sales.SemanticModel/definition -s "Production" -d "Sales" --force +``` + +Per-step purpose: `fab exists` fails fast on a wrong name; `mkdir -p` avoids `[InvalidPath]`; `te load` confirms the parse and pins the path; the baseline gates separate pre-existing breakage from edits being introduced; `--save` persists each mutation (staged in memory otherwise); the second gate is the quality bar before deploy; `te deploy` writes the definition back through XMLA, re-running BPA on the way out. + +Notes that bite: +- `fab export` nests TMDL under `<Model>.SemanticModel/definition/`. Run `te load` once to confirm whether the binary wants the `definition/` folder or its `.SemanticModel` parent, then use that exact path on every later `te` call. +- `te connect` state does not survive across separate shell calls. Pass `-m` (and `-s`/`-d` for remote) every time, or set `TE_SESSION=<name>` before the first call. +- The BPA gate is ON by default for `te deploy`. If pre-existing violations block the deploy, use `--skip-bpa` once and log it, or `--fix-bpa` to auto-remediate; do not silently bypass. +- `te deploy` overwrites by default. Add `--create-only` to refuse if the model already exists (it errors if it does), to prevent clobbering. + +### Variant: import back with `fab` instead of `te deploy` + +Use this when XMLA write is blocked (Pro/PPU without XMLA, or tenant policy). `fab import` goes through the Fabric item-definition REST API and works on any SKU. It does NOT run BPA or validation, so the `te` gate before it is the only safety net. + +```bash +te validate -m ./sales-export/Sales.SemanticModel/definition --errors-only +te bpa run --fail-on error -m ./sales-export/Sales.SemanticModel/definition +fab import "Production.Workspace/Sales.SemanticModel" -i ./sales-export/Sales.SemanticModel -f +``` + +`fab import` overwrites the whole item definition (no partial update) and does not refresh data. Pass the `.SemanticModel` folder (the one containing `.platform` and `definition/`), not the inner `definition/`. Trigger a refresh afterward if needed (workflow 5). + +### Variant: `te`-only pull (no `fab`) + +`te save` can read a remote model and write it to local disk, so the round-trip does not strictly require `fab export` when XMLA is available: + +```bash +te save -s "Production" -d "Sales" -o ./sales-export --serialization tmdl +``` + +Use `fab export` when XMLA is blocked or when the `.platform`/PBIP scaffolding is also wanted; use `te save` when an XMLA connection already exists and only the model source is wanted. + +## 2. Discover with `fab`, connect `te` by display name + +When the exact workspace or model name is unknown or its casing is uncertain, resolve it with `fab` first, then feed the canonical display name into `te -s`/`-d`. + +```bash +fab ls # list visible workspaces +fab find 'sales' -P type=SemanticModel -l # substring search; -l adds id + workspace_id +fab exists "Sales Analytics.Workspace/Sales Model.SemanticModel" # confirm the exact path + +te load -s "Sales Analytics" -d "Sales Model" # te takes the bare display name (no .Workspace suffix) +te bpa run --fail-on error -s "Sales Analytics" -d "Sales Model" # gate the live model, no local export +``` + +Boundary: `fab` resolves and confirms names against the Fabric REST API and OneLake catalog; `te` connects to the XMLA endpoint using the display name directly. Watch the path-shape mismatch: `fab` uses dot-extension syntax (`"Name.Workspace"`, `"Model.SemanticModel"`), but `te -s` takes the bare workspace display name and `te -d` the bare model name. Strip the `.SemanticModel` / `.Workspace` suffixes when handing off. Quote any name with spaces in both CLIs. + +## 3. Extract GUIDs with `fab`, refresh after a `te` change + +The Power BI REST API needs raw GUIDs that only `fab` can resolve from display names. `te` validates the model; `fab` triggers the refresh. + +```bash +te validate -s "Production" -d "Sales Model" --errors-only # don't refresh a broken model + +WS_ID=$(fab get "Production.Workspace" -q "id" | tr -d '"') +MODEL_ID=$(fab get "Production.Workspace/Sales Model.SemanticModel" -q "id" | tr -d '"') + +fab api -A powerbi "groups/$WS_ID/datasets/$MODEL_ID/refreshes" -X post -i '{"type":"Full"}' +fab api -A powerbi "groups/$WS_ID/datasets/$MODEL_ID/refreshes?\$top=1" -q "value[0].{status:status,started:startTime}" +``` + +`fab get -q "id"` returns a quoted JSON string; always pipe through `tr -d '"'` or the GUID carries literal quotes that break URL construction. `te refresh --type full -s ws -d model` also triggers a refresh over XMLA; use `fab api` when the GUIDs are already in scope or when XMLA refresh is not available. The refresh is async; the `?$top=1` check is a sanity probe, not a completion wait. + +## 4. Promote dev to test/prod with quality gates + +Export from dev, diff against the target, gate, then deploy. The local round-trip is deliberate: `fab cp` can copy workspace-to-workspace faster but skips every `te` gate. + +```bash +mkdir -p ./promote +fab export "Dev.Workspace/Sales.SemanticModel" -o ./promote -f + +# Structural diff against the live prod model over XMLA +te diff ./promote/Sales.SemanticModel/definition -s "Production" -d "Sales" + +te validate -m ./promote/Sales.SemanticModel/definition --errors-only +te bpa run --fail-on error -m ./promote/Sales.SemanticModel/definition + +# Deploy, including RLS roles and members +te deploy ./promote/Sales.SemanticModel/definition \ + -s "Production" -d "Sales" \ + --deploy-roles --deploy-role-members --force + +fab exists "Production.Workspace/Sales.SemanticModel" # confirm it landed +``` + +`--deploy-roles` / `--deploy-role-members` are opt-in; omit them when RLS is managed independently in prod. If XMLA is blocked, `te diff` cannot run against a live `-s`/`-d` target; instead `fab export` prod separately and diff two local folders: `te diff ./promote/Sales.SemanticModel/definition ./prod-export/Sales.SemanticModel/definition`. + +### Variant: governed promotion via deployment pipeline + +`te` provides the quality gate and an optional TMSL audit artifact; `fab` drives the Fabric deployment pipeline (which preserves item IDs across stages, so thin reports do not need rebinding). + +```bash +te bpa run --fail-on error --ci azdo --non-interactive -m ./dev-export/Sales.SemanticModel/definition + +# Optional: emit the TMSL a direct XMLA deploy WOULD run, as an audit artifact (does not execute) +te deploy ./dev-export/Sales.SemanticModel/definition -s "Dev" -d "Sales" --xmla - > ./audit/deploy.tmsl + +PIPELINE_ID=$(fab api "deploymentPipelines" -q "value[?displayName=='Sales Pipeline'].id | [0]" | tr -d '"') +DEV_STAGE=$(fab api "deploymentPipelines/$PIPELINE_ID/stages" -q "value[?order==\`0\`].id | [0]" | tr -d '"') +TEST_STAGE=$(fab api "deploymentPipelines/$PIPELINE_ID/stages" -q "value[?order==\`1\`].id | [0]" | tr -d '"') + +# Promote; capture the LRO id from the response header +fab api -X post "deploymentPipelines/$PIPELINE_ID/deploy" \ + -i "{\"sourceStageId\":\"$DEV_STAGE\",\"targetStageId\":\"$TEST_STAGE\",\"note\":\"BPA-gated\"}" \ + --show_headers + +# Poll the LRO to completion +until s=$(fab api "operations/$OPERATION_ID" -q "status" | tr -d '"'); [ "$s" = "Succeeded" ] || [ "$s" = "Failed" ]; do sleep 30; done +``` + +The TMSL audit artifact describes a direct XMLA deploy from the local source, not what the pipeline promotion will do; treat it as a reference, not a contract. Capture `OPERATION_ID` from the `x-ms-operation-id` header immediately; it is not reliably retrievable later. Pipelines copy definitions only; refresh separately (workflow 5). + +## 5. Post-deploy: refresh, then DAX regression tests + +A pipeline or XMLA deploy moves definitions, not data. Refresh with `fab`, wait, then run the `te` test suite against the live model. + +```bash +WS_ID=$(fab get "Test.Workspace" -q "id" | tr -d '"') +MODEL_ID=$(fab get "Test.Workspace/Sales.SemanticModel" -q "id" | tr -d '"') +fab api -A powerbi "groups/$WS_ID/datasets/$MODEL_ID/refreshes" -X post -i '{"type":"Full"}' + +# Wait for the async refresh before asserting (neither tool has a blocking wait; poll in a loop) +until s=$(fab api -A powerbi "groups/$WS_ID/datasets/$MODEL_ID/refreshes?\$top=1" -q "value[0].status" | tr -d '"'); [ "$s" = "Completed" ] || [ "$s" = "Failed" ]; do sleep 30; done + +te test run --suite ./.te-tests/ --ci azdo --trx test-results.trx --non-interactive -s "Test Workspace" -d "Sales" +``` + +Do not run `te test` before the refresh finishes; stale or empty data causes false failures. `te test run` needs a `.te-tests/` suite; scaffold one first with `te test init --example`. In CI, set `AZURE_CLIENT_ID`/`AZURE_CLIENT_SECRET`/`AZURE_TENANT_ID` and pass `--auth env`. + +## 6. Governance: enumerate with `fab`, audit with `te` + +Discovery only comes from `fab`; `te` has no workspace or item listing surface. Pair them for tenant-wide BPA audits, VertiPaq profiling, and unused-column sweeps. + +```bash +# Enumerate semantic models across all visible workspaces +fab find '' -P type=SemanticModel -l --output_format json > /tmp/models.json +jq -r '.[] | "\(.workspace)/\(.name)"' /tmp/models.json + +# Per model: export, then analyze locally +mkdir -p /tmp/audit +fab export "Production.Workspace/Sales.SemanticModel" -o /tmp/audit -f + +te bpa run /tmp/audit/Sales.SemanticModel/definition --rules ./BPARules.json --fail-on error --ci github +te deps --unused --hidden -m /tmp/audit/Sales.SemanticModel/definition --output-format json +te vertipaq /tmp/audit/Sales.SemanticModel/definition --columns --detail --top 20 +``` + +For governance fields `fab find` does not expose (last refresh, storage mode, owner, capacity SKU), use the fabric-cli skill's `scripts/search_across_workspaces.py` (note its filter is `--type Model`, not `SemanticModel`). VertiPaq stats from an offline TMDL export give column/relationship structure but not live row counts or cardinality; for those, point `te vertipaq` at a live `-s`/`-d` XMLA endpoint with `--stats`. `te deps --unused` flags objects with no DAX references, but a relationship key column can show as "unused" despite being load-bearing; inspect with `te get` before removing. + +## Boundaries and gotchas + +- **Path layout after `fab export`.** TMDL lands in `<Model>.SemanticModel/definition/`. `te validate`/`bpa run`/`vertipaq`/`deploy` operate on the model source; `fab import` wants the `.SemanticModel` folder (with `.platform`). Confirm the exact `te` target with `te load` first. +- **Return path: XMLA vs REST.** `te deploy` writes via XMLA and runs BPA inline but needs XMLA write (Premium/Fabric capacity, PPU with XMLA, or Trial). `fab import` writes via the Fabric REST API on any SKU but runs no gate. Pick by SKU and by whether the inline BPA gate is wanted. +- **`te diff` exit codes are documented inconsistently in the te-cli skill.** The exit-codes table (config-cicd-env.md) says `2` = models differ, `1` = generic failure; the command-reference table says `1` = differs, `2` = error. Do not branch on the exact code without verifying against the installed binary (`te diff a b; echo $?`). Both agree `0` = identical. +- **Two-document JSON from `te bpa run --fix`.** With `--output-format json`, a `--fix` run emits two concatenated JSON documents (scan result, then fix summary). Pipe through `jq --slurp` (`jq -s '.[0]'` / `.[1]`) or drop `--output-format json` and read text. A single-document parser fails with trailing-token errors. +- **Quoted GUIDs from `fab get`.** `fab get -q "id"` returns a quoted string; always `| tr -d '"'` before interpolating into an API path. +- **`fab` REST refresh path shape.** `fab api -A powerbi` uses the Power BI `groups/<ws-id>/datasets/<model-id>` shape; the Fabric REST API uses `workspaces/<ws-id>/semanticModels/<model-id>`. Same GUIDs, different URL. +- **No native pipe between the CLIs.** `fab find` / `search_across_workspaces.py` produce paths; a shell intermediary (`jq`, `while read`) feeds them into a `te` loop. `te` has no multi-model loop and no service-discovery command of its own. +- **Always `mkdir -p` before `fab export`** (it does not create parents), and **always `-f`** on `fab export`/`import` (skips the sensitivity-label / overwrite prompt). If sensitivity labels or DLP policies are in play, confirm with the user before exporting; `-f` strips the label on export. +- **CI flags on `te`.** Pass `--non-interactive` and `--force` (on `te deploy`) or the confirmation prompt defaults to `n` and hangs the pipeline. Use `--auth env` with `AZURE_CLIENT_*`; never put a secret on the command line. +- **Thin-report rebinding is a `fab` job.** After a `fab import` or `fab cp` of a thin report to a new workspace, rebind with `fab set "<ws>/<Report>.Report" -q semanticModelId -i "<target-model-id>"`. Deployment pipelines preserve IDs and skip this; manual import/copy does not. diff --git a/plugins/tabular-editor/skills/te-cli/references/get-te-cli.md b/plugins/tabular-editor/skills/te-cli/references/get-te-cli.md new file mode 100644 index 00000000..7baed6d9 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/references/get-te-cli.md @@ -0,0 +1,52 @@ +# Get the `te` CLI binary (agent self-service) + +The `te` (Tabular Editor CLI) binary is published per platform on a public CDN, +no account or auth needed: + + https://cdn.tabulareditor.com/files/cli/latest/te-<os>-<arch>.tar.gz (.zip on Windows) + +Pick the asset for the platform: + +- macOS arm64 (Apple Silicon): `te-osx-arm64.tar.gz` +- macOS x64 (Intel): `te-osx-x64.tar.gz` +- Linux x64: `te-linux-x64.tar.gz` +- Linux arm64: `te-linux-arm64.tar.gz` +- Windows x64: `te-win-x64.zip` +- Windows arm64: `te-win-arm64.zip` + +## macOS / Linux: detect, download, put on PATH + +```bash +os=$(uname -s | tr 'A-Z' 'a-z'); [ "$os" = darwin ] && os=osx +arch=$(uname -m); case "$arch" in arm64|aarch64) arch=arm64 ;; x86_64|amd64) arch=x64 ;; esac +mkdir -p "$HOME/.local/bin" +curl -fsSL "https://cdn.tabulareditor.com/files/cli/latest/te-$os-$arch.tar.gz" \ + | tar xz -C "$HOME/.local/bin" te +chmod +x "$HOME/.local/bin/te" +export PATH="$HOME/.local/bin:$PATH" # this shell +"$HOME/.local/bin/te" --version +``` + +Persist PATH across shells once (skip if `~/.local/bin` is already on PATH): + +```bash +echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc # or ~/.zshrc +``` + +## Windows (PowerShell): download, put on PATH + +```powershell +$arch = if ($env:PROCESSOR_ARCHITECTURE -eq 'ARM64') { 'arm64' } else { 'x64' } +$dir = "$env:LOCALAPPDATA\Programs\te" +New-Item -ItemType Directory -Force -Path $dir | Out-Null +Invoke-WebRequest "https://cdn.tabulareditor.com/files/cli/latest/te-win-$arch.zip" -OutFile "$env:TEMP\te.zip" +Expand-Archive -Force "$env:TEMP\te.zip" -DestinationPath $dir +[Environment]::SetEnvironmentVariable('Path', "$([Environment]::GetEnvironmentVariable('Path','User'));$dir", 'User') +& "$dir\te.exe" --version +``` + +## Verify and stay current + +`te --version` prints the build. The CDN `latest` path is always the newest at +download time; for a binary that keeps itself current, use the self-updating +`te` wrapper instead (`te --update`, plus a daily check on `te --version`). diff --git a/plugins/tabular-editor/skills/te-cli/references/gotchas.md b/plugins/tabular-editor/skills/te-cli/references/gotchas.md new file mode 100644 index 00000000..849c2703 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/references/gotchas.md @@ -0,0 +1,45 @@ +# `te` gotchas + +Sharp edges and non-obvious behavior. Companion to the te-cli skill (SKILL.md). + +## Gotchas + +### Path & property-name asymmetries + +- **MPartition path asymmetry**: `te add` for an MPartition uses `<Table>/<PartitionName>` (no `/Partitions/` segment). Every other partition command; `te rm`, `te get`, `te ls`, `te mv`, `te set`; uses `<Table>/Partitions/<PartitionName>`. Mixing these up errors with "Cannot add a MPartition at path … Check that -t matches the path shape." +- **Partition M property: `MExpression` for `te set`, `expression` for `te get`**: `te get <Table>/Partitions/<P>` displays the M as `expression`, but `te set -q expression …` errors with "Property 'expression' not found on MPartition. Did you mean: MExpression?"; use `-q MExpression -i "<M>" --save` to update. +- **`te mv` cannot rename a partition to its parent table's name**: `te mv <Table>/Partitions/<Table>_m <Table>/Partitions/<Table>` errors with either "Destination '<Table>/<Table>' already exists" or "Partition '<Table>' not found in table '<Table>'"; the destination path `<Table>/<Table>` resolves to the **table object itself**, not a partition slot inside the table. Passing `-t Partition` does **not** disambiguate. Workaround: rename via the `Name` property; `te set <Table>/Partitions/<Table>_m -q Name -i <Table> --save`. +- **Object paths use `/` as separator**, not `\` or `.`. Quote paths with spaces: `"Sales 2024"/Revenue`, `"_Measures/Total Revenue"`, `"Date/Date Hierarchy/Year"`. +- **`te ls` cannot list relationships**: both `te ls Relationships` and `te ls --type relationship` error with `Error: No objects match path 'Relationships'` even when the model has relationships (confirmed against a 23-relationship model, local TMDL and live XMLA alike). `relationship` is advertised in `te ls --type`'s help, but unlike every other container (`Tables`, `Measures`, `Roles`, `Perspectives`, `Cultures`, `Hierarchies`) the keyword is not wired to the collection, so it falls through to a literal path match. Tell: recognized-but-empty containers say `No objects match 'X'`; the broken keyword says `No objects match path 'X'` (note the word `path`). Enumerate relationships with DAX instead: `te query -q "EVALUATE INFO.VIEW.RELATIONSHIPS()"` (friendly `from ∞←1 to`, active flag, cross-filter direction), or `INFO.RELATIONSHIPS()` on older compat, or `te save` to TMDL and read `relationships.tmdl`. A single relationship is addressable by name via `te get Relationships/<name>`, but auto-assigned names (`Relationship 2`) make DAX enumeration the practical discovery path. + +### Object types and creation + +- **`-t DataColumn` is not accepted by `te add`**: data columns are declared at table creation via `--columns "Name:Type,..."` (auto-creates with `SourceColumn = Name`). Tune additional properties (`IsKey`, `IsHidden`, `FormatString`, `SummarizeBy`, `SortByColumn`, `Description`) afterwards with `te set`. Only `CalculatedColumn` can be added individually with `-t CalculatedColumn -i "<DAX>"`. +- **Workflow ordering; relationships before measures**: measures that use `RELATED()` or cross-table `CALCULATE()` are validated at save time. If the relationship doesn't exist yet, the save gate rejects with `DAX0002: Column '<T>'[<C>] doesn't have a relationship to any table available in the current context`. Add relationships before authoring dependent measures. If that order is impossible (scripted batch creation), use `--force` only as a last resort and run `te validate` immediately after to confirm no DAX errors remain before committing. +- **`te rm` on the last partition fails** with "Cannot remove last partition from table". Always add a replacement partition first, then remove the original. +- **Silent M syntax errors during partition authoring**: a typo in the partition M (unbalanced braces, an unquoted token) doesn't always raise at save time. After creating a data-bound table, sanity-check with `te get <Table>/Partitions/<Table>` to confirm `sourceType: M` and that the expression matches what was passed in. + +### Output shapes + +- **`te bpa run --output-format json`**: top-level `violations` is the **count** (integer), top-level `results` is the **list** of violation objects with `ruleId`, `severityLabel`, `objectName`, `canFix`, etc. Don't confuse the two when piping into `jq`. +- **`te bpa run --fix [--save] --output-format json` emits TWO concatenated JSON documents** to stdout, not one. **Never pipe it into a single-document JSON parser.** First comes the scan result (`{model, rulesEvaluated, violations, results, errors, ...}`), then a fix summary (`{fixed, fixErrors, skipped, fixedItems, fixErrorItems}`). Strict single-document parsers fail with an "extra data" / "trailing tokens" error on the second document. Prefer in this order: + 1. **`jq --slurp`** (shell-native, no interpreter dependency): `te bpa run --fix --save --output-format json | jq -s '.[0].violations'` (scan count) or `jq -s '.[1].fixed'` (fix count). `--slurp` reads concatenated documents into an array. + 2. **Drop `--output-format json` entirely** and rely on the human-readable text for one-shot summaries; simplest and works without any extra tooling. Pair with `--ci github`/`--ci azdo` when machine-actionable annotations are needed. + 3. **Tail-grep for the summary lines** (`grep -E '"violations":|"fixed":' | head -2`) when only the counts are needed. + +### Behavior + +- **`te ls` is filesystem-style, not workspace-style**. `te ls Sales` lists Sales' children (columns + measures), not "find Sales". Use `te find` for full-text search. +- **`--save` is opt-in for editing commands** (when `interactiveEditMode` is the default `stage`). Without it, `te set`, `te add`, `te rm`, `te mv`, `te replace`, `te format`, `te script`, `te macro run` operate in memory only and don't persist. See the Staging model section in SKILL.md for the full picture and the `save` / `revert` alternatives. +- **`te validate` does not exercise partition M**; it checks structural/DAX validity, not whether `Table.FromRows` literals parse or SQL endpoints respond. A model with broken partitions still passes `te validate` cleanly. Verify partitions explicitly after table creation. +- **`te connect` is session-scoped (per shell PID)**. Each fresh shell (each Claude Code `Bash` tool call spawns one) starts a new session without the active connection. Either set `TE_SESSION=<name>` to share state between shells, or pass `-m <model>` (and `-s`/`-d` where needed) explicitly to every command. +- **BPA config keys are nested under `bpa.`**. `te config set bpaOnDeploy false` will fail with "Unknown key"; the correct form is `te config set bpa.onDeploy false`. Same for `bpa.onSave`, `bpa.onMutation`, `bpa.rules`, `bpa.builtInRules`, `bpa.disabledBuiltInRuleIds`. +- **`--output-format` (stdout) vs `--serialization` (on-disk)** are different flags. The first picks how stdout is rendered (text/json/csv/tmsl/tmdl); the second picks the model file format (tmdl/bim/te-folder/pbip/database.json). Passing one when the other was meant gives a confusing error or silent wrong output. +- **`te deploy --create-only` fails if model exists**. Use without `--create-only` to overwrite (the default), or check with a probe call first. +- **`te deploy` confirmation prompt hangs CI** without `--force`. Default answer is `n` (safe), so non-interactive runs need `--force --non-interactive`. +- **BPA gate on deploy/save can mask sloppy commits**. If bypassing with `--skip-bpa`, log it loudly. Prefer `--fix-bpa` or address violations. +- **TE2 compat mode auto-detects** when args contain TE2-style flags but no `te` subcommand. Don't rely on this in scripts; be explicit with `TE_COMPAT=te2` so behavior is reproducible. +- **Local SSAS / Power BI Desktop are Windows-only**: `te connect --local` and `te connect "localhost:PORT"` won't work on macOS/Linux even though the binary runs there. +- **Preview banner reappears 14 days before the 2026-09-30 cutoff** regardless of `hidePreviewNotice`. Plan for the hard expiry. +- **Secrets on the cmd line leak** into `ps`, shell history, CI logs. Use `--auth env` with env vars, stdin (`-`), or `--auth managed-identity`. + diff --git a/plugins/tabular-editor/skills/te-cli/references/pbir-cli-tandem.md b/plugins/tabular-editor/skills/te-cli/references/pbir-cli-tandem.md new file mode 100644 index 00000000..2665fd9a --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/references/pbir-cli-tandem.md @@ -0,0 +1,252 @@ +# `te` + `pbir` tandem workflows + +`te` (Tabular Editor CLI) owns the semantic model: tables, columns, measures, relationships, DAX expressions, BPA, validation, and deployment to a workspace. `pbir` (pbir CLI) owns the Power BI report layer (.pbir): pages, visuals, field references and bindings, filters, themes, bookmarks, and extension measures. Neither tool crosses the line. `te` has no visibility into report JSON, and `pbir` cannot mutate TMDL or run DAX as a model edit. The contract that joins them is the `Table.Field` string: `te` changes an object's identity in the model, then `pbir` rewrites every report binding that still points at the old `Table.Field`. Run both halves, or ship a broken model or a broken report. + +Two rules before every command: model edits in `te` stage in memory and need `--save` to persist; `te replace` and `pbir fields replace` are dry-run-leaning (`te replace` previews unless `--save` is passed; `pbir fields replace` runs for real unless `--dry-run` is passed). When a flag or argument order is unfamiliar, run `te <command> --help` or `pbir <command> --help` first; both CLIs are evolving and the help text is authoritative. + +## 1. Rename a measure (model) and repair report bindings + +The most common refactor. `te mv` renames the object only; it does not rewrite DAX that calls the old name, so `te replace --in expressions` is a mandatory second step. + +```bash +# te: confirm the object and find DAX that calls it by name +te find "OldRevenue" --in names --paths-only -m ./Model.SemanticModel +te find "OldRevenue" --in expressions -m ./Model.SemanticModel # measures, calc columns, KPIs, role filters, calc-group selection +te deps "_Measures/OldRevenue" --downstream -m ./Model.SemanticModel # blast radius + +# te: rename the object, then fix every DAX reference to the old name +te mv "_Measures/OldRevenue" "_Measures/Revenue" --save -m ./Model.SemanticModel +te replace "OldRevenue" "Revenue" --in expressions --save -m ./Model.SemanticModel + +# te: gate before touching the report +te validate -m ./Model.SemanticModel --errors-only + +# pbir: find report bindings on the old reference (run --help to confirm arg order) +pbir fields find "Report.Report" -f "_Measures.OldRevenue" + +# pbir: preview, then apply the report-side rewrite +pbir fields replace "Report.Report" --from "_Measures.OldRevenue" --to "_Measures.Revenue" --dry-run +pbir fields replace "Report.Report" --from "_Measures.OldRevenue" --to "_Measures.Revenue" + +# pbir: confirm every binding resolves against the model +pbir validate "Report.Report" --fields +``` + +Per-step purpose: + +```yaml +te find --in names: confirm the measure exists and get its path before touching anything +te find --in expressions: discover DAX that calls the old name by value; te mv will NOT fix these +te deps --downstream: list measures/columns downstream to show the full impact +te mv: rename the TOM object (the name property only) +te replace --in expressions: rewrite every DAX call of the old name across the model; needs --save +te validate --errors-only: confirm no broken DAX references remain +pbir fields find: locate visuals, filters, and CF entries bound to the old reference +pbir fields replace --dry-run: preview the report rewrite +pbir fields replace: rewrite queryState projections, queryRefs, and nativeQueryRefs in one pass +pbir validate --fields: confirm all bindings resolve; zero broken references expected +``` + +## 2. Rename a column (model) and repair report bindings + +Same shape as a measure rename, plus column-only metadata to check after `te mv`. + +```bash +te find "OldColumnName" --in names --paths-only -m ./Model.SemanticModel # check for same name on other tables +te deps "Date/OldColumnName" --downstream -m ./Model.SemanticModel +te mv "Date/OldColumnName" "Date/NewColumnName" --save -m ./Model.SemanticModel +te replace "OldColumnName" "NewColumnName" --in expressions --save -m ./Model.SemanticModel +te validate -m ./Model.SemanticModel --errors-only + +pbir fields find "Report.Report" -f "Date.OldColumnName" +pbir fields replace "Report.Report" --from "Date.OldColumnName" --to "Date.NewColumnName" --dry-run +pbir fields replace "Report.Report" --from "Date.OldColumnName" --to "Date.NewColumnName" +pbir validate "Report.Report" --fields +``` + +After the rename, verify two column relationships that store the old name as a property, not as a reference `te replace` would catch: + +```bash +te get Date/SomeOtherColumn -q sortByColumn -m ./Model.SemanticModel # update with te set if it broke +te ls "Date/Geography/Levels" -m ./Model.SemanticModel # hierarchy levels take a column property +``` + +## 3. Rename a table (model) and repair all report bindings + +`pbir fields replace` works per `Table.Field`, not per table. There is no bulk table-prefix rewrite. Enumerate the affected fields first, then loop. Run the model-internal DAX rewrite before the rename so expressions are consistent at save time. + +```bash +te find "FACT_Sales" --in names -m ./Model.SemanticModel +te find "FACT_Sales" --in expressions -m ./Model.SemanticModel +te replace "FACT_Sales" "Sales" --in expressions --save -m ./Model.SemanticModel +te mv FACT_Sales Sales --save -m ./Model.SemanticModel +te validate -m ./Model.SemanticModel --errors-only + +# pbir: list fields, then replace each FACT_Sales.* binding individually +pbir fields list "Report.Report" --json +pbir fields replace "Report.Report" --from "FACT_Sales.Amount" --to "Sales.Amount" --dry-run +pbir fields replace "Report.Report" --from "FACT_Sales.Amount" --to "Sales.Amount" +# repeat the replace for every distinct FACT_Sales.<field> in the report +pbir validate "Report.Report" --fields +``` + +```yaml +te replace --in expressions: rewrites DAX text only; apostrophe-quoted refs need the quotes in the find term, e.g. te replace "'FACT_Sales'" "'Sales'" --in expressions --save +te replace substring risk: if the old name is a substring of another identifier, add --case-sensitive or --regex with anchors and review the preview +relationship endpoints: te mv renames the table object; confirm relationship integrity with te validate +``` + +## 4. Move a measure to a different table and update bindings + +`te mv` across tables is the only way to change an object's table ownership. Only fully table-qualified DAX (`SourceTable[Measure]`) breaks; unqualified `[Measure]` keeps resolving model-wide. + +```bash +te deps "SourceTable/MeasureName" --downstream -m ./Model.SemanticModel +te mv "SourceTable/MeasureName" "TargetTable/MeasureName" --save -m ./Model.SemanticModel +te find "SourceTable" --in expressions --paths-only -m ./Model.SemanticModel # decide if te replace is needed +te validate -m ./Model.SemanticModel --errors-only + +pbir fields find "Report.Report" -f "SourceTable.MeasureName" +pbir fields replace "Report.Report" --from "SourceTable.MeasureName" --to "TargetTable.MeasureName" +pbir validate "Report.Report" --fields +``` + +Display folder does not move with `te mv`; reset it on the moved measure if it should match the new table's folder structure (`te set "TargetTable/MeasureName" -q displayFolder -i "<folder>" --save`). + +## 5. Scaffold a model, then create a thin report against it + +`te` builds the TMDL model and deploys it; `pbir` creates a thin report bound to the published model `byConnection`. Deploy is a hard prerequisite for the `-c` workspace binding to resolve. + +```bash +# te: scaffold and author +te init ./Model.SemanticModel --serialization tmdl # PowerBI mode, compat 1702 default +te add Sales -t Table --columns "OrderID:Int64,Amount:Decimal,OrderDate:DateTime" --save -m ./Model.SemanticModel +te add "_Measures/Revenue" -t Measure -i "SUM(Sales[Amount])" --save -m ./Model.SemanticModel +te set "_Measures/Revenue" -q formatString -i "#,0.00" --save -m ./Model.SemanticModel +te set "_Measures/Revenue" -q displayFolder -i "Revenue" --save -m ./Model.SemanticModel + +# te: gate and deploy +te validate -m ./Model.SemanticModel --errors-only +te bpa run --fail-on error -m ./Model.SemanticModel +te deploy ./Model.SemanticModel -s "MyWorkspace" -d "Sales Model" --force --non-interactive + +# pbir: create thin report bound to the published model, build, validate +pbir new report "Sales.Report" -c "MyWorkspace/Sales Model.SemanticModel" +pbir pages rename "Sales.Report/Page 1.Page" "Overview" +pbir model "Sales.Report" -d # introspect tables/measures before binding +pbir add visual card "Sales.Report/Overview.Page" --title "Revenue" -d "Values:_Measures.Revenue" -t Measure --y 120 +pbir validate "Sales.Report" --fields +``` + +```yaml +te deploy --force --non-interactive: deploy prompts with n as the default and hangs scripts without --force; set both in CI +pbir new report -c: a workspace target produces a byConnection (thin) report; the model must be reachable in the workspace first +pbir add visual -t Measure: pass the type or -d defaults to a Column binding, which fails at runtime even though validate passes the JSON +pbir model -d: schema comes via TMDL, not DMV; -q runs EVALUATE DAX only +te validate scope: does not exercise M partitions; broken M surfaces only on refresh +``` + +To bind to a local model on disk instead of a workspace, create the report and then rebind with the documented local form: + +```bash +pbir report rebind "Sales.Report" --local "../Sales.SemanticModel" +``` + +## 6. Add a measure to a live model, then surface it in a bound report + +```bash +te ls Measures -m ./Model.SemanticModel # check naming conventions, avoid duplicates +te add "_Measures/Revenue YoY" -t Measure -i "DIVIDE([Revenue], CALCULATE([Revenue], SAMEPERIODLASTYEAR('Date'[Date]))) - 1" --save -m ./Model.SemanticModel +te set "_Measures/Revenue YoY" -q formatString -i "0.0%" --save -m ./Model.SemanticModel +te set "_Measures/Revenue YoY" -q displayFolder -i "Revenue" --save -m ./Model.SemanticModel +te set "_Measures/Revenue YoY" -q description -i "Year-over-year revenue growth" --save -m ./Model.SemanticModel +te validate -m ./Model.SemanticModel --errors-only +te deploy ./Model.SemanticModel -s "MyWorkspace" -d "Sales Model" --force --non-interactive + +pbir model "Sales.Report" --cache # refresh the report's cached model definition +pbir model "Sales.Report" -d -t _Measures | grep -i "YoY" +pbir add visual card "Sales.Report/Overview.Page" --title "Revenue YoY" -d "Values:_Measures.Revenue YoY" -t Measure --y 120 +pbir validate "Sales.Report" --fields +``` + +The Date table must be marked (`te set Date -q dataCategory -i Time --save`) for `SAMEPERIODLASTYEAR` to evaluate; `te validate` catches a missing mark. + +## 7. Remove a column: clear report references first, then delete + +Report-first, model-second. Clear the report bindings while the column still exists so validation can still resolve the type, then delete in the model. Reversing the order breaks the report on deploy. + +```bash +te deps Sales/OldRegionCode --downstream -m ./Model.SemanticModel # model-side dependents + +# pbir: find and remove the report references first +pbir fields find "Report.Report" -f "Sales.OldRegionCode" +pbir validate "Report.Report" +# surgical removal per visual is safer than a broad clear: +pbir visuals bind "Report.Report/Page.Page/Visual.Visual" -r "Category:Sales.OldRegionCode" + +# te: delete only after the report is clean +te rm Sales/OldRegionCode --dry-run -m ./Model.SemanticModel +te rm Sales/OldRegionCode --if-exists --save -m ./Model.SemanticModel +te validate -m ./Model.SemanticModel --errors-only +te deploy ./Model.SemanticModel -s "MyWorkspace" -d "Sales Model" --force --non-interactive +``` + +```yaml +removal granularity: pbir visuals bind -r removes one role binding on one visual; prefer it over a report- or page-wide pbir fields clear, which strips bindings broadly and can leave visuals with empty roles +sort-by dependency: te rm fails if the column is another column's sortByColumn target; clear that first with te set OtherColumn -q sortByColumn -i "" --save +``` + +## 8. Split a thick PBIP, edit the model, keep the report in sync + +`pbir` owns the structural split and the `definition.pbir` connection record; once the model is in the workspace, `te` edits it directly over the workspace endpoint. + +```bash +pbir model "ThickReport.Report" # confirm byPath (thick) +pbir report split-from-thick ThickProject --target "MyWorkspace.Workspace/Sales Model.SemanticModel" -F pbir +pbir model "ThickReport.Report" # confirm byConnection (thin) + +te load -s "MyWorkspace" -d "Sales Model" # confirm te reaches the published model +te set "_Measures/Revenue" -q description -i "Total net revenue" -s "MyWorkspace" -d "Sales Model" --save +te bpa run --fail-on error -s "MyWorkspace" -d "Sales Model" + +pbir validate "ThickReport.Report" --fields +``` + +After `split-from-thick` there is no local TMDL to pass to `-m`; use `-s`/`-d` for all later `te` commands. The split's publish step needs the Fabric CLI (`fab`) authenticated, separate from `te auth`. + +## 9. Deploy and publish together + +```bash +te validate -m ./Model.SemanticModel --errors-only && te bpa run --fail-on error -m ./Model.SemanticModel +te deploy ./Model.SemanticModel -s "MyWorkspace" -d "Sales" --force --non-interactive +pbir report rebind "Sales.Report" "MyWorkspace/Sales.SemanticModel" # byPath -> byConnection +pbir validate "Sales.Report" --fields # validate against the remote model +pbir publish "Sales.Report" "MyWorkspace/Sales" -f # positional args, not --workspace +``` + +`pbir report rebind` must come after `te deploy` completes, or validation at publish time fails. `pbir publish` takes positional source and destination, never `--workspace`. + +## Boundaries and gotchas + +```yaml +te owns: + - object identity (te mv / te set -q name) and DAX expression repair (te replace --in expressions) + - dependency analysis (te deps), validation (te validate), BPA (te bpa run), deploy (te deploy) + - te mv renames the object only; it does NOT rewrite DAX that calls the old name + - te replace previews by default; pass --save to persist; it is literal text find-replace (use --regex / --case-sensitive for substring or quoted-name cases) + - te find --in expressions covers measure DAX, calc columns, KPI expressions, partition M, role filters, and calc-group selection; it does NOT see report JSON + +pbir owns: + - report-layer references: visual queryState projections, filters, CF, slicer bindings + - pbir fields replace works per Table.Field; there is no table-level bulk rewrite, so a table rename is one replace per affected field (enumerate with pbir fields list first) + - pbir validate --fields resolves against the connected model; if the report is byPath it validates against the local TMDL, if byConnection against the workspace model (confirm with pbir model first) + - pbir add visual / visuals bind: pass -t Measure or the binding defaults to Column and fails at runtime + +Not covered by pbir fields replace (handle separately): + - extension measures in reportExtensions.json: inspect with pbir dax measures list / json; rename the object with pbir dax measures rename, but the DAX body must be re-authored manually + - visual calculations: locate with pbir dax viscalcs json and update the DAX separately + - bookmark data states: pbir validate --fields surfaces broken refs but does not repair captured slicer/filter state; re-test bookmarks after a rename + +Argument-order caveat: + - the pbir skill documents two forms for pbir fields find (report-first with -f, and search-term-first); run pbir fields find --help to confirm the build in use before scripting it +``` diff --git a/plugins/tabular-editor/skills/te-cli/references/semantic-modeling-practices.md b/plugins/tabular-editor/skills/te-cli/references/semantic-modeling-practices.md new file mode 100644 index 00000000..3bed35c7 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/references/semantic-modeling-practices.md @@ -0,0 +1,169 @@ +# Semantic modeling practices for `te` + +Companion to the `te-cli` skill. The skill teaches how to drive the CLI; this file teaches what to build with it. Each practice pairs a modeling decision with the `te` command that applies it, and cites the authoritative source (Microsoft Learn, SQLBI, or the Tabular Editor blog). Apply these after `te add` creates an object, and gate every batch with `te validate` and `te bpa run`. + +Three cross-cutting reminders that change every command below: + +- Mutations stage in memory by default. Pass `--save` to persist, or set `te config set interactiveEditMode save` for a build script. +- Each `Bash` call is a fresh shell, so `te connect` does not carry over. Pass `-m <path>` (and `-s`/`-d` for remote) on every command, or set `TE_SESSION=<name>` first. +- Property names are case-insensitive and match TOM. When unsure what a property is called or whether it is settable, run `te get <obj>` and `te set <obj> -q` first; those are the authoritative discovery tools. The less common properties below (`isAvailableInMDX`, `discourageImplicitMeasures`, `metadataPermission`, relationship `crossFilteringBehavior`) are real TOM properties but are not in the skill's documented `-q` list; confirm them on a real object before scripting them in a pipeline, or fall back to `te script` (TOM). + +## Dimensional design (star schema) + +Build a star: one fact table per business process at a single grain, surrounded by conformed dimensions joined dimension-to-fact by one-to-many relationships. The VertiPaq storage engine and the DAX engine are both optimized for this shape; dimensions filter and group, facts summarize, which is exactly how visuals generate DAX. Reserve a flat single-table model for small prototypes or a deliberate DirectQuery denormalization. + +| Practice | Why | `te` command | +|---|---|---| +| Model a star, not a flat table | flat models measured up to ~2.65x larger in RAM and up to 2x slower on complex queries; where flat wins it is marginal | inspect topology first: `te ls`, `te query -q "EVALUATE INFO.VIEW.RELATIONSHIPS()"` (relationships; `te ls` cannot list them), `te deps "Sales/Revenue"` | +| One consistent grain per fact table; never join fact-to-fact (header/detail) directly | mixing grains gives wrong aggregations; fact-to-fact becomes a limited relationship (SQLBI's 1.4B-row test: ~15x CPU, 17 minutes vs seconds for distinct count) | add a shared dimension and relate both facts to it rather than to each other | +| Flatten snowflake sub-dimensions into one dimension per business entity | extra hops cost CPU on every filter pass and a hierarchy cannot span tables; run-length encoding makes the duplication cheap. Keep a sub-dimension separate only when it is large and reused across parents | do the join in Power Query upstream, then relate the single dimension | +| Keep degenerate dimensions (order/invoice number) as hidden fact columns | a one-column table for a high-cardinality attribute adds size and clutter for no gain | `te set Sales/OrderNumber -q isHidden -i true --save` | +| Host model measures on a dedicated hidden measure table; prefix to sort it to the top | separates calculations from data and keeps the field list clean (organizational, not an engine requirement; below ~20 measures, display folders inside the source table are equally valid) | `te add _Measures -t Table --columns "_Measures:String" --partition-expression 'let Source = #table({"_Measures"}, {{""}}) in Source' --save`, then `te set _Measures -q isHidden -i true --save`, then `te mv Sales/Revenue "_Measures/Revenue" --save` | +| Use plain business names (Customer, Sales); drop DIM_/FACT_ prefixes | a table's role comes from cardinality, not its name; prefixes degrade Q&A and Copilot interpretation (organizational, bends to warehouse naming constraints) | `te mv FACT_Sales Sales --save` | + +Nuance to preserve: SQLBI shows a flat model can beat star for some simple low-cardinality groupings, so "star" is the default rather than a universal absolute; when flat wins it is marginal, when it loses it is far worse. + +Sources: [Understand star schema (MS Learn)](https://learn.microsoft.com/power-bi/guidance/star-schema), [The importance of star schemas (SQLBI)](https://www.sqlbi.com/articles/the-importance-of-star-schemas-in-power-bi/), [Star schema or single table (SQLBI)](https://www.sqlbi.com/articles/power-bi-star-schema-or-single-table/), [Header/Detail vs Star Schema (SQLBI)](https://www.sqlbi.com/articles/header-detail-vs-star-schema-models-in-tabular-and-power-bi/), [Naming conventions (Tabular Editor)](https://tabulareditor.com/blog/naming-conventions-for-power-bi-semantic-models). + +## Relationships + +Default every relationship to one-to-many cardinality and single cross-filter direction, flowing from the dimension to the fact. This is predictable, lets the storage engine pre-build join indexes, and avoids ambiguous filter paths. + +| Practice | Why | `te` command | +|---|---|---| +| Add the dimension-to-fact relationship before authoring measures that cross it | measures using `RELATED()` or cross-table `CALCULATE()` validate at save time and the gate rejects with `DAX0002` if the relationship is missing | `te add "Sales[ProductKey]->Product[ProductKey]" --save` | +| Keep single cross-filter direction; never set model-level bidirectional just to sync slicers | bidirectional on a model with 3+ related tables creates two propagation paths and subtly wrong numbers; scope the rare genuine need to one measure with `CROSSFILTER(..., BOTH)`. Since 2019 a visual-level "is not blank" measure filter replaces the slicer-sync use case | list relationships with `te query -q "EVALUATE INFO.VIEW.RELATIONSHIPS()"` (`te ls` cannot enumerate them; see gotchas), read one with `te get Relationships/<name>` (the `->` shorthand is for `te add` only); change `crossFilteringBehavior` only after confirming the property name with `te set <rel> -q`, or use `te script` | +| Treat many-to-many cardinality as a last resort; bridge dimension-to-dimension via two one-to-many relationships (exactly one leg bidirectional) | one-to-many gives a regular relationship with a blank row for RI violations; many-to-many cardinality gives a limited relationship with no join index, multiple passes, and silent drops of unmatched values | model the bridge with `te add` relationships rather than a direct many-to-many | +| When many-to-many is genuinely required, document that totals are non-additive | with shared membership the total counts each item once and will not equal the sum of visible rows; this is correct by design, the risk is silent misreading | add a `te set <measure> -q description` note on affected measures | +| Role-playing dimensions: duplicate the physical dimension with an active relationship per role; use one shared table with inactive relationships + `USERELATIONSHIP` only when simultaneous multi-role filtering is not needed | active relationships give correct drag-and-drop results with no DAX; inactive ones force a `USERELATIONSHIP` wrapper in every measure and break Q&A. Never use `USERELATIONSHIP` in a calculated column (use `LOOKUPVALUE`) | load the dimension twice in Power Query (Order Date, Ship Date), each with its own active relationship | +| Avoid one-to-one; merge the tables in Power Query and use display folders | one-to-one splits fields users expect together, blocks cross-table hierarchies, and adds blank rows | merge upstream, then `te set <col> -q displayFolder` to regroup | +| Prefer regular over limited relationships; validate referential integrity before deploy | regular relationships propagate in one engine pass; limited relationships force multiple passes and degrade above low-cardinality joins | `te validate -m ./model` and check for unmatched keys before `te deploy` | + +Sources: [Relationships (SQLBI)](https://www.sqlbi.com/articles/relationships-in-power-bi-and-tabular-models/), [Bi-directional guidance (MS Learn)](https://learn.microsoft.com/power-bi/guidance/relationships-bidirectional-filtering), [Bidirectional ambiguity (SQLBI)](https://www.sqlbi.com/articles/bidirectional-relationships-and-ambiguity-in-dax/), [Many-to-many guidance (MS Learn)](https://learn.microsoft.com/power-bi/guidance/relationships-many-to-many), [Regular and limited relationships (SQLBI)](https://www.sqlbi.com/articles/strong-and-weak-relationships-in-power-bi/), [Active vs inactive (MS Learn)](https://learn.microsoft.com/power-bi/guidance/relationships-active-inactive), [Ambiguous paths (Tabular Editor)](https://tabulareditor.com/blog/there-are-ambiguous-paths-in-power-bi). + +## VertiPaq, cardinality, and data types + +Column cardinality (distinct value count), not row count, is the primary driver of model size. VertiPaq stores a per-column dictionary plus a compressed per-row index, and compression is governed almost entirely by distinct values. Profile with `te vertipaq` and attack the highest "% of DB" columns first. + +| Practice | Why | `te` command | +|---|---|---| +| Profile by size and fix the biggest columns first | a high-cardinality column's dictionary alone can exceed 90% of its storage | `te vertipaq --columns --detail --top 20 -m ./model`; scope with `te vertipaq Sales` | +| Split a high-cardinality datetime into Date and Time parts (in Power Query, not a calculated column) | a sub-second datetime is near-unique; one real case went from 38.1% of DB to 0.3% (>99% reduction); a calculated column over the original reclaims nothing | fix in Power Query, then verify with `te vertipaq Sales/OrderDate` | +| Prefer narrow integer surrogate keys for relationship columns | the win is a smaller dictionary for the same distinct count, NOT an encoding switch (VertiPaq always hash-encodes relationship columns); SQLBI measured a relationship dropping from 4 MB to under 50 KB | `te set Sales/CustomerKey -q dataType -i int64 --save` | +| Use a Date data type (not integer YYYYMMDD) for the date key | SQLBI's 2B-row test found storage and scan essentially identical, so usability decides: Date enables native arithmetic and classic time intelligence | `te set Date/Date -q dataType -i dateTime --save` | +| Use Fixed Decimal (Currency) for monetary values; avoid Double/Single for aggregated columns | VertiPaq scans segments in parallel and floating-point addition is non-associative, so Double sums can vary between runs; Fixed Decimal is a deterministic scaled integer | `te set Sales/Amount -q dataType -i decimal --save` | +| Remove unused columns before deployment, not after | every imported column costs dictionary, data-segment, and attribute-hierarchy memory; removing one 50M-value identity column cut a model from 1.3 GB to 490 MB | `te deps Sales/SomeCol --downstream` to check, then `te rm Sales/SomeCol --dry-run` and `te rm Sales/SomeCol --if-exists --save` | +| Set `isAvailableInMDX` = false on high-cardinality hidden columns (keep it true for Sort By Column targets when MDX/Excel clients exist) | the per-column attribute hierarchy is pure overhead on hidden columns unreachable from MDX (one case: 1.1 GB of hierarchy) | confirm the property name first with `te set <col> -q`, then `te set Sales/Key -q isAvailableInMDX -i false --save` (or `te script`) | +| Reduce numeric precision to what analysis needs when the business agrees | fewer distinct values means smaller dictionaries; rounding loses auditability, so validate with owners and consider keeping a hidden full-precision copy | round in Power Query, then re-profile with `te vertipaq` | + +Nuance to preserve: integer-key superiority is about dictionary size, not a value-vs-hash encoding switch (the Tabular Editor blog rightly pushes back on "integer keys are mandatory" as relational dogma); state the mechanism correctly. The Date-vs-integer-key choice is usability-driven, not a performance win. + +Sources: [Optimizing high cardinality columns (SQLBI)](https://www.sqlbi.com/articles/optimizing-high-cardinality-columns-in-vertipaq/), [Data reduction techniques (MS Learn)](https://learn.microsoft.com/power-bi/guidance/import-modeling-data-reduction), [Optimizing semantic model size (Tabular Editor)](https://tabulareditor.com/blog/a-comprehensive-guide-to-optimizing-semantic-model-size-in-power-bi-and-fabric), [Date or Integer for dates (SQLBI)](https://www.sqlbi.com/articles/choosing-between-date-or-integer-to-represent-dates-in-power-bi-and-tabular/), [Choosing numeric data types (SQLBI)](https://www.sqlbi.com/articles/choosing-numeric-data-types-in-dax/), [Costs of relationships (SQLBI)](https://www.sqlbi.com/articles/costs-of-relationships-in-dax/). + +## Usability and AI-readiness metadata + +A model is consumer-ready and Copilot-ready only once measures and columns carry format strings, folders, descriptions, and the right summarization and visibility flags. Copilot and data agents read this metadata to ground DAX, so the same work that helps humans helps AI. + +| Practice | Why | `te` command | +|---|---|---| +| Set a format string on every visible measure and numeric/date column | unformatted values render raw and inconsistently; Copilot reads format strings when grounding DAX | `te set "_Measures/Revenue" -q formatString -i "#,0.00" --save` | +| Hide surrogate/foreign-key columns and set `summarizeBy` = none on numeric columns that should not aggregate | keys are scaffolding, not attributes; `none` removes the implicit-aggregation sigma so a key cannot be summed into nonsense, and Copilot/Q&A exclude hidden columns | `te set Sales/CustomerKey -q isHidden -i true --save` and `te set Sales/CustomerKey -q summarizeBy -i none --save` | +| Apply Sort By Column to display columns with non-alphabetical order (month, weekday, fiscal period) and hide the helper | without it "Month Name" sorts April, August, December; the sort column must be 1:1 with the display column | `te set Date/MonthName -q sortByColumn -i MonthNumber --save` then `te set Date/MonthNumber -q isHidden -i true --save` | +| Add descriptions to visible tables, columns, and measures; front-load the key guidance in the first 200 characters | descriptions feed Copilot/data-agent DAX (which reads the first 200 chars) and surface as service tooltips for humans | `te set "_Measures/Revenue" -q description -i "Net revenue after returns and discounts" --save` | +| Use human-readable, unique, business-aligned names; organize measures into display folders; avoid duplicate column names across tables | ambiguous or duplicated names make Copilot/Q&A misroute; folders keep a large field list navigable (purely cosmetic) | `te set "_Measures/Revenue" -q displayFolder -i "Revenue" --save`; qualify clashes as "Customer Name" vs "Store Name" with `te mv` | +| Mark `isKey` on dimension primary keys, then hide them | signals the unique grain the engine builds filters from, and keeps the field list clean | `te set Product/ProductKey -q isKey -i true --save` then `te set Product/ProductKey -q isHidden -i true --save` | +| Set data categories on geographic, URL, and image columns | enables map visuals, clickable links, and image rendering, and Copilot reads them for grounding | `te set Geo/City -q dataCategory -i City --save`, `te set Customer/Photo -q dataCategory -i ImageUrl --save` (confirm category values with `te get`) | + +For bulk metadata across many objects, prefer one `te script` pass over N `te set` calls; the model loads once, avoiding the ~1-2s startup per invocation: + +```bash +echo 'foreach (var m in Model.AllMeasures) if (string.IsNullOrEmpty(m.DisplayFolder)) m.DisplayFolder = "Uncategorized";' | te script -e - -m ./model --save +``` + +Sources: [Star schema - Measures (MS Learn)](https://learn.microsoft.com/power-bi/guidance/star-schema#measures), [Prepare a model for Copilot (MS Learn)](https://learn.microsoft.com/power-bi/create-reports/tutorial-copilot-power-bi-prepare-model), [Semantic model best practices for data agent (MS Learn)](https://learn.microsoft.com/fabric/data-science/semantic-model-best-practices), [Sort By Column side effects (SQLBI)](https://www.sqlbi.com/articles/side-effects-in-dax-of-the-sort-by-column-setting/), [Built-in BPA rules (Tabular Editor)](https://docs.tabulareditor.com/en/features/built-in-bpa-rules.html), [Naming conventions (Tabular Editor)](https://tabulareditor.com/blog/naming-conventions-for-power-bi-semantic-models). + +## Date tables and time intelligence + +Time intelligence needs one dedicated, shared Date dimension that all facts relate to, marked as a date table, with a contiguous date column at day grain spanning complete years. + +| Practice | Why | `te` command | +|---|---|---| +| Build one shared Date dimension; never drive time intelligence off a fact datetime column | classic time-intelligence needs a separate date spine to apply `REMOVEFILTERS` against, and one shared table lets a single slicer drive every fact | relate each fact date to `Date[Date]` with `te add "Sales[OrderDate]->Date[Date]" --save` | +| Mark the Date table (`dataCategory` = Time) and designate the date key | marking designates the date spine so time-intelligence functions (`DATESYTD`, `SAMEPERIODLASTYEAR`) know which column to clear and enumerate against, and it suppresses auto date/time; without it those functions intersect the current filter context and return silent wrong results | `te set Date -q dataCategory -i Time --save` then `te set Date/Date -q isKey -i true --save` | +| Remove auto date/time tables (`LocalDateTable_*`, `DateTableTemplate_*`) | auto date/time creates a hidden calculated table per date column, inflating size and refresh, cannot be shared, and is invisible to Excel/paginated/XMLA clients | find them with `te ls`, remove with `te rm <table> --save`; disable the toggle at the report/source so they do not regenerate on the next author save | +| Keep blank/null dates in the fact rather than inflating the Date table; guard comparisons with `NOT ISBLANK()` | blanks on the many side add a `(Blank)` slicer row and distort totals, and comparison operators treat `BLANK` as a pre-1900 date | author guarded measures with `te add`/`te set -q expression` | +| Standardize one org-wide Date definition (warehouse DimDate, dataflow, or parameterized DAX) | a shared definition prevents fiscal-year, week-numbering, and naming drift across models | import the warehouse DimDate where one exists | + +`te validate` checks structural and DAX validity but does not check date contiguity. Probe for gaps with a query: + +```bash +te query -q "EVALUATE ROW(\"Gap\", COUNTROWS(Date) - (MAX(Date[Date]) - MIN(Date[Date]) + 1))" -m ./model +``` + +A nonzero `Gap` means the date column is not contiguous. + +Sources: [Date tables guidance (MS Learn)](https://learn.microsoft.com/power-bi/guidance/model-date-tables), [Mark as Date table (SQLBI)](https://www.sqlbi.com/articles/mark-as-date-table/), [Auto date/time guidance (MS Learn)](https://learn.microsoft.com/power-bi/guidance/auto-date-time), [Blank in date columns (SQLBI)](https://www.sqlbi.com/articles/blank-in-date-columns-and-dax-time-intelligence-functions/). + +## Measures, calculated columns, and calculation groups + +Default to explicit DAX measures for all aggregated logic. Push row-level computation upstream, and use calculation groups to kill measure sprawl. + +| Practice | Why | `te` command | +|---|---|---| +| Author aggregations as explicit measures, not implicit sigma-column measures | measures are metadata-only and respond to filter context; implicit measures break MDX clients, let authors pick wrong aggregations, and cannot join calculation groups | `te add "_Measures/Total Sales" -t Measure -i "SUM(Sales[Amount])" --save` | +| Push row-level columns upstream: source/warehouse, then Power Query, then DAX calculated column, then measure | DAX calculated columns compress worse (SQLBI: ~4x larger in one case) because they skip the sort-order search and they extend refresh; Power Query columns can fold to the source | compute in Power Query; reserve DAX calculated columns for `RELATED`, `PATH`, or `COMBINEVALUES` keys | +| Remember calculated columns and tables are unsupported in DirectQuery and (as of mid-2026) Direct Lake | DirectQuery prohibits them (use `COMBINEVALUES` for multi-column keys); Direct Lake transcodes from Delta and does not materialize them, so push to the Lakehouse | for a calc column where unavoidable: `te add Sales/Bucket -t CalculatedColumn -i "<DAX>" --save` | +| Use calculation groups for time-intelligence or currency-conversion families with `SELECTEDMEASURE()`; set `discourageImplicitMeasures` = true | N base measures x M variants becomes N + M objects; calculation groups only fire on explicit measures, so discouraging implicit measures keeps them consistent | see the worked example below; confirm the `discourageImplicitMeasures` property name with `te set <group> -q`, then set via `te set` or `te script` | +| Scope items with `ISSELECTEDMEASURE()` rather than `SELECTEDMEASURENAME()` string comparison | `ISSELECTEDMEASURE` participates in rename fixup; a string literal silently breaks on rename | author the item DAX accordingly | +| Set a Format String Expression on any item that changes a measure's meaning (percentages, currency) | a YOY% item returning 0.12 displays "0" under a `#,##0` base format | `te set "Time Intelligence/YOY%" -q formatStringDefinition -i "\"0.0%\"" --save` | +| Set Precedence deliberately when multiple calculation groups coexist, and test with a trivial measure first | active items apply in precedence order (highest is outermost), which also decides whose format string wins | confirm the property name with `te set <group> -q` | +| Consider parameterized DAX UDF measures over calculation groups when calculations must be opt-in (tooltip pages, filter panes, flexible matrices) | calculation items apply to every measure in context and can corrupt non-numeric measures; UDF measures are called explicitly per measure | see [When DAX UDF measures beat calculation groups (Tabular Editor)](https://tabulareditor.com/blog/when-dax-udf-measures-beat-calculation-groups-for-time-intelligence) | + +Worked example, a Time Intelligence calculation group: + +```bash +te add "Time Intelligence" -t CalculationGroup -m ./model --save +te add "Time Intelligence/YTD" -t CalculationItem -i "CALCULATE(SELECTEDMEASURE(), DATESYTD('Date'[Date]))" -m ./model --save +te set "Time Intelligence/YTD" -q ordinal -i 0 --save +te set "Time Intelligence/YTD" -q formatStringDefinition -i "SELECTEDMEASUREFORMATSTRING()" --save +``` + +`expression`, `ordinal`, and `formatStringDefinition` are the documented `CalculationItem` properties. Confirm the calculation-item child-path form with `te ls "Time Intelligence"` after creating the group, before scripting many items in a pipeline. + +Sources: [Calculated columns and measures (SQLBI)](https://www.sqlbi.com/articles/calculated-columns-and-measures-in-dax/), [DAX calculated columns vs Power Query (SQLBI)](https://www.sqlbi.com/articles/comparing-dax-calculated-columns-with-power-query-computed-columns/), [Data reduction - custom columns (MS Learn)](https://learn.microsoft.com/power-bi/guidance/import-modeling-data-reduction#preference-for-custom-columns), [Calculation groups (MS Learn)](https://learn.microsoft.com/analysis-services/tabular-models/calculation-groups), [Understanding calculation groups (SQLBI)](https://www.sqlbi.com/articles/understanding-calculation-groups/), [Controlling format strings in calculation groups (SQLBI)](https://www.sqlbi.com/articles/controlling-format-strings-in-calculation-groups/). + +## Security and governance (RLS, OLS, BPA) + +Define roles, set permissions, attach filters, then validate and deploy. Prefer dynamic RLS, assign Entra ID groups rather than individuals, and treat BPA as the governance gate. + +```bash +te add Roles/RegionManagers -t Role -m ./model --save +te set Roles/RegionManagers -q modelPermission -i Read --save +# Dynamic filter: confirm the TablePermission add/path form with `te ls Roles/RegionManagers/TablePermissions` +te set Roles/RegionManagers/TablePermissions/Sales -q filterExpression -i "[Region] = USERPRINCIPALNAME()" --save +te validate -m ./model +te deploy ./model -s ws -d model --deploy-roles --deploy-role-members --force --ci github +``` + +| Practice | Why | +|---|---| +| Prefer dynamic RLS with `USERPRINCIPALNAME()` against a mapping table; assign Entra ID groups, not users | one dynamic role scales with master data; group membership changes in Entra ID without republishing. Use `USERPRINCIPALNAME` (not `USERNAME`, which returns `DOMAIN\user` in Desktop) | +| Filter on the dimension and let active relationships propagate; move the filter to the fact when the dimension exceeds ~131K rows or carries `USERELATIONSHIP` paths | small dimensions get a reusable bitmap index; above the threshold the cache never activates and dimension filtering becomes worse than fact filtering | +| One role per access tier; membership is additive (OR), not subtractive | a user in a `FALSE()` role and a `TRUE()` role sees everything; combining OLS and RLS for the same user across roles throws a query-time error, so keep them in one combined role | +| Give consumers only Viewer/Read; Admin/Member/Contributor and model-editor SPNs bypass RLS and OLS entirely | any Edit-level identity silently removes all security; keep ETL/XMLA identities separate and least-privileged | +| Avoid `LOOKUPVALUE` in RLS filters; propagate via relationships | `LOOKUPVALUE` in a security filter runs in the formula engine on every query and blocks storage-engine caching | +| Do not rely on `USERELATIONSHIP`/`CROSSFILTER` to override an RLS-carrying relationship; relocate the filter | RLS propagates only through active relationships, and the engine blocks `USERELATIONSHIP` on an RLS-carrying relationship; `TREATAS` workarounds need explicit semantic review | +| Set object-level security with `te`: per-role `metadataPermission` on a table or column (`= None` hides both the data and the object name). Use `te script` for TOM-level access if the property is not directly settable | DAX security targets tables and columns, not measures, so hiding a measure needs a sentinel-table workaround. Confirm the property name with `te set <obj> -q` before scripting it | +| Run BPA as a governance gate (Microsoft Analysis Services rule set as baseline + org rules); remember BPA detects whether roles exist, not whether they are correct | `te bpa run` loads rules from a URL and gates deploy on error-severity violations | + +```bash +te bpa run --rules https://raw.githubusercontent.com/microsoft/Analysis-Services/master/BestPracticeRules/BPARules.json --fail-on error --ci github -m ./model +``` + +Sources: [RLS guidance (MS Learn)](https://learn.microsoft.com/power-bi/guidance/rls-guidance), [Security cost (SQLBI)](https://www.sqlbi.com/articles/security-cost-in-analysis-services-tabular/), [RLS with inactive relationships (SQLBI)](https://www.sqlbi.com/articles/dax-limitations-with-inactive-relationships-and-row-level-security-rls/), [Object-level security (MS Learn)](https://learn.microsoft.com/analysis-services/tabular-models/object-level-security), [OLS in Power BI (Tabular Editor)](https://tabulareditor.com/blog/ols-in-power-bi), [RLS in Power BI semantic models (Tabular Editor)](https://tabulareditor.com/blog/row-level-security-in-power-bi-semantic-models), [Using the BPA (Tabular Editor)](https://docs.tabulareditor.com/common/using-bpa.html). + +The validate-then-BPA-then-format authoring loop and the per-save BPA speed knob are in SKILL.md; run those gates continuously, not only at deploy. + +## Scope notes + +These practices are grounded in the sources above. The research did not cover, and this file deliberately does not assert detailed guidance on: user-defined aggregation design (grain, agg-awareness precedence), incremental-refresh and partitioning strategy, composite/Direct Lake-plus-Import tuning, or absolute size thresholds at which star beats flat. For those, consult the live docs and verify against real VertiPaq measurements with `te vertipaq`. diff --git a/plugins/tabular-editor/skills/te-cli/references/te2-migration.md b/plugins/tabular-editor/skills/te-cli/references/te2-migration.md new file mode 100644 index 00000000..eb02a7a6 --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/references/te2-migration.md @@ -0,0 +1,52 @@ +# Migrating from TE2 (`TabularEditor.exe`) to `te` + +Companion to the te-cli skill (SKILL.md). + +## Migration from TE2 + +Activate TE2 compatibility three ways: + +```bash +mv te te2 && ./te2 Model.bim -S fix.csx -D server db -O # 1. binary rename +TE_COMPAT=te2 te Model.bim -S fix.csx -D server db -O # 2. env var +te Model.bim -S fix.csx -D server db -O # 3. auto-detect from flags +``` + +For the full mapping table (and an interactive single-flag lookup), run: + +```bash +te migrate # full table +te migrate -A # prompt for a TE2 flag, get equivalent +te migrate --output-format json # machine-readable for codemods +``` + +**Most-used mappings**: + +| TE2 flag | New CLI | +|---|---| +| `<file>` (positional) | `te <command> <path>` or `--model <path>` | +| `-S <file.csx>` / `-S "code"` | `te script -S <file>` / `-e "code"` | +| `-A <rules>` / `-AX <rules>` | `te bpa run --rules <rules>` (`-AX` = no model rules, which is default) | +| `-D <server> <db>` | `te deploy <model> -s <server> -d <db>` | +| `-O` | (default; overwrite) | +| `-C` | `--deploy-connections` | +| `-P` | `--deploy-partitions` | +| `-R` / `-M` / `-SHARED` | `--deploy-roles` / `--deploy-role-members` / `--deploy-shared-expressions` | +| `-FULL` | `--deploy-full` | +| `-X <file>` | `--xmla <file>` (use `-` for stdout) | +| `-V` / `-G` | `--ci vsts` (also `azdo`/`azure-devops`) / `--ci github` (also `gh`); on `validate`, `bpa run`, `deploy`, `script`, `test run` | +| `-T <file>` | `--trx <file>` | +| `-B <file>` | `te save -o <file> --serialization bim` | +| `-TMDL <dir>` | `te save -o <dir> --serialization tmdl` (default format) | +| `-F <dir>` | `te save -o <dir> --serialization te-folder` (or `--deploy-full` after `-D`) | +| `-Y` | `--deploy-partitions --skip-refresh-policy` | +| `-W` / `-E` | (default) | +| `-L <user> <pass>` (after `-D`) | `te auth login -u <id> -p <secret> -t <tenant>` (prefer env vars) | +| `-SC` | _Not yet implemented_ | + +**Behavioral differences from TE2**: +- `te deploy` runs BPA as pre-flight gate by default (TE2 didn't). `--skip-bpa` to disable, `--fix-bpa` to auto-fix. +- `te deploy` prompts for confirmation. CI must pass `--force`. +- All commands support `--output-format json` for machine-readable output. +- No `start /wait` wrapper needed on Windows; it's a normal console binary. + diff --git a/plugins/tabular-editor/skills/te-cli/references/workflows.md b/plugins/tabular-editor/skills/te-cli/references/workflows.md new file mode 100644 index 00000000..47487a8b --- /dev/null +++ b/plugins/tabular-editor/skills/te-cli/references/workflows.md @@ -0,0 +1,167 @@ +# `te` common workflows + +Multi-step recipes for the `te` CLI. Modeling-driven workflows (RLS roles, calculation groups, date tables) live in semantic-modeling-practices.md. + +## Common workflows + +### Build a new table with an M partition + +`te add <Table> -t Table` on a model with no provider data source creates an **MPartition** by default. `--partition-expression "<M>"` delivers the M to that partition. Combine with `--columns` for a one-shot data-bound table: + +```bash +te add Sales -t Table \ + --columns "OrderID:Int64,Amount:Decimal,OrderDate:DateTime" \ + --partition-expression "$(cat <<'EOF' +let + Source = Sql.Database("server", "db"), + Sales = Source{[Schema="dbo", Item="Sales"]}[Data] +in + Sales +EOF +)" -m ./model --save +``` + +Result: one table, one MPartition with the M in place, columns typed. `te get Sales/Partitions/Sales -q sourceType` returns `M`. + +**Variants:** + +```bash +# Columns + placeholder M (filled later via te set <Table>/Partitions/<Name> -q MExpression …) +te add Sales -t Table --columns "Id:Int64,Amount:Decimal" -m ./model --save + +# M only, no explicit column schema (columns auto-discovered at refresh from the M output) +te add Sales -t Table --partition-expression "<M>" -m ./model --save + +# Force M explicitly when the model has a legacy provider DS that would otherwise win +te add Sales -t Table --source-type m --columns "..." --partition-expression "<M>" -m ./model --save + +# Opt into a legacy Query partition on a modern model +te add LegacyT -t Table --source-type query -q query -i "SELECT * FROM dbo.Foo" -m ./model --save +``` + +**Inline-data partitions** (demo models, placeholder tables, calc-group hosts): use `#table({...}, {{...}})` inside a single-quoted heredoc. Example for a `_Measures` placeholder table: + +```bash +te add _Measures -t Table --columns "_Measures:String" \ + --partition-expression 'let Source = #table({"_Measures"}, {{""}}) in Source' \ + -m ./model --save +``` + +**Heuristic for `-q expression -i "<value>"`** (when neither `--source-type` nor `--partition-expression` is set): the M.Analyzer lexer tokenises the value and reports M if the first token is `(`, `[`, the `let` keyword, or an identifier starting with `#` (`#table`, `#date`, `#shared`, ...). Everything else (including bare identifiers and SQL-shaped strings) reports as Query, with a stderr hint pointing at `--source-type m` if the user meant M. Leading comments are skipped. + +**Pre-validation errors** (fail before mutation): +- `--source-type calculated` paired with `-t Table` → use `-t CalculatedTable` +- `--source-type m` on a model with a provider data source → remove the DS, or use `--source-type query` +- `--source-type m` on Compatibility Level < 1400 → upgrade the model +- `--source-type` combined with `--mode directlake` → DL/Entity partitions are picked automatically + +**Updating an existing partition's M** after creation: `te set Sales/Partitions/Sales -q MExpression -i "<M>" --save` (note `MExpression`, not `expression`, despite `te get` displaying the property as `expression`). + +### Convert TMDL ↔ BIM ↔ PBIP + +```bash +te save ./model.bim -o ./tmdl-out # BIM → TMDL folder +te save ./tmdl-folder -o ./model.bim --serialization bim # TMDL → BIM +te save ./model.bim -o ./project --serialization pbip --supporting-files # BIM → PBIP (.platform / definition.pbism) +``` + +### Deploy from local TMDL with BPA gate + CI annotations + +```bash +te deploy ./model \ + -s "powerbi://api.powerbi.com/v1.0/myorg/MyWorkspace" -d "MySemanticModel" \ + --force --ci github # BPA gate runs by default; pipeline fails on violations +``` + +For CI without strict BPA gating: add `--skip-bpa` (one-shot) or `te config set bpa.onDeploy false`. To auto-fix instead of failing: `--fix-bpa`. + +### Generate TMSL/XMLA without deploying + +```bash +te deploy ./model -s ws -d model --xmla deploy.tmsl +te deploy ./model -s ws -d model --xmla - > deploy.tmsl # to stdout +``` + +### Refresh single partition with dry-run safety + +```bash +te refresh --table Sales --partition "Sales.2024" --type full --dry-run > refresh.tmsl +# Review TMSL, then drop --dry-run to execute +``` + +### Find unused measures and remove with preview + +```bash +te deps --unused --hidden # discover candidates +te rm Sales/UnusedMeasure --dry-run # confirm impact +te rm Sales/UnusedMeasure --if-exists --save # idempotent removal +``` + +### Mirror remote workspace for local editing + +```bash +te connect MyWorkspace MyModel -w ./local-mirror # remote → local TMDL +# Edit locally, push commits, experiment with `te set`, `te add`, etc. +te save # intended to write to both source (remote) and mirror (local); verify with `te connect --help` before relying on bidirectional mirroring +``` + +### Bulk DAX format and BPA fix as one batch + +```bash +te format --save && te bpa run --fix --save +``` + +### Run a TE3 C# script against a remote model + +```bash +te script -S ./scripts/format-all-dax.csx -s ws -d model --save +echo "foreach (var t in Model.Tables) t.Name = t.Name.Replace(\"_\", \" \");" | te script -e - --save +``` + +### Snapshot + compare for regression testing + +```bash +te test snapshot # capture baseline +# … make changes … +te test compare # detect drift +``` + + +## Additional authoring workflows + +Modeling-driven recipes (mark a date table, calculation groups, RLS roles) live in semantic-modeling-practices.md, paired with the rationale for each. The recipes below are the remaining structural-object workflows. The `te` CLI is in preview; confirm any flag or path shape below with `te <command> --help` (or `te ls <container>` to see the exact child-path form) before scripting it in a pipeline. + +### Perspectives + +Perspectives are saved field-list views. Create the perspective, then add tables and objects to it through the `Perspectives/<perspective>/...` path. + +```bash +te add "Perspectives/Sales View" -t Perspective -m ./model --save +te add "Perspectives/Sales View/Sales" -m ./model --save # add the Sales table to the perspective +te add "Perspectives/Sales View/_Measures/Revenue" -m ./model --save # add a single measure +te ls "Perspectives/Sales View" # confirm membership +``` + +### Translations and cultures + +Translations live on a culture object; the per-object translated strings are bracket-indexed properties (`TranslatedNames[<culture>]`, `TranslatedDescriptions[<culture>]`). + +```bash +te add Cultures/fr-FR -t Culture -m ./model --save +te set "_Measures/Revenue" -q "TranslatedNames[fr-FR]" -i "Revenu" -m ./model --save +te set "_Measures/Revenue" -q "TranslatedDescriptions[fr-FR]" -i "Revenu net" -m ./model --save +``` + +### Incremental refresh setup + +`te incremental-refresh` manages the policy on a table (`show`, `set`, `remove`, `apply`). A policy requires the `RangeStart` and `RangeEnd` `NamedExpression` parameters in the model first; the partition M must filter on them. The exact flags for `set` (granularity, rolling/archive window, detect-data-changes) are not pinned in this skill, so read them from the binary before use: + +```bash +te incremental-refresh set --help # confirm the granularity / window / detect-changes flag names +te incremental-refresh show Sales -m ./model +te incremental-refresh apply Sales -m ./model # re-evaluate the policy, create/expand partitions +``` + +### Field parameters + +A field parameter is a `CalculatedTable` whose DAX uses the `NAMEOF(...)` pattern plus specific annotations that Power BI Desktop expects. Hand-authoring the exact DAX and annotations through `te add`/`te set` is error-prone; prefer the field-parameter macro in the `c-sharp-scripting` skill (run via `te macro run` or `te script`), then verify with `te get <Table> --output-format tmdl`. diff --git a/plugins/tabular-editor/skills/te-docs/SKILL.md b/plugins/tabular-editor/skills/te-docs/SKILL.md index 98cc6685..c93545eb 100644 --- a/plugins/tabular-editor/skills/te-docs/SKILL.md +++ b/plugins/tabular-editor/skills/te-docs/SKILL.md @@ -1,6 +1,6 @@ --- name: te-docs -version: 26.20 +version: 26.24 description: Tabular Editor documentation search and configuration file guidance (.tmuo, Preferences.json, UiPreferences.json, Layouts.json). Automatically invoke when the user asks about "TE docs", "Tabular Editor features", "TE3 preferences", ".tmuo files", "workspace database settings", "DAX editor settings", "per-model TE3 configuration", or needs to search Tabular Editor documentation for how-to guidance. --- diff --git a/plugins/tabular-editor/skills/te2-cli/SKILL.md b/plugins/tabular-editor/skills/te2-cli/SKILL.md index ed17e9d6..eb823804 100644 --- a/plugins/tabular-editor/skills/te2-cli/SKILL.md +++ b/plugins/tabular-editor/skills/te2-cli/SKILL.md @@ -1,6 +1,6 @@ --- name: te2-cli -version: 26.20 +version: 26.24 description: CLI syntax reference for Tabular Editor 2 (TabularEditor.exe); deployment, scripting, BPA analysis, and CI/CD integration. Automatically invoke when the user mentions "TabularEditor.exe", TE2 CLI flags (-D, -S, -A, -B, -TMDL, -O, -C), or asks to "deploy a model via CLI", "set up CI/CD for Power BI", "automate model deployment", "run BPA from command line", "save model as TMDL". --- @@ -8,6 +8,9 @@ description: CLI syntax reference for Tabular Editor 2 (TabularEditor.exe); depl Command-line interface for Tabular Editor 2 (TE2). This skill covers the `TabularEditor.exe` executable; TE3 has its own CLI and is a separate product. +> [!NOTE] +> The TE2 CLI (`TabularEditor.exe`) is still supported and stable, and remains a solid choice for existing Windows CI/CD pipelines. It is now considered outdated against the newer cross-platform Tabular Editor CLI (`te`), which is built on TE3 and is in preview until Q4 2026. For the `te` command set and a flag-by-flag migration, see the `te-cli` skill. Download `te` preview binaries from the [Tabular Editor CLI downloads page](https://tabulareditor.com/download-tabular-editor-cli). + ## Installation diff --git a/release-notes/v26.20.md b/release-notes/previous-releases/26.20.md similarity index 100% rename from release-notes/v26.20.md rename to release-notes/previous-releases/26.20.md diff --git a/release-notes/v26.24.md b/release-notes/v26.24.md new file mode 100644 index 00000000..154a6198 --- /dev/null +++ b/release-notes/v26.24.md @@ -0,0 +1,12 @@ +# v26.24 + +- Added a `te-cli` skill (tabular-editor plugin) for the cross-platform Tabular Editor CLI (`te`): grounded modeling guidance, a TE2-to-TE3 migration reference, and a vendor-sync script that pulls the upstream skill on merge +- Added a consolidated `semantic-model` skill (semantic-models plugin) spanning the design / build / refresh / review lifecycle and routing every change through a `te`-first cascade (`te` CLI, then a model MCP or TOM via `connect-pbid`, then TMDL via the `tmdl` skill); it replaces `review-semantic-model`, which was removed +- Power BI Desktop Bridge integration across the report and model skills. The bridge is Desktop's per-process JSON-RPC named pipe that reloads the report canvas and captures page screenshots. The `pbir-cli` skill drives it through the new `pbir desktop list/refresh/screenshot` verbs to close the loop locally (mutate, refresh, screenshot, inspect) with auto-refresh on save, absolute report paths, and live local-engine query routing for thick reports (new `references/desktop-integration.md`); `connect-pbid` pairs it with TOM to verify model edits against the rendered report, drives the pipe raw when `pbir` is absent, and documents port-to-report correlation via the msmdsrv process tree (new `references/desktop-bridge.md`); `pbir-format` covers the `powerbi-desktop` npm CLI for the same loop. Reload semantics verified empirically: `file.reload` picks up on-disk PBIR edits, but theme files and TMDL need a close and reopen +- `connect-pbid` script fixes: `create-field-parameter.ps1` built a non-functional field parameter through two nonexistent TOM APIs while reporting success (now uses `GroupByColumns` + `JsonExtendedProperty`, fails fast, and cleans up); `debug-dax.ps1` polls for async DAXEvaluationLog delivery instead of missing events; `daxlib.sh` errors clearly when `jq` is missing and supports `--help`; the quickstart dedupes ports and skips thin-report engines; `load-tmdl.ps1` warns when `-AddMeasure` runs without `-Save` +- pbi-desktop hooks: `refresh-cache` and `check-ri` now read executed `.ps1` contents (they never fired for the recommended `-File` workflow); the validate hooks no longer block the plugin's own bundled scripts; hook README synced to `hooks.json`. The pbip PBIR hook validates page `displayOption` against the schema enum and matches nested and absolute `.Report` paths +- `pbir-format`: repaired the bundled K201 example (phantom `pages.json` entries, dangling theme reference, stale bookmark sections and queryRef) in both plugin copies; resolved the `midColor` token contradiction (use `neutral`); refreshed schema-version drift notes +- Expanded the report skills (`review-report`, `pbi-report-design`, `create-pbi-report`, `deneb-visuals`, `svg-visuals`, `python-visuals`, `r-visuals`, `pbir-format`) with reference material on accessibility, performance, interactivity, chart selection, mobile layout, filter panes, theming, and PBIR shapes +- Refined the `dax` skill's performance-optimization, patterns, engine-internals, and model-optimization references (contributed by Justin Martin) +- Rebuilt the `tmdl-validate` binaries (v0.2.0) for linux-x64 and windows-x64; clarified TMDL indentation spec vs PBIP convention in the tmdl docs +- Added a release-age-bypass hook plus non-personal settings and statusline updates to `useful-stuff` diff --git a/scripts/bump_release_version.py b/scripts/bump_release_version.py new file mode 100644 index 00000000..26e07718 --- /dev/null +++ b/scripts/bump_release_version.py @@ -0,0 +1,39 @@ +"""Bump the lockstep release version across marketplace, plugins, and skills. + +Usage: python scripts/bump_release_version.py <old> <new> +""" + +import json +import pathlib +import re +import sys + +OLD, NEW = sys.argv[1], sys.argv[2] +root = pathlib.Path(__file__).resolve().parent.parent +changed = [] + +mp = root / ".claude-plugin" / "marketplace.json" +data = json.loads(mp.read_text(encoding="utf-8")) +if data["metadata"]["version"] == OLD: + text = mp.read_text(encoding="utf-8").replace(f'"version": "{OLD}"', f'"version": "{NEW}"') + mp.write_text(text, encoding="utf-8") + changed.append(mp) + +for pj in root.glob("plugins/*/.claude-plugin/plugin.json"): + text = pj.read_text(encoding="utf-8") + if f'"version": "{OLD}"' in text: + pj.write_text(text.replace(f'"version": "{OLD}"', f'"version": "{NEW}"'), encoding="utf-8") + changed.append(pj) + +for sk in root.glob("plugins/*/skills/*/SKILL.md"): + text = sk.read_text(encoding="utf-8") + new_text = re.sub( + rf"^version: {re.escape(OLD)}$", f"version: {NEW}", text, count=1, flags=re.MULTILINE + ) + if new_text != text: + sk.write_text(new_text, encoding="utf-8") + changed.append(sk) + +for path in changed: + print("bumped:", path.relative_to(root)) +print(f"total: {len(changed)} files {OLD} -> {NEW}") diff --git a/scripts/sync-te-cli-skill.sh b/scripts/sync-te-cli-skill.sh new file mode 100755 index 00000000..a5f5c06e --- /dev/null +++ b/scripts/sync-te-cli-skill.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Mirror the te-cli skill from its upstream home (github.com/TabularEditor/CLI) +# into this marketplace's tabular-editor plugin, and re-apply two local-only +# augmentations that upstream does not carry. +# +# What it does, every run: +# 1. Pull upstream SKILL.md + references/ (the skill is maintained upstream and +# vendored here; this stops the copy from drifting). +# 2. Preserve the local `version:` frontmatter (keeps the marketplace lockstep, +# all plugins/skills on the same YY.WW, untouched). +# 3. (Re)write references/get-te-cli.md -- a local-only reference telling the +# agent how to download the `te` binary per platform and put it on PATH. +# 4. Inject a one-line pointer to that reference into SKILL.md, just under the +# title. Both survive every sync because they are re-applied here. +# +# Upstream CHANGELOG/LICENSE/README are distribution metadata and are not vendored. +# Fail-open: no network / bad clone / missing upstream file leaves the current +# skill untouched and exits 0 (never blocks a git pull). +# +# Usage: +# scripts/sync-te-cli-skill.sh sync now +# scripts/sync-te-cli-skill.sh --write-local (re)write only the local-only +# get-te-cli.md + pointer, no network +# scripts/sync-te-cli-skill.sh --install-hook install a post-merge git hook so +# every `git pull` runs this sync +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DEST="$REPO_ROOT/plugins/tabular-editor/skills/te-cli" +UPSTREAM="https://github.com/TabularEditor/CLI.git" +SUBDIR="skills/te-cli" +GETREF="$DEST/references/get-te-cli.md" +POINTER='To get the `te` CLI yourself (as the agent), see [references/get-te-cli.md](references/get-te-cli.md).' + +write_get_ref() { + mkdir -p "$DEST/references" + cat > "$GETREF" <<'GETREF_EOF' +# Get the `te` CLI binary (agent self-service) + +The `te` (Tabular Editor CLI) binary is published per platform on a public CDN, +no account or auth needed: + + https://cdn.tabulareditor.com/files/cli/latest/te-<os>-<arch>.tar.gz (.zip on Windows) + +Pick the asset for the platform: + +- macOS arm64 (Apple Silicon): `te-osx-arm64.tar.gz` +- macOS x64 (Intel): `te-osx-x64.tar.gz` +- Linux x64: `te-linux-x64.tar.gz` +- Linux arm64: `te-linux-arm64.tar.gz` +- Windows x64: `te-win-x64.zip` +- Windows arm64: `te-win-arm64.zip` + +## macOS / Linux: detect, download, put on PATH + +```bash +os=$(uname -s | tr 'A-Z' 'a-z'); [ "$os" = darwin ] && os=osx +arch=$(uname -m); case "$arch" in arm64|aarch64) arch=arm64 ;; x86_64|amd64) arch=x64 ;; esac +mkdir -p "$HOME/.local/bin" +curl -fsSL "https://cdn.tabulareditor.com/files/cli/latest/te-$os-$arch.tar.gz" \ + | tar xz -C "$HOME/.local/bin" te +chmod +x "$HOME/.local/bin/te" +export PATH="$HOME/.local/bin:$PATH" # this shell +"$HOME/.local/bin/te" --version +``` + +Persist PATH across shells once (skip if `~/.local/bin` is already on PATH): + +```bash +echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bashrc # or ~/.zshrc +``` + +## Windows (PowerShell): download, put on PATH + +```powershell +$arch = if ($env:PROCESSOR_ARCHITECTURE -eq 'ARM64') { 'arm64' } else { 'x64' } +$dir = "$env:LOCALAPPDATA\Programs\te" +New-Item -ItemType Directory -Force -Path $dir | Out-Null +Invoke-WebRequest "https://cdn.tabulareditor.com/files/cli/latest/te-win-$arch.zip" -OutFile "$env:TEMP\te.zip" +Expand-Archive -Force "$env:TEMP\te.zip" -DestinationPath $dir +[Environment]::SetEnvironmentVariable('Path', "$([Environment]::GetEnvironmentVariable('Path','User'));$dir", 'User') +& "$dir\te.exe" --version +``` + +## Verify and stay current + +`te --version` prints the build. The CDN `latest` path is always the newest at +download time; for a binary that keeps itself current, use the self-updating +`te` wrapper instead (`te --update`, plus a daily check on `te --version`). +GETREF_EOF +} + +inject_pointer() { + # Idempotent: insert the pointer just after the first H1 if it is not already there. + grep -qF 'references/get-te-cli.md' "$DEST/SKILL.md" 2>/dev/null && return 0 + local t; t="$(mktemp)" || return 0 + awk -v p="$POINTER" ' + { print } + /^# / && !done { print ""; print p; done=1 } + ' "$DEST/SKILL.md" > "$t" && mv "$t" "$DEST/SKILL.md" +} + +install_hook() { + local hook="$REPO_ROOT/.git/hooks/post-merge" + mkdir -p "$(dirname "$hook")" + cat > "$hook" <<EOF +#!/usr/bin/env bash +# Auto-installed by scripts/sync-te-cli-skill.sh. Mirrors the te-cli skill from +# TabularEditor/CLI after every git pull. Safe to delete; re-add with +# 'scripts/sync-te-cli-skill.sh --install-hook'. +exec "$REPO_ROOT/scripts/sync-te-cli-skill.sh" +EOF + chmod +x "$hook" + printf 'installed post-merge hook -> %s\n' "$hook" +} + +case "${1:-}" in + --install-hook|install-hook) install_hook; exit 0 ;; + --write-local) write_get_ref; inject_pointer; printf 'wrote local-only get-te-cli.md + SKILL pointer\n'; exit 0 ;; + -h|--help) sed -n '2,33p' "$0"; exit 0 ;; +esac + +command -v git >/dev/null 2>&1 || { printf 'sync-te-cli: git not found; skipping\n' >&2; exit 0; } +[ -f "$DEST/SKILL.md" ] || { printf 'sync-te-cli: %s missing; skipping\n' "$DEST/SKILL.md" >&2; exit 0; } + +tmp="$(mktemp -d "${TMPDIR:-/tmp}/te-cli-sync.XXXXXX")" || exit 0 +trap 'rm -rf "$tmp"' EXIT + +if ! git clone --depth 1 --quiet "$UPSTREAM" "$tmp" 2>/dev/null; then + printf 'sync-te-cli: upstream clone failed (offline?); keeping current skill\n' >&2; exit 0 +fi +SRC="$tmp/$SUBDIR" +{ [ -f "$SRC/SKILL.md" ] && [ -d "$SRC/references" ] && ls "$SRC/references/"*.md >/dev/null 2>&1; } || { + printf 'sync-te-cli: upstream %s incomplete; aborting\n' "$SUBDIR" >&2; exit 0; } + +# Preserve this repo's version frontmatter (the lockstep value). +ver="$(awk -F': *' '/^version:/{print $2; exit}' "$DEST/SKILL.md")" +ver="${ver:-0.0.0}" + +cp "$SRC/SKILL.md" "$DEST/SKILL.md" +mkdir -p "$DEST/references" +# Drop upstream-managed references (keep the local-only one), then copy upstream's. +find "$DEST/references" -maxdepth 1 -name '*.md' ! -name 'get-te-cli.md' -delete 2>/dev/null +cp "$SRC/references/"*.md "$DEST/references/" + +# Re-stamp the preserved version onto the first frontmatter `version:` line. +VER="$ver" perl -pi -e 'if(!$d && s/^version:.*$/version: $ENV{VER}/){$d=1}' "$DEST/SKILL.md" + +# Re-apply the local-only augmentations (upstream does not carry these). +write_get_ref +inject_pointer + +if git -C "$REPO_ROOT" diff --quiet -- plugins/tabular-editor/skills/te-cli 2>/dev/null; then + printf 'sync-te-cli: up to date with upstream (version kept at %s)\n' "$ver" +else + printf 'sync-te-cli: te-cli skill updated from upstream (version kept at %s); review and commit:\n' "$ver" + git -C "$REPO_ROOT" --no-pager diff --stat -- plugins/tabular-editor/skills/te-cli +fi +exit 0 diff --git a/useful-stuff/README.md b/useful-stuff/README.md index 9bd72fd5..62c1304a 100644 --- a/useful-stuff/README.md +++ b/useful-stuff/README.md @@ -8,7 +8,8 @@ Misc. useful shit for making Claude Code more effective, safe, or satisfying bas | [`hooks/block-pip`](hooks/block-pip/) | Block `pip`/`pip3`, suggest `uv` | | [`hooks/block-destructive-commands`](hooks/block-destructive-commands/) | Block `rm -rf ~/`, force push to main, `git reset --hard`, `chmod 777` | | [`hooks/block-secrets-exposure`](hooks/block-secrets-exposure/) | Block .env reads, keychain/keyring access, cloud CLI token extraction | +| [`hooks/block-release-age-bypass`](hooks/block-release-age-bypass/) | Block `--no-minimum-release-age` / `--minimum-release-age=0` bypass flags on bun/npm | | [`agent-scripts/`](agent-scripts/) | Helper scripts for installing and running agentic tooling (e.g. enabling Windows long paths so the Copilot CLI install doesn't blow up on `Filename too long`) | -| [`status-lines/`](status-lines/) | Claude Code statusline with host + cwd, git, model + effort, time, and usage meters; segmented for easy customization | -| [`agent-settings/`](agent-settings/) | Sanitized `~/.claude/settings.json` template with the five Bash safety hooks (rm -rf home, npm, pip, ssh, op read) and opinionated defaults | +| [`status-lines/`](status-lines/) | Two-line Claude Code statusline (version, host + cwd, git, vim mode, model + effort, time, usage meters); segmented for easy customization | +| [`agent-settings/`](agent-settings/) | Sanitized `~/.claude/settings.json` template with the six Bash safety hooks (rm -rf home, npm, pip, ssh, op read, release-age bypass) and opinionated defaults | | [`package-cooldowns/`](package-cooldowns/) | One-shot setup script that configures uv, bun, pnpm, npm, and pip to ignore packages released in the last 7 days. Reduces supply-chain blast radius | diff --git a/useful-stuff/agent-settings/README.md b/useful-stuff/agent-settings/README.md index d152b184..1275fabb 100644 --- a/useful-stuff/agent-settings/README.md +++ b/useful-stuff/agent-settings/README.md @@ -19,8 +19,10 @@ A reference `~/.claude/settings.json` for projects or user settings. | `verbose: false`, `showTurnDuration: true` | Slim transcript with turn timings | | `agentPushNotifEnabled: true` | Push notifications when an agent finishes a background task | | `voiceEnabled: true` | Voice input on | -| `statusLine` | Points at `~/.claude/statusline.sh`; pair this with `useful-stuff/status-lines/` | -| `hooks.PreToolUse[Bash]` | Five Bash safety hooks; see below | +| `skillListingMaxDescChars`, `skillListingBudgetFraction` | Tuning for how much of the skill-listing context budget descriptions may use | +| `autoUpdatesChannel: "stable"`, `tui: "fullscreen"`, `skipWorkflowUsageWarning: true` | Stable update channel, fullscreen TUI, and skip the workflow-usage warning | +| `statusLine` (with `hideVimModeIndicator: true`) | Points at `~/.claude/statusline.sh`; pair with `useful-stuff/status-lines/`. Hides Claude Code's built-in vim indicator since the statusline shows its own | +| `hooks.PreToolUse[Bash]` | Six Bash safety hooks; see below | ## Example Bash safety hooks @@ -33,6 +35,7 @@ These are some example hooks that I use for safety purposes. They aren't a guara | `pip` / `pip3` | Any command that invokes `pip` or `pip3` | Nudge toward `uv`; remove this if you use `pip` directly | | `ssh` / `scp` / `sftp` | Any of those three | If your agent should never reach out over the network with the user's keys, leave this on. Remove if you want agent-driven remote management | | `op read` | The `op read <vault-uri>` form (matches `op://` literals) | Forces secret access through `op run --env-file=... -- <cmd>` so the secret value never lands in a tool result or transcript. There are other better ways to do this using i.e. OneCLI to facilitate use of credentials only at runtime. | +| release-age bypass | `--no-minimum-release-age`, `--minimum-release-age=0`, `--ignore-min-release-age` on bun/npm | Stops the agent disabling the freshly-published-package guard (supply-chain risk). See `hooks/block-release-age-bypass/` | ## What's deliberately not here @@ -40,7 +43,7 @@ Things in the original settings.json that were personal and have been stripped: - `model`: pinned default model. Set whatever you want - `EDITOR` / `VISUAL`: editor command -- Sound hooks (SessionStart, Notification, PreCompact, PostToolUse-on-Bash-stderr): all point at local Peon WoW-voice scripts that aren't in this repo. Add your own if you want audio cues +- Sound hooks (SessionStart, Notification, PreCompact, PostToolUse-on-Bash-stderr): all point at local audio scripts that aren't in this repo. Add your own if you want audio cues - `theme`, `editorMode`, `preferredNotifChannel`: personal UI choices - `enabledPlugins` and `extraKnownMarketplaces`: per-developer plugin selection - `spinnerVerbs`: cosmetic diff --git a/useful-stuff/agent-settings/settings.json b/useful-stuff/agent-settings/settings.json index 5eafe5ff..992ff82e 100644 --- a/useful-stuff/agent-settings/settings.json +++ b/useful-stuff/agent-settings/settings.json @@ -42,6 +42,11 @@ "type": "command", "command": "input=$(cat); cmd=$(echo \"$input\" | jq -r '.tool_input.command // empty'); if echo \"$cmd\" | grep -qE 'op[[:space:]]+read'; then jq -n '{hookSpecificOutput:{hookEventName:\"PreToolUse\",permissionDecision:\"deny\",permissionDecisionReason:\"op read is blocked to prevent credential exposure. Use op run to inject secrets instead.\"}}' && exit 0; fi; exit 0", "if": "Bash(*op://*)" + }, + { + "type": "command", + "command": "input=$(cat); cmd=$(echo \"$input\" | jq -r '.tool_input.command // empty'); if echo \"$cmd\" | grep -qE -- '--no-minimum-release-age|--minimum-release-age[= ]+0|--ignore-min-release-age'; then jq -n '{hookSpecificOutput:{hookEventName:\"PreToolUse\",permissionDecision:\"deny\",permissionDecisionReason:\"BLOCKED: --no-minimum-release-age (or equivalent --minimum-release-age=0) bypasses bun/npm protection against freshly-published packages (supply-chain risk). Wait for the release age window, or download the tarball directly with curl from registry.npmjs.org and inspect it.\"}}' && exit 0; fi; exit 0", + "if": "Bash(*minimum-release-age*)" } ] } @@ -50,7 +55,8 @@ "statusLine": { "type": "command", "command": "bash ~/.claude/statusline.sh", - "refreshInterval": 60 + "refreshInterval": 60, + "hideVimModeIndicator": true }, "feedbackSurveyRate": 0, "spinnerTipsEnabled": false, @@ -65,5 +71,10 @@ "todoFeatureEnabled": false, "agentPushNotifEnabled": true, "skipAutoPermissionPrompt": true, - "voiceEnabled": true + "voiceEnabled": true, + "skillListingMaxDescChars": 9001, + "skillListingBudgetFraction": 0.05, + "autoUpdatesChannel": "stable", + "tui": "fullscreen", + "skipWorkflowUsageWarning": true } diff --git a/useful-stuff/hooks/block-release-age-bypass/README.md b/useful-stuff/hooks/block-release-age-bypass/README.md new file mode 100644 index 00000000..26b8ca2f --- /dev/null +++ b/useful-stuff/hooks/block-release-age-bypass/README.md @@ -0,0 +1,18 @@ +# Block release-age bypass flags + +A PreToolUse hook that blocks `bun`/`npm` flags which disable the minimum-release-age guard: `--no-minimum-release-age`, `--minimum-release-age=0` (or `--minimum-release-age 0`), and `--ignore-min-release-age`. + +## Why + +- **Supply-chain protection.** A minimum release age (e.g. only install packages published more than N days ago) is one of the strongest defenses against a freshly-compromised package version: malicious releases are usually caught and yanked within hours to days. Bypassing it reinstates that window of exposure. +- **Agent safety.** Agents auto-approve installs. If an agent can pass `--no-minimum-release-age` to get past a blocked install, the protection is worthless. This hook removes that escape hatch so the agent has to wait out the age window or inspect the tarball by hand instead. + +## How it works + +The hook fires when the Bash command contains `minimum-release-age` (the `if` condition), then `grep` narrows to the actual bypass forms before emitting a `permissionDecision: "deny"`. Legitimate uses that set a non-zero age are not blocked. + +The deny message points the agent at the safe alternatives: wait for the release-age window, or download the tarball directly from `registry.npmjs.org` with `curl` and inspect it. + +## Installation + +Copy the hook entry into your `~/.claude/settings.json` under `hooks.PreToolUse`. See `settings.json.example` for the full structure. If you already have a `PreToolUse` matcher for `Bash`, add the hook object to the existing `hooks` array rather than creating a duplicate matcher. Pair it with the `block-npm` and `block-pip` hooks for a fuller package-manager policy. diff --git a/useful-stuff/hooks/block-release-age-bypass/hook.json b/useful-stuff/hooks/block-release-age-bypass/hook.json new file mode 100644 index 00000000..4d028c1b --- /dev/null +++ b/useful-stuff/hooks/block-release-age-bypass/hook.json @@ -0,0 +1,5 @@ +{ + "type": "command", + "command": "input=$(cat); cmd=$(echo \"$input\" | jq -r '.tool_input.command // empty'); if echo \"$cmd\" | grep -qE -- '--no-minimum-release-age|--minimum-release-age[= ]+0|--ignore-min-release-age'; then jq -n '{hookSpecificOutput:{hookEventName:\"PreToolUse\",permissionDecision:\"deny\",permissionDecisionReason:\"BLOCKED: --no-minimum-release-age (or equivalent --minimum-release-age=0) bypasses bun/npm protection against freshly-published packages (supply-chain risk). Wait for the release age window, or download the tarball directly with curl from registry.npmjs.org and inspect it.\"}}' && exit 0; fi; exit 0", + "if": "Bash(*minimum-release-age*)" +} diff --git a/useful-stuff/hooks/block-release-age-bypass/settings.json.example b/useful-stuff/hooks/block-release-age-bypass/settings.json.example new file mode 100644 index 00000000..7edb34fd --- /dev/null +++ b/useful-stuff/hooks/block-release-age-bypass/settings.json.example @@ -0,0 +1,16 @@ +{ + "hooks": { + "PreToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "input=$(cat); cmd=$(echo \"$input\" | jq -r '.tool_input.command // empty'); if echo \"$cmd\" | grep -qE -- '--no-minimum-release-age|--minimum-release-age[= ]+0|--ignore-min-release-age'; then jq -n '{hookSpecificOutput:{hookEventName:\"PreToolUse\",permissionDecision:\"deny\",permissionDecisionReason:\"BLOCKED: --no-minimum-release-age (or equivalent --minimum-release-age=0) bypasses bun/npm protection against freshly-published packages (supply-chain risk). Wait for the release age window, or download the tarball directly with curl from registry.npmjs.org and inspect it.\"}}' && exit 0; fi; exit 0", + "if": "Bash(*minimum-release-age*)" + } + ] + } + ] + } +} diff --git a/useful-stuff/status-lines/README.md b/useful-stuff/status-lines/README.md index 12f12e2b..80625c3a 100644 --- a/useful-stuff/status-lines/README.md +++ b/useful-stuff/status-lines/README.md @@ -1,6 +1,6 @@ # Statusline -A Claude Code statusline script with five segments: host + cwd, git status, model + effort, time, and three usage meters. Each segment is a separate file under `statusline.d/` and is sourced in numeric order, so you can edit one without touching the rest. +A Claude Code statusline script laid out over two lines (line 1: time, host + cwd, git; line 2: version, vim mode, model + effort, usage meters). Each segment is a separate file under `statusline.d/` and is sourced in numeric order, so you can edit one without touching the rest. ## Layout @@ -8,11 +8,13 @@ A Claude Code statusline script with five segments: host + cwd, git status, mode status-lines/ ├── statusline.sh # entrypoint; reads JSON from Claude Code on stdin, sources segments ├── statusline.d/ -│ ├── 02-host-cwd.sh # host-colored cwd -│ ├── 03-git.sh # branch, change count, current PR number +│ ├── 01-version.sh # Claude Code version (off by default) +│ ├── 02-host-cwd.sh # host-colored cwd with a per-language repo glyph +│ ├── 03-git.sh # branch, change count, PR number (GitHub + Azure DevOps), worktree │ ├── 04-model.sh # model name, version, effort dots +│ ├── 04a-vim.sh # vim mode indicator (when editorMode is vim) │ ├── 05-time.sh # HH:MM -│ └── 06-meters.sh # context %, 5-hour rate %, 7-day rate % +│ └── 06-meters.sh # context %, 5-hour and 7-day rate % with projection └── README.md ``` @@ -20,11 +22,13 @@ status-lines/ | Segment | Output | |---|---| -| host-cwd | hostname + current directory, colored per host. `$HOME` collapses to the hostname so paths read as `host/project/...` | -| git | `<branch> <+adds> <-deletes>` plus the current PR number if `gh pr view` resolves one. Untracked files count as adds. `not tracking` when not in a repo | +| version | Claude Code version. Off by default; enable with `ENABLE_VERSION=TRUE` | +| host-cwd | hostname + current directory, colored per host, with a per-language glyph for git repos. `$HOME` collapses to the hostname so paths read as `host/project/...` | +| git | `<branch> <+adds> <-deletes>` plus the current PR number, resolved from GitHub (Claude Code provides it) or Azure DevOps (via `az repos pr list`, cached on disk). Worktree-aware. Untracked files count as adds. `not tracking` when not in a repo | +| vim | current vim mode (NORMAL / INSERT / VISUAL), colored lualine-style. Empty unless `editorMode` is `vim` | | model | NerdFonts robot icon plus family name (Opus / Sonnet / Haiku). Older releases keep their version suffix; the family-latest hides it. Effort dots are calibrated per family | | time | `HH:MM` | -| meters | Context window %, 5-hour rate-limit %, 7-day rate-limit %. Each colored by threshold (dim / yellow / orange / red / maroon) | +| meters | Context window %, 5-hour and 7-day rate-limit % with a linear projection to cycle end. Each colored by threshold (dim / yellow / orange / red / maroon) | ## Install @@ -53,6 +57,8 @@ ENABLE_GIT=TRUE ENABLE_MODEL=TRUE ENABLE_TIME=TRUE ENABLE_METERS=TRUE +ENABLE_VERSION=FALSE +ENABLE_VIM=TRUE ``` Set any to anything other than `TRUE` and the segment is skipped. Drop new segment files into `statusline.d/` named `<NN>-<name>.sh` and wire a `load_segment` call at the bottom of `statusline.sh` to add your own. @@ -98,7 +104,7 @@ Effort calibration lives in the model case-statement. Opus 4.7 has 5 levels (low ## Requirements -- `bash`, `jq`, `git`, `gh` (for PR detection; the segment degrades silently if missing) +- `bash`, `jq`, `git`, `gh` (GitHub PR detection); optionally `az` for Azure DevOps PR detection. The git segment degrades silently if any are missing - A Nerd Font in your terminal for the robot icon and branch glyph (JetBrainsMono NF 3.4.0 was used during development) - `timeout` (Linux) or `gtimeout` (macOS via coreutils); falls back to no-timeout if neither is available - On Windows, `cygpath` is used to POSIX-ify the cwd if available diff --git a/useful-stuff/status-lines/statusline.d/01-version.sh b/useful-stuff/status-lines/statusline.d/01-version.sh new file mode 100644 index 00000000..7d00926b --- /dev/null +++ b/useful-stuff/status-lines/statusline.d/01-version.sh @@ -0,0 +1,8 @@ +# Claude Code version -- leftmost segment on line 2 so the model segment +# starts at roughly the same column as the cwd on line 1. +# Glyph: nf-fa-tag (U+F02B) -- placeholder will be substituted by the +# theme/install script. +version=$(echo "$input" | jq -r '.version // empty' 2>/dev/null) +if [ -n "$version" ]; then + seg " ${version}" +fi diff --git a/useful-stuff/status-lines/statusline.d/02-host-cwd.sh b/useful-stuff/status-lines/statusline.d/02-host-cwd.sh index 93db6a67..7aadf464 100644 --- a/useful-stuff/status-lines/statusline.d/02-host-cwd.sh +++ b/useful-stuff/status-lines/statusline.d/02-host-cwd.sh @@ -1 +1,94 @@ -seg "${host_color} ${dir}${R}" +# Leading icon: language glyph for git repos, folder icon elsewhere. +# Always in host colour to stay visually unified with the path. +# Detection cached in $TMPDIR/sl-lang and invalidated by .git/HEAD or +# .git/index mtime change. Hot path ~2ms; cold path capped at first 2000 +# tracked files (~50ms worst case). +icon=$'\xef\x81\xbb' # U+F07B nf-fa-folder (fallback) + +repo_root=$(git -C "$cwd" rev-parse --show-toplevel 2>/dev/null) +if [ -n "$repo_root" ]; then + cache_dir="${TMPDIR:-/tmp}/sl-lang" + [ -d "$cache_dir" ] || mkdir -p "$cache_dir" + cache_file="$cache_dir/${repo_root//\//_}" + head_file="$repo_root/.git/HEAD" + index_file="$repo_root/.git/index" + + mt=$(stat -f '%m' "$cache_file" "$head_file" "$index_file" 2>/dev/null \ + || stat -c '%Y' "$cache_file" "$head_file" "$index_file" 2>/dev/null) + cm=$(echo "$mt" | sed -n '1p'); hm=$(echo "$mt" | sed -n '2p'); im=$(echo "$mt" | sed -n '3p') + : "${cm:=0}" "${hm:=0}" "${im:=0}" + + if [ ! -f "$cache_file" ] || [ "$cm" -lt "$hm" ] || [ "$cm" -lt "$im" ]; then + git -C "$repo_root" ls-files 2>/dev/null | head -2000 | awk -F. ' + NF>1 { + ext = tolower($NF) + if (ext ~ /^(png|jpg|jpeg|gif|svg|ico|webp|bmp|tiff|woff|woff2|ttf|otf|eot|wav|mp3|mp4|mov|m4a|ogg|flac|pdf|zip|tar|gz|bz2|xz|7z|bin|exe|dll|dylib|so|class|jar|pyc|pyo|lock|map|min|snap|xsd|xml)$/) next + counts[ext]++ + } + END { + top = ""; max = 0 + for (e in counts) if (counts[e] > max) { max = counts[e]; top = e } + print top + } + ' > "$cache_file" + fi + + lang=$(< "$cache_file") + + # nf-md (Material Design) language glyphs -- chunkier than seti. + # Codepoints above U+FFFF, so 4-byte UTF-8; bash hex escapes for ASCII-safe source. + case "$lang" in + md) icon=$'\xf3\xb0\x8d\x94' ;; + py) icon=$'\xf3\xb0\x8c\xa0' ;; + rs) icon=$'\xf3\xb1\x98\x97' ;; + ts|tsx) icon=$'\xf3\xb0\x9b\xa6' ;; + js|jsx|mjs|cjs) icon=$'\xf3\xb0\x8c\x9e' ;; + go) icon=$'\xf3\xb0\x9f\x93' ;; + lua) icon=$'\xf3\xb0\xa2\xb1' ;; + c|h) icon=$'\xf3\xb0\x99\xb1' ;; + cpp|cc|cxx|hpp) icon=$'\xf3\xb0\x99\xb2' ;; + cs) icon=$'\xf3\xb0\x8c\x9b' ;; + swift) icon=$'\xf3\xb0\x9b\xa5' ;; + kt|kts) icon=$'\xf3\xb0\xae\xaa' ;; + html|htm) icon=$'\xf3\xb1\x8f\x9c' ;; + css|scss|sass) icon=$'\xf3\xb1\x8f\x80' ;; + sh|bash|zsh) icon=$'\xf3\xb1\x86\x83' ;; + rb) icon=$'\xf3\xb0\xb4\xad' ;; + json) icon=$'\xee\xac\x8f' ;; + yaml|yml) icon=$'\xee\x9a\xa8' ;; + toml|conf|ini) icon=$'\xee\x98\x95' ;; + ps1|psm1) icon=$'\xee\xaf\x87' ;; + sql) icon=$'\xf3\xb1\xad\x98' ;; + esac +fi + +# URL-encode for OSC 8 hyperlinks. Keeps the unreserved set + slashes intact and +# percent-encodes the rest, so paths with spaces or # survive the file:// URL. +url_encode() { + local s="$1" i c result="" hex + for ((i=0; i<${#s}; i++)); do + c="${s:i:1}" + case "$c" in + [a-zA-Z0-9/._~-]) result+="$c" ;; + *) printf -v hex '%%%02X' "'$c"; result+="$hex" ;; + esac + done + printf '%s' "$result" +} + +# Worktree mode: when an anthropic-managed worktree session is active, replace +# the cwd display with the worktree triple (path · name · branch) in orange so +# isolation is visually obvious. 03-git.sh suppresses the branch in this mode. +# Both modes wrap the path in an OSC 8 hyperlink; alacritty's hint handler +# (scripts/alacritty-url-open.sh) opens it in `alacritty -e nvim` on click. +if [ -n "$wt_path" ]; then + wt_glyph=$'\xf3\xb0\x99\x85' # U+F0645 nf-md-file_tree + wt_display="$wt_path" + [ -n "$wt_name" ] && wt_display="$wt_display${DIM} · ${ORANGE}$wt_name" + [ -n "$wt_branch" ] && wt_display="$wt_display${DIM} · ${ORANGE}$wt_branch" + osc_url=$(url_encode "$wt_path") + seg "${ORANGE}${wt_glyph} \033]8;;file://${osc_url}\a${wt_display}\033]8;;\a${R}" +else + osc_url=$(url_encode "$cwd") + seg "${host_color}${icon} \033]8;;file://${osc_url}\a${dir}\033]8;;\a${R}" +fi diff --git a/useful-stuff/status-lines/statusline.d/03-git.sh b/useful-stuff/status-lines/statusline.d/03-git.sh index ce84fd69..eebdbc70 100644 --- a/useful-stuff/status-lines/statusline.d/03-git.sh +++ b/useful-stuff/status-lines/statusline.d/03-git.sh @@ -3,28 +3,239 @@ branch=$(cd "$cwd" 2>/dev/null && git branch --show-current 2>/dev/null) if [ -z "$branch" ]; then seg "${DIM}not tracking${R}" else - seg " $branch" + [ -z "$wt_path" ] && seg " $branch" + # PR detection. + # + # GitHub: Claude Code populates input.pr.{number,review_state} for us (free). + # Azure DevOps: not provided by Claude; we detect via the origin URL and + # query `az repos pr list`. Result is cached on disk so we don't pay the + # ~2s az roundtrip on every statusline render. + if [ -z "$pr_number" ]; then + remote_url=$(cd "$cwd" 2>/dev/null && git remote get-url origin 2>/dev/null) + if [ -n "$remote_url" ]; then + CACHE_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/statusline" + PR_CACHE_TTL=60 + cache_key=$(printf '%s|%s' "$remote_url" "$branch" | sha1sum 2>/dev/null | cut -d' ' -f1) + cache_file="$CACHE_DIR/pr_${cache_key}" + + # Use cached result if fresh. stat -c is Linux, -f %m is BSD/Mac. + if [ -f "$cache_file" ]; then + mtime=$(stat -c %Y "$cache_file" 2>/dev/null || stat -f %m "$cache_file" 2>/dev/null) + if [ -n "$mtime" ]; then + age=$(( $(date +%s) - mtime )) + if [ "$age" -lt "$PR_CACHE_TTL" ]; then + # shellcheck disable=SC1090 + . "$cache_file" + fi + fi + fi + + # Live lookup if cache miss + if [ -z "$pr_number" ] && [ -z "$pr_lookup_done" ]; then + case "$remote_url" in + *dev.azure.com*|*visualstudio.com*) + ado_org=""; ado_project=""; ado_repo="" + if [[ "$remote_url" =~ dev\.azure\.com/([^/]+)/(.+)/_git/(.+)$ ]]; then + ado_org="${BASH_REMATCH[1]}" + ado_project="${BASH_REMATCH[2]}" + ado_repo="${BASH_REMATCH[3]}" + elif [[ "$remote_url" =~ //([^.]+)\.visualstudio\.com/(.+)/_git/(.+)$ ]]; then + ado_org="${BASH_REMATCH[1]}" + ado_project="${BASH_REMATCH[2]}" + ado_repo="${BASH_REMATCH[3]}" + elif [[ "$remote_url" =~ :v3/([^/]+)/([^/]+)/(.+)$ ]]; then + ado_org="${BASH_REMATCH[1]}" + ado_project="${BASH_REMATCH[2]}" + ado_repo="${BASH_REMATCH[3]}" + fi + # Project name may be URL-encoded ("Tabular%20Editor%20Learn") + if [ -n "$ado_project" ] && command -v python3 >/dev/null 2>&1; then + ado_project=$(python3 -c 'import sys,urllib.parse;print(urllib.parse.unquote(sys.argv[1]))' "$ado_project" 2>/dev/null || printf '%s' "$ado_project") + fi + + if [ -n "$ado_org" ] && [ -n "$ado_repo" ] && [ -n "$ado_project" ] && command -v az >/dev/null 2>&1; then + pr_json=$(_timeout 3 az repos pr list \ + --repository "$ado_repo" \ + --project "$ado_project" \ + --source-branch "$branch" \ + --status active \ + --organization "https://dev.azure.com/$ado_org" \ + 2>/dev/null) + pr_number=$(printf '%s' "$pr_json" | jq -r '.[0].pullRequestId // empty' 2>/dev/null) + + if [ -n "$pr_number" ]; then + # URL-encode project name once (spaces -> %20) for the URL + ado_project_enc=$(python3 -c 'import sys,urllib.parse;print(urllib.parse.quote(sys.argv[1]))' "$ado_project" 2>/dev/null || printf '%s' "$ado_project") + pr_url="https://dev.azure.com/${ado_org}/${ado_project_enc}/_git/${ado_repo}/pullrequest/${pr_number}" + is_draft=$(printf '%s' "$pr_json" | jq -r '.[0].isDraft // false' 2>/dev/null) + # Excluding the PR creator from approval counting -- DevOps policy + # typically has "allow requestors to approve own changes = false", + # so the creator's vote shouldn't tint the state. Rejections from + # anyone (including creator) still flag changes_requested. + creator_id=$(printf '%s' "$pr_json" | jq -r '.[0].createdBy.id // empty' 2>/dev/null) + non_creator_max=$(printf '%s' "$pr_json" | jq -r --arg cid "$creator_id" '[.[0].reviewers[] | select(.id != $cid) | .vote] | max // 0' 2>/dev/null) + any_min=$(printf '%s' "$pr_json" | jq -r '[.[0].reviewers[].vote] | min // 0' 2>/dev/null) + if [ "$is_draft" = "true" ]; then + pr_review="draft" + elif [ "${any_min:-0}" -lt 0 ] 2>/dev/null; then + pr_review="changes_requested" + elif [ "${non_creator_max:-0}" -ge 10 ] 2>/dev/null; then + pr_review="approved" + else + pr_review="pending" + fi + fi + fi + ;; + esac + + # Write cache regardless of outcome -- empty result still saves us from + # re-running the slow lookup until the TTL expires. + mkdir -p "$CACHE_DIR" 2>/dev/null + { + printf 'pr_number=%q\n' "${pr_number}" + printf 'pr_review=%q\n' "${pr_review}" + printf 'pr_url=%q\n' "${pr_url}" + echo 'pr_lookup_done=1' + } > "$cache_file" 2>/dev/null + fi + fi + fi + + if [ -n "$pr_number" ]; then + case "$pr_review" in + approved) pr_c="$GREEN" ;; + pending) pr_c="$YELLOW" ;; + changes_requested) pr_c="$RED" ;; + draft) pr_c="$DIM" ;; + *) pr_c="" ;; + esac + + # If we don't already have a URL (GitHub path via input JSON, or stale cache), + # derive it from the remote. Supports github.com SSH/HTTPS and falls through + # silently for hosts we don't recognise. + if [ -z "$pr_url" ]; then + [ -z "$remote_url" ] && remote_url=$(cd "$cwd" 2>/dev/null && git remote get-url origin 2>/dev/null) + case "$remote_url" in + *github.com*) + if [[ "$remote_url" =~ github\.com[:/](.+/.+)(\.git)?$ ]]; then + gh_path="${BASH_REMATCH[1]%.git}" + pr_url="https://github.com/${gh_path}/pull/${pr_number}" + fi + ;; + esac + fi + + pr_text="#${pr_number}" + + pr_visible=" ${pr_text}" + if [ -n "$pr_url" ]; then + # OSC 8 wraps the entire glyph+number so the whole segment is clickable. + # BEL terminator (\007) avoids a backslash collision with following \033. + pr_visible=$(printf '\033]8;;%s\007%s\033]8;;\007' "$pr_url" "$pr_visible") + fi + if [ -n "$pr_c" ]; then + seg "${pr_c}${pr_visible}${R}" + else + seg "${pr_visible}" + fi + fi + + # Unpushed commits (hidden when 0). Same colour as branch glyph (default fg). + unpushed=$(cd "$cwd" 2>/dev/null && git rev-list --count @{u}..HEAD 2>/dev/null) + [ -z "$unpushed" ] && unpushed=0 + if [ "$unpushed" -gt 0 ]; then + plural="s" + [ "$unpushed" -eq 1 ] && plural="" + seg " $unpushed commit$plural" + fi + + # File-level counts from `git status --porcelain`, categorised by INTENT. + # Untracked directories ("?? dir/") are expanded via find -maxdepth 2 + # to get the real file count without the cost of -uall on huge trees. + status_out=$(cd "$cwd" 2>/dev/null && git status --porcelain 2>/dev/null) + if [ -n "$status_out" ]; then + added=0; modified=0; deleted=0 + while IFS= read -r line; do + [ -z "$line" ] && continue + code="${line:0:2}" + fpath="${line:3}" + case "$code" in + '??') + case "$fpath" in + */) + n=$(find "$cwd/$fpath" -maxdepth 2 -type f 2>/dev/null | wc -l) + added=$((added + n)) ;; + *) added=$((added+1)) ;; + esac + ;; + *D*) deleted=$((deleted+1)) ;; + *A*) added=$((added+1)) ;; + *[MTRC]*) modified=$((modified+1)) ;; + esac + done <<< "$status_out" + + file_seg="" + [ "$added" -gt 0 ] && file_seg+="${GREEN}?:${added}${R}" + if [ "$modified" -gt 0 ]; then + [ -n "$file_seg" ] && file_seg+=" " + file_seg+="${YELLOW}M:${modified}${R}" + fi + if [ "$deleted" -gt 0 ]; then + [ -n "$file_seg" ] && file_seg+=" " + file_seg+="${RED}D:${deleted}${R}" + fi + [ -n "$file_seg" ] && seg " $file_seg" + fi + + # LOC delta: git diff HEAD covers staged+unstaged for tracked files (including deletes); + # untracked new files are counted separately via find + wc -l. diff_stat=$(cd "$cwd" 2>/dev/null && git diff HEAD --shortstat 2>/dev/null) add=$(echo "$diff_stat" | grep -oE '[0-9]+ insertion' | grep -oE '^[0-9]+' | head -1) - del=$(echo "$diff_stat" | grep -oE '[0-9]+ deletion' | grep -oE '^[0-9]+' | head -1) - untracked=$(cd "$cwd" 2>/dev/null && git ls-files --others --exclude-standard 2>/dev/null | wc -l | tr -d ' ') - + del=$(echo "$diff_stat" | grep -oE '[0-9]+ deletion' | grep -oE '^[0-9]+' | head -1) [ -z "$add" ] && add=0 [ -z "$del" ] && del=0 - [ -z "$untracked" ] && untracked=0 - total_add=$((add + untracked)) - if [ "$total_add" -eq 0 ] && [ "$del" -eq 0 ]; then - seg "${DIM}no changes${R}" + # Count lines in untracked files (new content not in git diff) + untracked_lines=0 + while IFS= read -r line; do + [ -z "$line" ] && continue + code="${line:0:2}" + fpath="${line:3}" + if [ "$code" = "??" ]; then + full="$cwd/$fpath" + case "$fpath" in + */) n=$(find "$full" -maxdepth 2 -type f -print0 2>/dev/null | xargs -0 wc -l 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1) ;; + *) n=$(wc -l < "$full" 2>/dev/null) ;; + esac + [ -z "$n" ] && n=0 + untracked_lines=$((untracked_lines + n)) + fi + done <<< "$status_out" + add=$((add + untracked_lines)) + + + if [ "$add" -eq 0 ] && [ "$del" -eq 0 ]; then + # Only show "no changes" when there are also no file-level changes at all + if [ -z "$status_out" ]; then + seg "${DIM}no changes${R}" + fi else - changes="" - [ "$total_add" -gt 0 ] && changes+="${GREEN}+${total_add}${R}" - [ "$total_add" -gt 0 ] && [ "$del" -gt 0 ] && changes+=" " - [ "$del" -gt 0 ] && changes+="${RED}-${del}${R}" - seg "$changes" + loc="" + [ "$add" -gt 0 ] && loc+="${GREEN}+${add}${R}" + [ "$add" -gt 0 ] && [ "$del" -gt 0 ] && loc+=" " + [ "$del" -gt 0 ] && loc+="${RED}-${del}${R}" + seg " $loc" fi - pr=$(cd "$cwd" 2>/dev/null && _timeout 2 gh pr view --json number 2>/dev/null | jq -r '.number // empty' 2>/dev/null) - [ -n "$pr" ] && seg " #${pr}" + # Staged-files indicator at the very end of the git segment, dim + if [ -n "$status_out" ]; then + staged_total=$(echo "$status_out" | grep -cE '^[MADRCU]') + if [ "$staged_total" -gt 0 ]; then + plural="s" + [ "$staged_total" -eq 1 ] && plural="" + seg "${DIM}(${staged_total} staged change${plural})${R}" + fi + fi fi diff --git a/useful-stuff/status-lines/statusline.d/04a-vim.sh b/useful-stuff/status-lines/statusline.d/04a-vim.sh new file mode 100644 index 00000000..9d002a79 --- /dev/null +++ b/useful-stuff/status-lines/statusline.d/04a-vim.sh @@ -0,0 +1,16 @@ +# Vim mode indicator. Shows current mode from $vim_mode (set in statusline.sh +# from JSON .vim.mode). Colour mirrors LazyVim/lualine convention: +# NORMAL=blue, INSERT=green, VISUAL/VISUAL LINE/VISUAL BLOCK=purple. +# Glyph: nf-custom-neovim (U+E6AE), the green diamond. +# Empty if editorMode is not "vim" in ~/.claude/settings.json. + +if [ -n "$vim_mode" ]; then + glyph=$'\xee\x9a\xae' + case "$vim_mode" in + NORMAL) vcolor="\033[38;5;75m" ;; + INSERT) vcolor="\033[38;5;154m" ;; + VISUAL|"VISUAL LINE"|"VISUAL BLOCK") vcolor="\033[38;5;99m" ;; + *) vcolor="$DIM" ;; + esac + seg "${vcolor}${glyph} ${vim_mode}${R}" +fi diff --git a/useful-stuff/status-lines/statusline.d/05-time.sh b/useful-stuff/status-lines/statusline.d/05-time.sh index fc5bff21..9c8b62c4 100644 --- a/useful-stuff/status-lines/statusline.d/05-time.sh +++ b/useful-stuff/status-lines/statusline.d/05-time.sh @@ -1 +1 @@ -seg " $(date +%H:%M)" +seg " $(date +%H:%M) " diff --git a/useful-stuff/status-lines/statusline.d/06-meters.sh b/useful-stuff/status-lines/statusline.d/06-meters.sh index ef5488be..b2bbc124 100644 --- a/useful-stuff/status-lines/statusline.d/06-meters.sh +++ b/useful-stuff/status-lines/statusline.d/06-meters.sh @@ -1,9 +1,86 @@ -for val_label in "${ctx_pct}:C" "${rate_5h}:S" "${rate_7d}:W"; do - val="${val_label%%:*}" - label="${val_label##*:}" - if [ -n "$val" ] && [ "$val" != "null" ]; then - pct=$(printf "%.0f" "$val" 2>/dev/null || echo "0") - color=$(pct_color "$pct") - seg "${color}${pct}% ${label}${R}" +# Bullet bar with linear projection to end of cycle. +# used: current usage percentage (integer) +# resets_at: unix timestamp when this window's quota refreshes +# cycle: length of the window in seconds (5h = 18000, 7d = 604800) +# label: letter shown before the bar (S, W, etc.) +render_bullet () { + local used=$1 resets_at=$2 cycle=$3 label=$4 + + local projected=$used + if [ -n "$resets_at" ] && [ "$resets_at" -gt 0 ] 2>/dev/null; then + local now=$(date +%s) + local until_reset=$((resets_at - now)) + if [ "$until_reset" -gt 0 ] && [ "$until_reset" -lt "$cycle" ]; then + local elapsed=$((cycle - until_reset)) + # Skip projection until we are at least 5% into the cycle to avoid + # the extrapolation flapping in the opening minutes. + local min_elapsed=$((cycle / 20)) + if [ "$elapsed" -ge "$min_elapsed" ]; then + projected=$(( used * cycle / elapsed )) + fi + fi fi -done + [ "$projected" -gt 999 ] 2>/dev/null && projected=999 + + # Layered fill: 5 shades, each spans a 20% band. As usage climbs, the + # next shade overpaints from the left, so every cell shows the topmost + # layer that has reached it. Projection is conveyed only by the ↑ + # overflow glyph below; the bar itself reflects used% only. + local layer k + if [ "$used" -le 0 ] 2>/dev/null; then + layer=0; k=0 + else + layer=$(( (used + 19) / 20 )) + [ "$layer" -gt 5 ] && layer=5 + k=$(( (used - (layer - 1) * 20) / 2 )) + [ "$k" -gt 10 ] && k=10 + [ "$k" -lt 0 ] && k=0 + fi + + local bar="" i cell_layer cell_char cell_color + for i in 1 2 3 4 5 6 7 8 9 10; do + if [ "$layer" -eq 0 ] 2>/dev/null; then + cell_layer=0 + elif [ "$i" -le "$k" ] 2>/dev/null; then + cell_layer=$layer + else + cell_layer=$((layer - 1)) + fi + case "$cell_layer" in + 0) cell_char="░"; cell_color="$DIM" ;; + 1) cell_char="▒"; cell_color="$DIM" ;; + 2) cell_char="▒"; cell_color="$YELLOW" ;; + 3) cell_char="▓"; cell_color="$ORANGE" ;; + 4) cell_char="█"; cell_color="$BRIGHT_RED" ;; + 5) cell_char="█"; cell_color="$MAROON" ;; + esac + bar="${bar}${cell_color}${cell_char}" + done + bar="${bar}${R}" + + local pct_color_val + pct_color_val=$(pct_color "$used") + local overflow="" + local BLINK="\033[5m" + # ↑ as soon as the projection is on track to hit the limit (>=100). + [ "$projected" -ge 100 ] 2>/dev/null && overflow=" ${BLINK}${CRIMSON}󰀦${R}" + + seg "${pct_color_val}${used}% ${label}${R} ${bar}${overflow}" +} + +# Context window: no time-based projection (no rolling cycle). +if [ -n "$ctx_pct" ] && [ "$ctx_pct" != "null" ]; then + pct=$(printf "%.0f" "$ctx_pct" 2>/dev/null || echo "0") + color=$(pct_color "$pct") + seg "${color}${pct}% C${R}" +fi + +# 5-hour and 7-day windows: bullet bar with projection. +if [ -n "$rate_5h" ] && [ "$rate_5h" != "null" ]; then + used=$(printf "%.0f" "$rate_5h" 2>/dev/null || echo "0") + render_bullet "$used" "$rate_5h_resets" $((5 * 3600)) "S" +fi +if [ -n "$rate_7d" ] && [ "$rate_7d" != "null" ]; then + used=$(printf "%.0f" "$rate_7d" 2>/dev/null || echo "0") + render_bullet "$used" "$rate_7d_resets" $((7 * 24 * 3600)) "W" +fi diff --git a/useful-stuff/status-lines/statusline.sh b/useful-stuff/status-lines/statusline.sh index af8d5a0f..01fd73bd 100755 --- a/useful-stuff/status-lines/statusline.sh +++ b/useful-stuff/status-lines/statusline.sh @@ -1,8 +1,8 @@ #!/bin/bash # -# Claude Code statusline. Each segment lives in statusline.d/<NN>-<name>.sh -# and is sourced in numeric order. Toggle a segment with the TRUE/FALSE -# flags below. Replace the hostname patterns in the host-color section +# Claude Code statusline. Segments live in statusline.d/<NN>-<name>.sh, sourced in +# numeric order; toggle each with the TRUE/FALSE flags below (two-line layout). +# Replace the hostname patterns in the display_host and host-color case blocks # with names from your own machines. ENABLE_HOST_CWD=TRUE @@ -10,11 +10,16 @@ ENABLE_GIT=TRUE ENABLE_MODEL=TRUE ENABLE_TIME=TRUE ENABLE_METERS=TRUE +ENABLE_VERSION=FALSE +ENABLE_VIM=TRUE # ---------------------------------------------------------------------------- -# Portable timeout. Linux has `timeout`; macOS has neither unless coreutils -# is installed (`gtimeout`). Falls back to running without a timeout. +# Edit the TRUE/FALSE flags above to toggle segments. Each segment lives in +# scripts/statusline.d/<NN>-<name>.sh and is sourced in numeric order. # ---------------------------------------------------------------------------- + +# Portable timeout — Linux has `timeout`, macOS has neither unless coreutils is installed (`gtimeout`). +# Falls back to running the command without a timeout if neither is available. if command -v timeout >/dev/null 2>&1; then _timeout() { timeout "$@"; } elif command -v gtimeout >/dev/null 2>&1; then @@ -28,16 +33,13 @@ input=$(cat) cwd=$(echo "$input" | jq -r '.cwd // empty' 2>/dev/null) [ -z "$cwd" ] && cwd="$PWD" # Normalize Windows paths (C:\foo\bar) to POSIX (/c/foo/bar) so backslashes -# do not get eaten by `echo -e` (\a -> bell, \b -> backspace, etc.) +# don't get eaten by echo -e (\a → bell, \b → backspace, etc.) command -v cygpath >/dev/null 2>&1 && cwd=$(cygpath -u "$cwd" 2>/dev/null || printf '%s' "$cwd") host=$(hostname -s 2>/dev/null) host_lower=$(echo "$host" | tr '[:upper:]' '[:lower:]') - -# Optional: shorten a long hostname for the display string. Add your own -# entries here if a machine's hostname is awkward to read. case "$host_lower" in - # example-long-hostname) display_host="short" ;; + # example-long-host) display_host="short" ;; # optional: shorten an awkward hostname *) display_host="$host" ;; esac dir=$(echo "$cwd" | sed "s|$HOME|$display_host|") @@ -48,6 +50,14 @@ effort_level=$(echo "$input" | jq -r '.effort.level // empty' 2>/dev/null) ctx_pct=$(echo "$input" | jq -r '.context_window.used_percentage // empty' 2>/dev/null) rate_5h=$(echo "$input" | jq -r '.rate_limits.five_hour.used_percentage // empty' 2>/dev/null) rate_7d=$(echo "$input" | jq -r '.rate_limits.seven_day.used_percentage // empty' 2>/dev/null) +rate_5h_resets=$(echo "$input" | jq -r '.rate_limits.five_hour.resets_at // empty' 2>/dev/null) +rate_7d_resets=$(echo "$input" | jq -r '.rate_limits.seven_day.resets_at // empty' 2>/dev/null) +vim_mode=$(echo "$input" | jq -r '.vim.mode // empty' 2>/dev/null) +pr_number=$(echo "$input" | jq -r '.pr.number // empty' 2>/dev/null) +pr_review=$(echo "$input" | jq -r '.pr.review_state // empty' 2>/dev/null) +wt_path=$(echo "$input" | jq -r '.worktree.path // empty' 2>/dev/null) +wt_name=$(echo "$input" | jq -r '.worktree.name // empty' 2>/dev/null) +wt_branch=$(echo "$input" | jq -r '.worktree.branch // empty' 2>/dev/null) R="\033[0m" DIM="\033[38;5;241m" @@ -65,18 +75,16 @@ CHARTREUSE="\033[38;5;154m" PURPLE="\033[38;5;141m" CRIMSON="\033[38;5;160m" -# Model icons. NerdFonts MDI (nf-md-robot_*), confirmed present in -# JetBrainsMono NF 3.4.0. +# Model icons: NerdFonts MDI (nf-md-robot_*), confirmed present in JetBrainsMono NF 3.4.0 # 󱚝 U+F169D nf-md-robot_angry 󱜙 U+F1719 nf-md-robot_happy 󱜚 U+F171A nf-md-robot_happy_outline -if echo "$model_full" | grep -qi "opus"; then model="Opus"; model_color="$RED"; model_icon="󱚝" -elif echo "$model_full" | grep -qi "haiku"; then model="Haiku"; model_color="$YELLOW"; model_icon="󱜚" -elif echo "$model_full" | grep -qi "sonnet"; then model="Sonnet"; model_color="$ORANGE"; model_icon="󱜙" +if echo "$model_full" | grep -qi "opus"; then model="Opus"; model_color="$RED"; model_icon="󱚝" +elif echo "$model_full" | grep -qi "haiku"; then model="Haiku"; model_color="$YELLOW"; model_icon="󱜚" +elif echo "$model_full" | grep -qi "sonnet";then model="Sonnet"; model_color="$ORANGE"; model_icon="󱜙" else model=""; model_color=""; model_icon="" fi -# Hide the version on the family-latest model (assumed default), show it on -# older releases (e.g. "Opus 4.6"). Bump LATEST_*_ID when a new model takes -# over the family. +# Hide version on the family-latest model (assumed default), show it on older +# releases (e.g. "Opus 4.6"). Bump LATEST_*_ID when a new model takes over the family. LATEST_OPUS_ID="opus-4-7" LATEST_SONNET_ID="sonnet-4-6" LATEST_HAIKU_ID="haiku-4-5" @@ -84,6 +92,8 @@ if [ -n "$model" ]; then case "$model_id" in *$LATEST_OPUS_ID*|*$LATEST_SONNET_ID*|*$LATEST_HAIKU_ID*) : ;; *) + # Prefer model_id (always has version, e.g. "claude-opus-4-6") since + # display_name is just the family per the docs example. model_version=$(echo "$model_id" | grep -oE '[0-9]+-[0-9]+' | head -1 | tr '-' '.') [ -z "$model_version" ] && model_version=$(echo "$model_full" | grep -oE '[0-9]+\.[0-9]+' | head -1) [ -n "$model_version" ] && model="$model $model_version" @@ -91,10 +101,9 @@ if [ -n "$model" ]; then esac fi -# Effort dots, calibrated per model. Haiku has no effort support and stays -# blank. Opus 4.7: 5 levels (low/medium/high/xhigh/max). Opus 4.6 + Sonnet -# 4.6: 4 levels (low/medium/high/max; xhigh falls back to high). See -# code.claude.com/docs/en/model-config. +# Effort dots, calibrated per model. Haiku has no effort support and stays blank. +# Opus 4.7: 5 levels (low/medium/high/xhigh/max). Opus 4.6 + Sonnet 4.6: 4 levels +# (low/medium/high/max; xhigh falls back to high). See code.claude.com/docs/en/model-config. case "$model" in Haiku*) effort_dots="" @@ -133,24 +142,25 @@ case "$model" in ;; esac -# Host color. Replace the example entries with your own machine hostnames so -# each host gets a distinct color in the statusline. Anything not matched -# falls through to the default. case "$host_lower" in - hostname1) host_color="$MINT" ;; - hostname2) host_color="$PINK" ;; - hostname3) host_color="$PASTEL_BLUE" ;; - hostname4) host_color="$PURPLE" ;; - *) host_color="$PINK" ;; + hostname1) host_color="$MINT" ;; + hostname2) host_color="$PINK" ;; + hostname3) host_color="$PASTEL_BLUE" ;; + hostname4) host_color="$YELLOW" ;; + *) host_color="$PINK" ;; esac SEP="${DIM} · ${R}" out="" seg() { - if [ -z "$out" ]; then out="$1" - else out="${out}${SEP}$1" + local tail="${out: -1}" + if [ -z "$out" ] || [ "$tail" = $'\n' ]; then + out="${out}$1" + else + out="${out}${SEP}$1" fi } +nl() { out="${out}"$'\n'; } # Apply threshold color to a percentage value pct_color() { @@ -172,10 +182,18 @@ load_segment() { . "$STATUSLINE_D/$file" } -load_segment "$ENABLE_HOST_CWD" 02-host-cwd.sh -load_segment "$ENABLE_GIT" 03-git.sh -load_segment "$ENABLE_MODEL" 04-model.sh -load_segment "$ENABLE_TIME" 05-time.sh -load_segment "$ENABLE_METERS" 06-meters.sh +# Two-line layout. Version segment on line 2 is the visual counterpart +# of time on line 1 -- both leftmost, both short, dim. Keeps the model +# segment column-aligned with the cwd above. +# line 1: time · host/cwd · git +# line 2: version · model · meters +load_segment "$ENABLE_TIME" 05-time.sh +load_segment "$ENABLE_HOST_CWD" 02-host-cwd.sh +load_segment "$ENABLE_GIT" 03-git.sh +nl +load_segment "$ENABLE_VERSION" 01-version.sh +load_segment "$ENABLE_VIM" 04a-vim.sh +load_segment "$ENABLE_MODEL" 04-model.sh +load_segment "$ENABLE_METERS" 06-meters.sh echo -e "$out"