diff --git a/AGENTS.md b/AGENTS.md index e96e4a56a..d079bc6ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,12 +2,27 @@ This file provides guidance to AI Agents when working with code in this repository. +> [!IMPORTANT] +> `AGENTS.md` is the single source of truth for agent guidance. `CLAUDE.md` and `.github/copilot-instructions.md` are git symlinks (mode `120000`) pointing here. Never replace those symlinks with real files — edit this file instead. + ## Repository Overview The FinOps Toolkit is an open-source collection of tools for adopting and implementing FinOps capabilities in the Microsoft Cloud. It contains templates, PowerShell modules, workbooks, optimization engines, and supporting documentation organized in a modular architecture. ## Common Commands +### First-time setup + +`src/scripts/Init-Repo.ps1` installs the required tooling (Az PowerShell, Bicep CLI) and optional tooling (VS Code, Bicep module, NPM, Pester): + +```powershell +./src/scripts/Init-Repo -All # required + optional tooling +./src/scripts/Init-Repo -Pester # required + Pester only +./src/scripts/Init-Repo -All -WhatIf # preview +``` + +**Pester 6.0.0 or later is required.** The suite uses `-AllowNullOrEmptyForEach`, which Pester 5 rejects during discovery — the affected file is silently skipped. `Test-PowerShell.ps1` resolves the module version explicitly and fails with an actionable message if only an older Pester is present. CI pins the same floor (`.github/workflows/dev.yml`). + ### Building and Development ```bash @@ -19,11 +34,19 @@ pwsh -Command ./src/scripts/Build-Toolkit # Build FinOps hubs pwsh -Command ./src/scripts/Build-Toolkit finops-hub +# Build a single workbook +pwsh -Command ./src/scripts/Build-Toolkit "-workbook" + # Build specific components -npm run build-ps # PowerShell module only -pwsh -Command ./src/scripts/Build-Bicep # Bicep templates -pwsh -Command ./src/scripts/Build-Workbook # Azure Monitor workbooks -pwsh -Command ./src/scripts/Build-OpenData # Open data files +npm run build-ps # PowerShell module (Invoke-Task Build.PsModule) +pwsh -Command ./src/scripts/Build-PowerShell # PowerShell module (wraps the Invoke-Build task) +pwsh -Command ./src/scripts/Build-Bicep ../bicep-registry/ # single Bicep Registry module +pwsh -Command ./src/scripts/Build-Workbook # Azure Monitor workbooks +pwsh -Command ./src/scripts/Build-OpenData # Open data files (check in generated files manually) +pwsh -Command ./src/scripts/Invoke-Task -Task # Invoke-Build tasks (e.g. Build.PsModule) + +# Load the locally built module +pwsh -Command 'Remove-Module FinOpsToolkit -EA SilentlyContinue; Import-Module -FullyQualifiedName ./src/powershell/FinOpsToolkit.psm1' # Deploy for testing npm run deploy-test @@ -38,22 +61,43 @@ pwsh -Command ./src/scripts/Package-Toolkit -Build ### Testing +`src/scripts/Test-PowerShell.ps1` is the entry point for all Pester runs. It runs unit tests by default; naming any test type runs only those types. + ```bash # Run PowerShell unit tests -npm run pester +pwsh -Command ./src/scripts/Test-PowerShell # or -pwsh -Command Invoke-Pester -Output Detailed -Path ./src/powershell/Tests/Unit/* +npm run pester -# Run integration tests +# Run lint / integration / everything +pwsh -Command ./src/scripts/Test-PowerShell -Lint pwsh -Command ./src/scripts/Test-PowerShell -Integration +pwsh -Command ./src/scripts/Test-PowerShell -AllTests -# Run specific test categories +# Run specific test categories (combine freely) +# Cost, Data, Docs, Exports, FOCUS, Hubs, Toolkit, Workbooks, Actions, Private pwsh -Command ./src/scripts/Test-PowerShell -Hubs -Exports -# Lint PowerShell code -pwsh -Command ./src/scripts/Test-PowerShell -Lint +# Re-run only the tests that failed in the previous run +pwsh -Command ./src/scripts/Test-PowerShell -RunFailed +``` + +Run a **single test file** or a **single test case** with Pester directly (import Pester 6 explicitly so a side-by-side Pester 3/5 install cannot win): + +```bash +# One file +pwsh -Command 'Import-Module Pester -MinimumVersion 6.0.0; Invoke-Pester -Output Detailed -Path ./src/powershell/Tests/Unit/Get-FinOpsRegion.Tests.ps1' + +# One Describe/Context/It by name +pwsh -Command 'Import-Module Pester -MinimumVersion 6.0.0; Invoke-Pester -Output Detailed -Path ./src/powershell/Tests/Unit/Get-FinOpsRegion.Tests.ps1 -FullNameFilter "*returns all regions*"' ``` +After a `Test-PowerShell` run, inspect these globals to debug: + +- `$global:ftk_TestPowerShell_Results` — full result object from the last run +- `$global:ftk_TestPowerShell_Summary` — failed tests only +- `$global:ftk_TestPowerShell_FailedTests` — the Pester config used by `-RunFailed` + ### Bicep Development ```bash @@ -68,23 +112,27 @@ az deployment group what-if --resource-group myRG --template-file template.bicep ### High-Level Structure -- **`/src/templates/`** - ARM/Bicep infrastructure templates with modular namespace organization +- **`/src/templates/`** - ARM/Bicep infrastructure templates (`finops-hub`, `finops-alerts`, `finops-workbooks`, `agent-plugin`, `finops-hub-copilot*`) - **`/src/powershell/`** - PowerShell module with public/private functions and comprehensive tests +- **`/src/queries/`** - KQL query catalog (`catalog/`, `INDEX.md`, `KPI.md`, `finops-hub-database-guide.md`) +- **`/src/bicep-registry/`** - Bicep Registry modules (multi-scope build) - **`/src/optimization-engine/`** - Azure Optimization Engine for cost recommendations - **`/src/workbooks/`** - Azure Monitor workbooks for governance and optimization - **`/src/open-data/`** - Reference data (pricing, regions, services) with utilities -- **`/src/scripts/`** - Build automation and development tools +- **`/src/power-bi/`** - Power BI reports (built manually, not by the build scripts) +- **`/src/scripts/`** - Build automation and development tools (see `src/scripts/README.md`) +- **`/plugins/`**, **`/.claude-plugin/`** - Agent plugin packaging (`plugins/microsoft-finops-toolkit`) - **`/docs/`** - Jekyll documentation website -- **`/docs-mslearn/`** - Microsoft Learn documentation website -- **`/docs-wiki/`** - GitHub wiki documentation +- **`/docs-mslearn/`** - Microsoft Learn documentation website (includes `toolkit/changelog.md`) +- **`/docs-wiki/`** - GitHub wiki documentation (authoritative dev process + coding guidelines) ### Current Architectural Reorganization -The FinOps hubs solution is actively migrating to a namespace-based modular structure: +The FinOps hubs solution is actively migrating to a namespace-based modular structure under `src/templates/finops-hub/modules/`: -- **`Microsoft.FinOpsHubs/`** - Core FinOps Hub infrastructure modules +- **`Microsoft.FinOpsHubs/`** - Core FinOps Hub infrastructure modules, split into `Core`, `Analytics`, `IngestionQueries`, `Recommendations`, `AzureResourceGraph`, and `RemoteHub` - **`Microsoft.CostManagement/`** - Cost management exports and schemas -- **`fx/`** - Shared foundation components (hub-types, scripts, utilities) +- **`fx/`** - Shared foundation components: `hub-types.bicep`, `hub-app.bicep`, `hub-storage.bicep`, `hub-database.bicep`, `hub-identity.bicep`, `hub-vault.bicep`, `hub-deploymentScript.bicep`, `hub-eventTrigger.bicep`, plus `scripts/` and version/tag files ### Template Architecture @@ -106,8 +154,11 @@ Key patterns: - **`Public/`** - User-facing cmdlets (Get-_, Set-_, New-\*, etc.) - **`Private/`** - Internal utilities and helpers -- **`Tests/Unit/`** - Pester unit tests with mocking +- **`en-US/`** - Localized strings (validated by `Tests/Unit/LocalizedData.Tests.ps1`) +- **`Tests/Lint/`** - Repo-wide standards tests (`Lint.Tests.ps1`, `KqlJoinKinds.Tests.ps1`, `MsLearnDocs.Tests.ps1`) +- **`Tests/Unit/`** - Pester unit tests with mocking. Note these cover far more than cmdlets — hub Bicep/KQL guards (`HubsKqlOperators`, `HubsIngestionQueries`, `HubsPrivateNetworking`, `HubsContractedCostGuard`, `HubsAdfTriggerTimeZones`), GitHub Actions parity, and docs links all live here - **`Tests/Integration/`** - End-to-end Azure integration tests +- **`Tests/Initialize-Tests.ps1`** - Reimports `FinOpsToolkit.psm1` and dot-sources `src/scripts/Monitor.ps1`; test files reference it rather than importing the module themselves - **Module manifest** defines exports and dependencies ### Data Flow and Integration @@ -152,8 +203,8 @@ The PowerShell-based build system: ### Version Management -- Central version in `package.json` (currently 12.0.0) -- Synchronized across all components via build scripts +- Central version in `package.json` (source of truth; prerelease format is `.0.0-dev.0`) +- Synchronized across all components via `src/scripts/Update-Version.ps1`; read the current value with `src/scripts/Get-Version` - Individual `ftkver.txt` files distributed to modules - Git tags correspond to release versions @@ -206,6 +257,17 @@ This repository supports production infrastructure managing significant revenue. - Documentation uses Jekyll conventions - Build artifacts are generated, not checked in +### Changelog + +User-facing changes must be added to `docs-mslearn/toolkit/changelog.md`. Full rules are in the "Changelog" section of `docs-wiki/Coding-guidelines.md`. Key points: + +- All changes for the upcoming release go in **one** version section — never create a duplicate `## v{version}` heading +- Group by tool with an H3 heading linking to the tool's doc page plus its version (e.g. `### [FinOps hubs](...) v14`), matching the tool order used in previous releases +- Category order: Added, Changed, Fixed, Deprecated, Removed. Omit empty categories +- One past-tense sentence per entry, ending with a period, linking the issue as `([#{number}]({url}))` when one exists +- Write for users, not developers — no implementation details, no filler entries +- Prefix breaking changes with `**Breaking:**` and list them first in their category + ### Coding Standards - Always follow the content and coding standards defined in `docs-wiki/Coding-guidelines.md` diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 92501d68d..59458c98b 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 08/22/2026 +ms.date: 08/27/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -28,7 +28,11 @@ The following section lists features and enhancements that are currently in deve ### [FinOps hubs](hubs/finops-hubs-overview.md) - **Added** + - Added optional invoice download, which saves your Microsoft invoice files in the hub data lake every month, organized by billing period, billing profile, and purchase order number. Enable it in the **Invoices** step of the deployment wizard. - Added VNet and private network modes, including opt-in NAT Gateway support for private mode; NAT Gateway incurs additional cost when enabled ([#2163](https://github.com/microsoft/finops-toolkit/pull/2163)). + - Added optional AWS FOCUS ingestion, which copies FOCUS 1.2 cost and usage exports from an Amazon S3 bucket into the hub data lake every day so AWS costs are normalized alongside Microsoft Cloud costs. Enable it in the **Multicloud** step of the deployment wizard. + - Added a **Usage optimization** page to the Azure Data Explorer dashboard that surfaces Azure Advisor cost recommendations and the hub's built-in recommendations, with estimated savings summarized by impact, recommendation type, resource type, and subscription, plus detail tables and a collection freshness view. + - Added the `Add-FinOpsHubResourceGraphReader` PowerShell command to grant the Data Factory managed identity Reader access to subscriptions or management groups used by Resource Graph recommendations. - **Changed** - Clarified that the FinOps toolkit exclusively manages the FinOps hub virtual network and documented customer-managed private endpoints as the preferred private-access topology, with virtual network peering as a secondary option ([#2156](https://github.com/microsoft/finops-toolkit/issues/2156)). - Replaced redundant `tolower()` comparisons in hub KQL with case-insensitive operators (`has`, `=~`, `!~`) so the engine can use the term index instead of scanning every row ([#2213](https://github.com/microsoft/finops-toolkit/issues/2213)). diff --git a/docs-mslearn/toolkit/hubs/configure-recommendations.md b/docs-mslearn/toolkit/hubs/configure-recommendations.md index aa7644600..99b52415c 100644 --- a/docs-mslearn/toolkit/hubs/configure-recommendations.md +++ b/docs-mslearn/toolkit/hubs/configure-recommendations.md @@ -26,6 +26,20 @@ Before you begin, you must have: - [Deployed a FinOps hub instance](finops-hubs-overview.md#create-a-new-hub). - Assigned the **Reader** role to the Data Factory managed identity on the management groups or subscriptions you want to query. This permission must be configured separately from the FinOps hub deployment. +If the role has not been assigned, use the `Add-FinOpsHubResourceGraphReader` cmdlet after connecting +to Azure. The command is safe to rerun because it checks for an existing assignment first: + +```powershell +Connect-AzAccount +Add-FinOpsHubResourceGraphReader ` + -Scope '' ` + -HubName '' +``` + +Use a subscription ID to grant access to one subscription, or a management group resource ID to grant +access to all subscriptions below that management group. The command must be run by an account that +can create role assignments at the requested scope. +
## How recommendations are processed diff --git a/docs-mslearn/toolkit/hubs/template.md b/docs-mslearn/toolkit/hubs/template.md index 3073275b1..1fd014ee0 100644 --- a/docs-mslearn/toolkit/hubs/template.md +++ b/docs-mslearn/toolkit/hubs/template.md @@ -3,7 +3,7 @@ title: FinOps hub template description: Learn about what's included in the FinOps hub template including parameters, resources, and outputs. author: flanakin ms.author: micflan -ms.date: 06/03/2026 +ms.date: 08/24/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit @@ -101,6 +101,18 @@ Here are the parameters you can use to customize the deployment: | **enableRecommendations** | Bool | Optional. Enable recommendations ingested from Azure Resource Graph based on configurable queries. The Data Factory managed identity requires Reader role on management groups or subscriptions to execute Resource Graph queries. | False | | **enableAHBRecommendations** | Bool | Optional. Enable Azure Hybrid Benefit recommendations that flag VMs and SQL VMs without Azure Hybrid Benefit enabled. May generate noise if your organization does not have on-premises licenses. Requires enableRecommendations. | False | | **enableSpotRecommendations** | Bool | Optional. Enable non-Spot AKS cluster recommendations that flag AKS clusters with autoscaling but not using Spot VMs. May generate noise since Spot VMs are only appropriate for interruptible workloads. Requires enableRecommendations. | False | +| **enableInvoiceDownload** | Bool | Optional. Enable automatic download of Microsoft invoice files into the hub data lake. Only supported for Microsoft Customer Agreement (MCA) and Microsoft Partner Agreement (MPA) billing accounts. The Data Factory managed identity requires Billing Reader role on the billing account. | False | +| **invoiceBillingAccounts** | String | Optional. Billing account IDs to download invoices for, separated by a new line, comma, or semicolon. Requires enableInvoiceDownload. Leave empty to use the billing account scopes monitored by this hub. | | +| **invoiceScheduleDay** | Int | Optional. Day of the month to download invoices from the previous month. Requires enableInvoiceDownload. | 10 | +| **enableAwsFocusIngestion** | Bool | Optional. Enable daily collection of FOCUS 1.2 cost and usage exports from an Amazon S3 bucket. Requires an AWS Data Exports FOCUS 1.2 export and an AWS access key with read access to the bucket. AWS charges apply for S3 requests and data transfer out. | False | +| **awsBucketName** | String | Optional. Name of the Amazon S3 bucket that contains the FOCUS export. Requires enableAwsFocusIngestion. | | +| **awsBucketPath** | String | Optional. Path to the FOCUS export root folder in the bucket. This is the folder that contains the `data` and `metadata` subfolders created by AWS Data Exports. Requires enableAwsFocusIngestion. | | +| **awsAccountId** | String | Optional. AWS account ID (management or payer account) used to identify the source scope in the data lake. Requires enableAwsFocusIngestion. | | +| **awsRegion** | String | Optional. AWS region of the S3 bucket. Requires enableAwsFocusIngestion. | "us-east-1" | +| **awsAccessKeyId** | String | Optional. Access key ID for the AWS IAM user with read access to the bucket. Requires enableAwsFocusIngestion. | | +| **awsSecretAccessKey** | String | Optional. Secret access key for the AWS IAM user. Stored in the hub key vault. Requires enableAwsFocusIngestion. | | +| **awsFocusVersion** | String | Optional. FOCUS version of the AWS export. Allowed values: `1.2`. Requires enableAwsFocusIngestion. | "1.2" | +| **multiCloudScheduleHour** | Int | Optional. Hour of the day (UTC, 0-23) to collect multicloud exports. Requires enableAwsFocusIngestion. | 4 | | **enablePublicAccess** | Bool | Optional. Disable public access to the data lake (storage firewall). | True | | **virtualNetworkAddressPrefix** | String | Optional. IP Address range for the private virtual network used by FinOps hubs. Accepts any subnet size from `/8` to `/26` with a minimum of `/26` required. `/26` is recommended to avoid wasting IPs unless you need additional address space for services like Power BI VNet Data Gateway. Internally, the following subnets will be created: `/28` for private endpoints, another `/28` subnet for temporary deployment scripts (container instances), and `/27` for Azure Data Explorer, if enabled. | '10.20.30.0/26' | diff --git a/docs-wiki/design/Multicloud-FOCUS-design.md b/docs-wiki/design/Multicloud-FOCUS-design.md new file mode 100644 index 000000000..409972edb --- /dev/null +++ b/docs-wiki/design/Multicloud-FOCUS-design.md @@ -0,0 +1,566 @@ +# Design: optional AWS and Google FOCUS ingestion in FinOps hub + +> Technical detail for the plan approved in [Multicloud-FOCUS-plan](./Multicloud-FOCUS-plan.md). Based on a reading of the current code in `src/templates/finops-hub`. +> Working branch: `arthursilvany/multicloud-focus`. + +--- + +## 1. The discovery that defines the architecture + +The hub **already has** the entire FOCUS → Parquet → ADX pipeline. All that is missing is **delivering the file to the right door**. + +The current chain, confirmed in code: + +``` +Cost Management export + → writes files + manifest.json to the "msexports" container + → msexports_ManifestAdded trigger (BlobEventsTrigger, storagePathEndsWith: 'manifest.json') + → msexports_ExecuteETL pipeline (reads the manifest, picks the schema) + → msexports_ETL_ingestion (converts to Parquet in the "ingestion" container) + → ingestion_ManifestAdded trigger + → ADX / Fabric ingestion pipeline +``` + +Evidence: + +| Fact | File | +| --- | --- | +| Trigger fires **only** on `manifest.json` | `Microsoft.CostManagement/Exports/app.bicep:1712-1728` | +| Same for the ingestion trigger | `Microsoft.FinOpsHubs/Analytics/app.bicep:682-698` | +| FOCUS 1.0 / 1.0r2 / 1.2 schemas already published to the `config` container | `Microsoft.CostManagement/Exports/app.bicep:59-85` | +| Datasets for CSV, **gzip**, and **Parquet** already exist | `Microsoft.CostManagement/Exports/app.bicep:164-254` | + +**Design consequence:** do not build a parallel ETL. The AWS/Google connector only has to: + +1. copy the FOCUS files from the bucket into `msexports//...`; +2. write a compatible `manifest.json` **last**. + +From there, everything that already exists works unchanged — including Parquet conversion, retention, and ADX ingestion. Because the trigger only reacts to `manifest.json`, writing the data first and the manifest afterward is safe by construction. + +This also settles the format question: AWS delivers FOCUS as `.csv.gz` or `.parquet`, and GCS as `.csv`/`.parquet` — all three are already handled by the `msexports`, `msexports_gzip`, and `msexports_parquet` datasets. + +--- + +## 2. The `manifest.json` contract + +Fields actually consumed by `msexports_ExecuteETL` (extracted from the `activity('Read Manifest').output.firstRow.*` expressions): + +| Field | Use | Value for AWS/GCP | +| --- | --- | --- | +| `exportConfig.type` | 1st part of the schema file name | `FocusCost` | +| `exportConfig.dataVersion` | 2nd part of the schema file name (and **only** that — see §8/R2) | `1.2-aws` / `1.2-gcp` (see §2.1) | +| `exportConfig.exportName` | logical export name | `aws-focus` / `gcp-focus` | +| `exportConfig.resourceId` | derives the scope (= path segment) | see §2.2 | +| `runInfo.runId` | run identity | GUID generated in the pipeline | +| `runInfo.startDate` | data period | start of the file's month | +| `blobCount` / `blobs[].blobName` | file list | populated by Get Metadata | +| `dataRowCount` | empty-export short-circuit | **omit the property** — never `null` or `0` (see §10) | +| `retention.msexports.days` | cleanup | copy from `settings.json` | +| `additionalColumns`, `translator` | come from the schema file, not the manifest | omit | + +The manifest is assembled inside the pipeline (a `Set Variable` activity + a `Copy` with `JsonSink`), not as a static file — its content depends on the run. + +### 2.1 How the schema is selected — the extension point + +``` +schemaFile = toLower(concat(exportDatasetType, '_', exportDatasetVersion, , '.json')) +``` + +The `` (`_ea` / `_mca`) is only applied when `mcaColumnToCheck` is non-null, and that variable is null for `FocusCost` — it is only populated for `pricesheet`, `reservationtransactions`, and `reservationrecommendations`. So for FOCUS the file name is determined **entirely** by two fields we control in the synthetic manifest. + +Consequence: publishing `focuscost_1.2-aws.json` and setting `dataVersion: '1.2-aws'` makes the ETL load the correct schema **without a single change to the existing pipeline**. `exportConfig.type`, by contrast, must be exactly `FocusCost` — see §8d. + +### 2.2 Value of `exportConfig.resourceId` + +Recommended: `/aws/` and `/gcp/`, **always lowercase** (see §10 — the ADX tag comparison is case-sensitive, and case variations silently duplicate data). + +This produces the paths `Costs/2026/08/aws/123456789012/` and `Costs/2026/08/gcp/my-project/`, which are readable and isolated from Azure scope paths. The first segment is the `hubDataset` (`focuscost` → `Costs`), not the `exportDatasetType`. See §8/R1 for the analysis that supports this freedom of format. + +--- + +## 3. New parameters + +### `main.bicep` and `modules/hub.bicep` + +These follow the `enableInvoiceDownload` pattern exactly (`main.bicep:52-57`). + +```bicep +@description('Optional. Enable ingestion of FOCUS cost data exported from Amazon Web Services. Requires an S3 bucket with a FOCUS 1.0 or 1.2 export and an access key stored during deployment. Default: false.') +param enableAwsFocusIngestion bool = false + +@description('Optional. Name of the Amazon S3 bucket that contains the FOCUS export. Requires enableAwsFocusIngestion.') +param awsBucketName string = '' + +@description('Optional. Path prefix within the S3 bucket where FOCUS files are written. Requires enableAwsFocusIngestion.') +param awsBucketPrefix string = '' + +@description('Optional. AWS region of the S3 bucket. Requires enableAwsFocusIngestion.') +param awsRegion string = '' + +@description('Optional. AWS access key ID used to read the S3 bucket. Requires enableAwsFocusIngestion.') +param awsAccessKeyId string = '' + +@description('Optional. AWS secret access key used to read the S3 bucket. Stored in Key Vault. Requires enableAwsFocusIngestion.') +@secure() +param awsSecretAccessKey string = '' + +@description('Optional. FOCUS version of the AWS export. Allowed: 1.0, 1.2. Default: 1.0.') +@allowed(['1.0', '1.2']) +param awsFocusVersion string = '1.0' +``` + +The Google equivalent uses the names `enableGoogleFocusIngestion`, `googleBucketName`, `googleBucketPrefix`, `googleProjectId`, `googleAccessKeyId`, `googleSecretAccessKey` (GCS HMAC), and `googleFocusVersion`. + +Common to both: + +```bicep +@description('Optional. Hour of the day (UTC) to collect multicloud FOCUS files. Default: 4.') +@minValue(0) +@maxValue(23) +param multiCloudScheduleHour int = 4 +``` + +**Key design point:** the defaults leave everything off. An existing deployment that runs `main.bicep` without these parameters changes in no way — no new resources, no new cost. + +### Telemetry — `modules/hub.bicep` + +`telemetryString` (`hub.bicep:206-222`) is limited to 12 characters. Add only two flags: + +```bicep +// A = AWS FOCUS ingestion, G = Google FOCUS ingestion +enableAwsFocusIngestion ? 'A' : '' +enableGoogleFocusIngestion ? 'G' : '' +``` + +--- + +## 4. Portal UI — `createUiDefinition.json` + +Current step structure: `pricing`, `retention`, `recommendations`, `invoices`, `advanced`, `tags`. + +Add a `multicloud` step between `invoices` and `advanced`, mirroring the layout of the `invoices` step (`createUiDefinition.json:841-930`): + +``` +- multicloud (label: "🆕 Multicloud") + * multicloudIntro [Microsoft.Common.TextBlock] + * enableAws [Microsoft.Common.CheckBox] + * aws [Microsoft.Common.Section] visible: [steps('multicloud').enableAws] + - bucketName [Microsoft.Common.TextBox] + - bucketPrefix [Microsoft.Common.TextBox] + - region [Microsoft.Common.TextBox] + - accessKeyId [Microsoft.Common.TextBox] + - secretAccessKey [Microsoft.Common.PasswordBox] + - focusVersion [Microsoft.Common.DropDown] (1.0 | 1.2) + * enableGoogle [Microsoft.Common.CheckBox] + * google [Microsoft.Common.Section] visible: [steps('multicloud').enableGoogle] + - (same fields + projectId) + * schedule [Microsoft.Common.Section] + * permissions [Microsoft.Common.Section] (text describing the minimum IAM policy) +``` + +Outputs, in the same style as the existing ones (`createUiDefinition.json:1102`): + +```json +"enableAwsFocusIngestion": "[steps('multicloud').enableAws]", +"awsBucketName": "[if(steps('multicloud').enableAws, steps('multicloud').aws.bucketName, '')]", +"awsSecretAccessKey": "[if(steps('multicloud').enableAws, steps('multicloud').aws.secretAccessKey, '')]" +``` + +The `if(...)` is mandatory: it guarantees that hidden fields never leak stale values into the template — the same pattern already used for `remoteHubStorageUri`/`remoteHubStorageKey`. + +Use `Microsoft.Common.PasswordBox` for secrets so they never appear on screen or in portal history. + +--- + +## 5. New modules + +Two sibling apps, following the structure of `Microsoft.Billing/Invoices` (the newest and most complete optional app in the repository): + +``` +modules/Microsoft.FinOpsHubs/AmazonWebServices/ + app.bicep + metadata.bicep + README.md +modules/Microsoft.FinOpsHubs/GoogleCloud/ + app.bicep + metadata.bicep + README.md +``` + +Publisher = `Microsoft.FinOpsHubs`, because Microsoft is who publishes the connector. This keeps the resources in the hub's own Data Factory / Key Vault / storage instead of creating a second Data Factory when `publisherIsolation` is enabled in the future (`hub-types.bicep`, `newApp`). + +### `app.bicep` header (required repository pattern) + +```bicep +import { finOpsToolkitVersion, HubAppProperties, privateRoutingForLinkedServices, isSupportedVersion } from '../../fx/hub-types.bicep' +import { AppMetadata as CoreMetadata } from '../Core/metadata.bicep' +import { AppMetadata as ExportsMetadata } from '../../Microsoft.CostManagement/Exports/metadata.bicep' + +metadata hubApp = { + id: 'Microsoft.FinOpsHubs.AmazonWebServices' + version: '$$ftkver$$' + dependencies: ['Microsoft.FinOpsHubs.Core', 'Microsoft.CostManagement.Exports'] +} + +@validate(x => isSupportedVersion(x.version, '13.0', ''), 'AWS FOCUS ingestion requires FinOps hubs version 13.0 or higher.') +param core CoreMetadata +``` + +The dependency on `Microsoft.CostManagement.Exports` is real and not optional: it owns the `msexports` container and the schema files. + +### Resources created per app + +| Type | Name (AWS) | Name (Google) | Purpose | +| --- | --- | --- | --- | +| Key Vault secret | `aws-secret-access-key` | `gcp-secret-access-key` | via `fx/hub-vault.bicep` | +| Linked service | `aws_s3` | `gcp_storage` | `AmazonS3` / `GoogleCloudStorage` | +| Dataset | `aws_focus_source` | `gcp_focus_source` | `Binary` + `AmazonS3Location` / `GoogleCloudStorageLocation` | +| Dataset | `aws_focus_landing` | `gcp_focus_landing` | `Binary` in the `msexports` container | +| Dataset | `aws_focus_manifest` | `gcp_focus_manifest` | `Json` in the `msexports` container | +| Pipeline | `aws_CollectFocusExport` | `gcp_CollectFocusExport` | copy + generate manifest | +| Trigger | `aws_DailySchedule` | `gcp_DailySchedule` | daily `ScheduleTrigger` | + +The linked service uses a Key Vault secret exactly like RemoteHub does (`Microsoft.FinOpsHubs/RemoteHub/app.bicep:88-98`): + +```bicep +resource linkedService_awsS3 'linkedservices' = { + name: 'aws_s3' + properties: { + type: 'AmazonS3' + typeProperties: { + authenticationType: 'AccessKey' + accessKeyId: awsAccessKeyId + secretAccessKey: { + type: 'AzureKeyVaultSecret' + store: { referenceName: app.keyVault, type: 'LinkedServiceReference' } + secretName: awsSecretSecretName + } + } + ...privateRoutingForLinkedServices(app.hub) + } +} +``` + +The `...privateRoutingForLinkedServices(app.hub)` spread is not optional — without it the linked service ignores the Managed VNet when the hub runs on a private network. + +### `*_CollectFocusExport` pipeline activities + +The pipeline is **manifest-driven**, not listing-driven. An earlier draft of this section proposed a `Get Metadata` (`childItems`) listing of the bucket prefix followed by a `Filter`. That approach is wrong and must not be used: in "create new" mode the `data/billing_period=YYYY-MM/` folder accumulates one subfolder per daily refresh, so a recursive listing copies every refresh and duplicates the month's costs. Only the provider's own manifest identifies the current run. See §10, design consequence #2. + +The pipeline is invoked once per collection period. A parent pipeline iterates the current and previous month (§10, "Collection window") and calls the child pipeline through `ExecutePipeline`, which also keeps each period in its own variable scope. + +1. **Set Run Id** — `@guid()`, used as `runInfo.runId` and as the destination subfolder. +2. **Set Billing Period** — derive `yyyy-MM` from `@addToTime(utcNow(), periodOffsetMonths, 'Month')`. +3. **Load Settings** — `Lookup` on the `config` dataset to read `retention.msexports.days`. +4. **Read Source Manifest** — `Lookup` on the provider manifest (`//metadata/billing_period=/-Manifest.json`) read **directly from S3/GCS**. This file is never copied into `msexports` — its schema is incompatible with the Cost Management contract (§10, design consequence #1). +5. **Copy FOCUS Files** — `ForEach` over `dataFiles` (parallel) with a binary `Copy` from S3/GCS to `msexports/////`. Each item is a full `s3://` URI and must have the `s3:///` prefix stripped to obtain the object key. +6. **Build Blob List** — `ForEach` (sequential) appending one `{ "blobName": "..." }` entry per copied file. Sequential because `AppendVariable` is not safe inside a parallel `ForEach`. `blobName` is the path **inside the `msexports` container**, matching how the ETL passes it to `msexports_parquet` as `blobPath`. +7. **Build Manifest** — `Set Variable` assembling the JSON contract from §2. Emit `blobCount = length(dataFiles)` and **omit `dataRowCount` entirely**; `exportConfig.resourceId` must be lowercase. +8. **Write Manifest** — `Copy` with a `JsonSink` writing `manifest.json` to the **same folder**, with `dependsOn: Succeeded` on steps 5 and 6. + +Step 8 depending on `Succeeded` (not `Completed`) is what guarantees the manifest is never published over a partial copy. Because the manifest lands last, it is also the signal that fires the existing `msexports_ManifestAdded` trigger — no new trigger wiring is needed on the ETL side. + +> **Data Factory constraint.** A container activity cannot contain another container activity. A `ForEach` or `Until` nested inside an `If` or `Switch` **deploys successfully but fails at runtime** with `Container activity cannot include another container activity`. Every loop above is therefore top level; conditional behavior is expressed by iterating an empty array (`@if(cond, json('[]'), realArray)`) rather than by wrapping the loop in an `If`. + +### Folder structure in `msexports` + +``` +msexports/ +├── aws////{data files, manifest.json} +└── gcp////{data files, manifest.json} +``` + +The `aws/` and `gcp/` prefixes isolate the sources and avoid collisions with Cost Management scope paths. The `` subfolder keeps concurrent or repeated runs of the same period from overwriting each other mid-copy; idempotency in the data lake is handled downstream by the `drop-by` tag mechanism described in §10, which keys on the stable `Costs/YYYY/MM/aws/` destination rather than on this staging path. + +--- + +## 6. Wiring in `modules/hub.bicep` + +Insert after the Invoices block (`hub.bicep:368-380`), following the same format: + +```bicep +//------------------------------------------------------------------------------ +// Multicloud FOCUS ingestion +//------------------------------------------------------------------------------ + +module awsFocus 'Microsoft.FinOpsHubs/AmazonWebServices/app.bicep' = if (enableAwsFocusIngestion) { + name: 'Microsoft.FinOpsHubs.AmazonWebServices' + params: { + app: newApp(hub, 'Microsoft.FinOpsHubs', 'AmazonWebServices') + core: core.outputs.metadata + exports: cmExports.outputs.metadata + bucketName: awsBucketName + // ... + } +} +``` + +Also add `awsFocus` / `googleFocus` to the `dependsOn` of the `startTriggers` module (`hub.bicep:423-442`, `dependsOn` array at `425-432`) — otherwise the new triggers stay stopped after deployment, because they are started by `Init-DataFactory.ps1`, which is called by `fx/hub-initialize.bicep`. + +--- + +## 7. File checklist + +| # | File | Action | +| --- | --- | --- | +| 1 | `src/templates/finops-hub/main.bicep` | + parameters, + passthrough | +| 2 | `src/templates/finops-hub/modules/hub.bicep` | + parameters, + 2 modules, + telemetry, + `dependsOn` | +| 3 | `src/templates/finops-hub/createUiDefinition.json` | + `multicloud` step, + outputs | +| 4 | `.../modules/Microsoft.FinOpsHubs/AmazonWebServices/{app,metadata}.bicep` + `README.md` | new | +| 5 | `.../modules/Microsoft.FinOpsHubs/GoogleCloud/{app,metadata}.bicep` + `README.md` | new | +| 6 | `.../Microsoft.CostManagement/Exports/schemas/focuscost_1.2-aws.json` | ✅ **created and validated** — 56 mappings, checked one by one against a real manifest (§10) and end to end in a deployed hub (§11) | +| 7 | `.../Microsoft.CostManagement/Exports/schemas/focuscost_1.2-gcp.json` | new (see R2) | +| 8 | `.../Microsoft.CostManagement/Exports/app.bicep` | ✅ **done** — AWS schema registered in the `files:` map | +| 9 | `src/templates/finops-hub/.build.config` | + 2 READMEs under `ignore` | +| 10 | `docs-mslearn/toolkit/hubs/template.md` | + rows in the parameter table | +| 11 | `docs-mslearn/toolkit/hubs/configure-multicloud.md` | new how-to | +| 12 | `docs-mslearn/toolkit/changelog.md` | **Added** entry under FinOps hubs | +| 13 | `src/powershell/Public/Deploy-FinOpsHub.ps1` | + equivalent parameters | +| 14 | `src/powershell/Tests/Unit/Deploy-FinOpsHub.Tests.ps1` | + test cases | + +Items 9, 10, and 12 are repository requirements, not optional: `.build.config` must ignore module READMEs (otherwise they ship in the Azure Quickstart Templates package), and the changelog has its own rules in `docs-wiki/Coding-guidelines.md`. + +Note on item 8: the schemas live in the `Microsoft.CostManagement.Exports` app because it owns the `msexports` container and publishes the `schemas/` folder. Alternative, if isolation is preferred: each multicloud app publishes its own schema via `fx/hub-storage.bicep` to the same path — this avoids touching the Exports app, at the cost of spreading the responsibility. + +--- + +## 8. Risks + +**R1 — `exportConfig.resourceId` — RESOLVED, low risk.** Spike complete. The real expression: + +``` +scope = split(toLower(exportConfig.resourceId), '/providers/microsoft.costmanagement/exports/')[0] +destination = replace(concat(hubDataset, '/', year, '/', month, '/', toLower(scope), ...), '//', '/') +``` + +`split()` with a missing delimiter returns the whole string in `[0]`. There is no resource ID decomposition, format validation, or lookup — the value is used only as a path segment, lowercased, with `//` collapsed. Any string works. The original hypothesis that an ARN would break the parsing was wrong. An Azure-format pseudo-scope is not needed; see §2.2. + +**R2 — FOCUS schema parity — RESOLVED, requires per-provider schemas.** Validated against a **real** AWS FOCUS file (snappy parquet, 60 columns, 19,827 rows). The file is **FOCUS 1.2**, not 1.0. + +Comparison with `focuscost_1.2.json` (104 mappings): + +| | Count | Note | +| --- | --- | --- | +| FOCUS columns matching exactly by name | 53 | reusable unchanged | +| Azure `x_*` columns missing from the AWS file | 51 | `x_BillingProfileId`, `x_SkuMeterId`, … — cannot be mapped | +| AWS columns missing from the hub schema | 7 | `AvailabilityZone`, `x_Operation`, `x_ServiceCode`, `x_Discounts`, 3× `PricingCurrency*` | + +So the existing schema **cannot be reused**. `schemas/focuscost_1.2-aws.json` was published with **56 mappings**: the 53 shared ones (types inherited from the hub's 1.2 schema) plus `AvailabilityZone`, `x_Operation`, and `x_ServiceCode` — all three confirmed as existing columns of the ADX `Costs_raw` table. + +Omitted because they do not exist in the ADX schema: `x_Discounts` (`map`) and `PricingCurrencyContractedUnitPrice` / `PricingCurrencyEffectiveCost` / `PricingCurrencyListUnitPrice`. Including them would require changing `IngestionSetup_RawTables.kql`, `HubSetup_v1_2.kql`, and the final tables — a change that also affects Azure data and is out of scope for this feature. Record it as a known gap in the README. + +**Correction to an earlier design error:** the schema file's `additionalColumns` **cannot** be used to stamp provenance. The ETL applies: + +``` +intersection( + [{"name":"x_SourceProvider","value":"Microsoft"}, {"name":"x_SourceName","value":"Cost Management"}, + {"name":"x_SourceType","value":""}, {"name":"x_SourceVersion","value":""}], + activity('Load Schema Mappings').output.firstRow.additionalColumns +) +``` + +(`Exports/app.bicep:1225`). Because it is an **intersection** with an array of values fixed to `Microsoft` / `Cost Management`, an object with `"value":"AWS"` never survives. And because **all** 14 schema files in the repository have `additionalColumns: []`, the intersection is always empty today — the ETL does not stamp `x_Source*` for any dataset. + +**Good consequence:** `dataVersion` does not leak into `x_SourceType` / `x_SourceVersion`. It is purely a schema-selection parameter with no side effects. The residual risk flagged earlier **does not exist**. + +**R3 — private networking (medium).** With `enablePublicAccess = false`, egress to S3/GCS depends on the ADF Managed VNet and the NAT Gateway (`enableNatGateway`). Document that private networking + multicloud requires `enableNatGateway = true`. + +**R4 — long-lived secrets (medium).** AWS/GCS access keys do not expire on their own. The secret is already created in Key Vault, but the README must require rotation and a minimum IAM policy (`s3:GetObject` + `s3:ListBucket` restricted to the prefix). + +**R5 — egress cost (low).** The transfer leaves the source provider and is billed by it. Document this alongside the cost estimate, as the Invoices README already does. + +--- + +## 8b. Spike results + +Run on `arthursilvany/multicloud-focus`, by static reading of the ETL and schemas, and validated against a **real** AWS FOCUS export. + +| Item | Initial hypothesis | Result | +| --- | --- | --- | +| R1 — `resourceId` parsing | High risk; could invalidate the approach | **Disproved.** It is just a path segment. Free format. | +| R2 — reusing `focuscost_1.2.json` | Probably reusable | **Disproved.** 51 of 104 columns are Azure-specific. Needs a per-provider schema. | +| Schema selection | Hardcoded in the ETL | **Better than expected.** Derived from `type` + `dataVersion`, and `dataVersion` is free and side-effect-free. | +| Provenance via `additionalColumns` | Ready-made mechanism | **Disproved.** The `intersection()` with fixed `Microsoft` values blocks it. See R2. | +| Multicloud provenance | Would need a new column | **Already solved upstream.** See §8c. | + +### 8c. ADX already supports AWS and GCP + +A finding that reduces the scope of the feature. `IngestionSetup_v1_0.kql:367-372` already classifies the provider from the shape of the data: + +```kusto +| extend ProviderName = case( + isnotempty(ProviderName), ProviderName, + isnotempty(coalesce(x_CostCategories, x_Discount, x_Operation, x_ServiceCode, x_UsageType)), 'AWS', + isnotempty(coalesce(tostring(UsageAmount), tostring(x_Cost), ..., x_Project, x_ServiceId)), 'GCP', + isnotempty(coalesce(x_BillingProfileId, x_InvoiceSectionId)), 'Microsoft', + '' +) +| extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) +| extend x_SourceVersion = coalesce(x_SourceVersion, case(...)) +``` + +The `Costs_raw` table already declares `x_Operation` and `x_ServiceCode` with the comment `// AWS 1.0`, and `AvailabilityZone` as `// FOCUS 0.5+`. The real AWS file arrives with `ProviderName = 'AWS'` already populated, so classification succeeds on the first branch. + +**Conclusion: no provenance work is required.** Simply delivering the data into `Costs_raw` is enough — ADX classifies, versions, and routes it on its own. + +### 8d. Constraints fixed by table routing + +`Analytics/app.bicep:1815` derives the destination table from the **first folder segment** of the ingestion path: + +``` +table = concat(first(split(containerFolderPath, '/')), '_raw') +``` + +and `Exports/app.bicep:847` maps `exportDatasetType = 'focuscost'` → `hubDataset = 'Costs'`, falling back to the type name itself. Therefore: + +| Manifest field | Required value | Reason | +| --- | --- | --- | +| `exportConfig.type` | **`FocusCost`** (exact) | any other value produces a `` folder and a `_raw` table, which does not exist | +| `exportConfig.dataVersion` | free | only selects the schema file | + +Using a suffix in `dataVersion` already has precedent in the repository: `focuscost_1.0-preview(v1).json` and `focuscost_1.2-preview.json`. + +--- + +## 9. Execution order + +1. ~~R1 and R2 spikes~~ — **complete**, see §8b. +2. ~~Write `focuscost_1.2-aws.json` and validate it against a real AWS FOCUS file~~ — **complete**, see §10 and §11. +3. Complete AWS app (Bicep + README). +4. Google app, reusing the validated format. +5. UI, PowerShell parameters, and tests. +6. Documentation and changelog. + +Validation at each stage: + +```powershell +az bicep build --file src/templates/finops-hub/main.bicep --stdout +./src/scripts/Build-Toolkit finops-hub +./src/scripts/Deploy-Toolkit finops-hub -Build -WhatIf +./src/scripts/Test-PowerShell -Lint -Hubs +``` + +The first mandatory regression test is a deployment **with both flags off**, comparing the what-if against the baseline: the result must be empty. + +--- + +## 10. AWS source topology (BCM Data Exports) + +Confirmed in the [AWS documentation](https://docs.aws.amazon.com/cur/latest/userguide/dataexports-export-delivery.html). + +### S3 layout + +``` +s3://///data/billing_period=YYYY-MM/ # "overwrite" mode +s3://///data/billing_period=YYYY-MM/-/ # "create new" mode +s3://///metadata/billing_period=YYYY-MM/-Manifest.json +``` + +Files: `-.snappy.parquet` or `-.csv.gz`, where `chunk` is a 5-digit number starting at `00001`. + +> The AWS documentation spells the partition key as `BILLING_PERIOD=`, but the real export delivers lowercase `billing_period=`. The pipeline must not write that literal: the path always comes from the manifest's `dataFiles` field. + +### Three design consequences + +**1. The AWS `Manifest.json` must not be copied into `msexports`.** Its schema is completely different from the Cost Management contract (§2) — if the trigger picked it up, `Read Manifest` would return nulls and the ETL would fail. It must be **read** by a `Lookup` directly against S3 and never written to the `msexports` container. Only the synthetic manifest we build is written there. + +Mitigating factor: the AWS file is named `-Manifest.json`, and the trigger filters on `storagePathEndsWith: 'manifest.json'`. Even so, Event Grid case sensitivity must not be relied on as a safety mechanism — the rule is simply not to copy it. + +**2. Reading the AWS manifest is mandatory, not optional.** In "create new" mode, the `data/billing_period=YYYY-MM/` folder accumulates **one subfolder per daily refresh**. A recursive `Get Metadata` would copy them all and duplicate the month's costs. Only the `Manifest.json` at the `metadata//` level identifies the current run. + +**3. The manifest is the completeness signal.** AWS publishes it only after all data files have landed — it is the exact equivalent of the Cost Management `manifest.json` and removes the need for any "stable file" heuristic. + +### Collection window + +AWS may update the previous period for up to two weeks after it closes. The pipeline must iterate over **two** periods per run — the current month and the previous month — not just the current one. + +### Idempotency: why a daily refresh does not duplicate data + +`ingestionId = runInfo.runId` from the synthetic manifest, and the ADX post-ingestion cleanup (`Analytics/app.bicep:1423`) removes extents tagged `drop-by:` but **not** `drop-by:`. In other words: by generating a new `runId` on every run and keeping the destination path stable per period (`Costs/YYYY/MM/aws/`), each refresh **replaces** the whole month instead of accumulating. This is the same mechanism used by Azure exports. + +#### The `drop-by` tag is case-sensitive — and includes the file name + +Two details of the mechanism above were confirmed empirically and constrain the format of `exportConfig.resourceId`: + +1. **The tag carries the full blob path, including the file name** — not just the folder. For example: `drop-by:Costs/2026/05/aws/390402570720/2026-05-20T22_19_25.548Z-_CCOE-PRODAM-AWS-00001.snappy.parquet`. Because the AWS file name embeds `-`, it **changes on every refresh**. Replacement only works because the ADX cleanup matches on the folder prefix; any change to the path format breaks idempotency. +2. **The comparison is case-sensitive.** The ETL applies `toLower()` to the scope (§8/R1), so `resourceId: /aws/390402570720` always writes to `Costs/YYYY/MM/aws/390402570720/`. A manual load into `.../AWS/390402570720/` produces a distinct tag and both sets **coexist**, doubling the cost. + +Observed in the lab: after ingesting the same parquet through both routes, `Costs_final_v1_2` held 39,654 rows (2 × 19,827) and USD 164,511.26 (2 × USD 82,264.63), under two tags that differ only by `AWS` vs `aws`: + +``` +drop-by:Costs/2026/05/AWS/390402570720/.snappy.parquet +drop-by:Costs/2026/05/aws/390402570720/.snappy.parquet +``` + +**Design consequence:** `exportConfig.resourceId` must **always** be emitted in lowercase by the `AmazonWebServices`/`GoogleCloud` modules. Any case variation in the account or provider identifier produces silent duplication — no error, no alert, just double the cost. + +### The exact contract consumed by the ETL, revisited + +| Real ETL expression | Implication for the synthetic manifest | +| --- | --- | +| `replace(substring(runInfo.startDate, 0, 7), '-', '')` | `startDate` must be ISO: `YYYY-MM-01T00:00:00Z`. Derivable from the `billing_period=YYYY-MM` segment present in `dataFiles[0]`. | +| `last(split(blobs[0].blobName, '.'))` | selects the dataset by extension: `parquet`, `gz`, or `csv`. `*.snappy.parquet` resolves to `parquet`. | +| `ForEach(blobs)` → `item().blobName` | `blobName` is the path **inside the `msexports` container**, not the S3 key. | +| `last(split(replace(replace(blobName,'.gz',''),'.csv','.parquet'), '/'))` | destination file name; preserves `.snappy.parquet`. | +| `blobCount` / `dataRowCount` | if `blobCount` is zero or null the ETL short-circuits. Populate it with `length(dataFiles)`. `dataRowCount` is only evaluated **if the property exists** (`contains(firstRow, 'dataRowCount')`) — since the AWS manifest carries no row count, the property must be **omitted**, never written as `0`. | + +### The AWS `Manifest.json` contract — RESOLVED + +Fixed against a **real** manifest (`CCOE-PRODAM-AWS`, period `2026-05`): + +```json +{ + "executionId": "9b6b32ef-2b70-3d10-9448-07e23ebce6b9", + "exportArn": "arn:aws:bcm-data-exports:us-east-1::export/-", + "columns": [ { "name": "AvailabilityZone", "type": "string" }, ... ], + "dataFiles": [ + "s3://///data/billing_period=2026-05/2026-06-01T23:48:02.356Z-9b6b32ef-2b70-3d10-9448-07e23ebce6b9/-00001.snappy.parquet" + ], + "additionalOutputFiles": [] +} +``` + +Six points the `Lookup` and the synthetic manifest must respect: + +| Confirmed fact | Consequence | +| --- | --- | +| The field is **`dataFiles`**, not `files`. | Settles the disagreement between public sources. | +| Each item is a **full `s3://` URI**, not a relative key. | The `Copy` must strip `s3:///` to obtain the object key. Do not apply the dataset's `bucketName` on top of the raw URI. | +| **There is no period field** in the manifest. | `billing_period` is derived from the path in `dataFiles[0]` (or from the partition the manifest was read from), and the synthetic manifest's ISO `startDate` follows from it. | +| **There is no row count.** | Omit `dataRowCount` from the synthetic manifest (see the table above). `blobCount = length(dataFiles)`. | +| The path contains `-`, and `executionId` is in the manifest itself. | Confirms design consequence #2: the customer is in "create new" mode and the folder accumulates one refresh per day. Copying only what is listed in `dataFiles` is mandatory. | +| `columns` carries the name and type of every delivered column. | This gives a free schema-drift detector: compare against the 56 mappings in `focuscost_1.2-aws.json` before copying, and fail with a clear message instead of silently ingesting truncated data. | + +`additionalOutputFiles` was empty in this export; the pipeline should ignore it. + +### Schema validation against the real manifest + +The manifest's 60 column names were compared one by one against `focuscost_1.2-aws.json`: + +- **56 of 56 schema mappings exist in the manifest**, with compatible types (`string`→`String`, `double`→`Decimal`, `timestamp`→`DateTimeOffset`). +- **No schema column is missing** from the AWS file. +- The 4 manifest columns without a mapping are exactly the omissions already documented in R2: `x_Discounts`, `PricingCurrencyContractedUnitPrice`, `PricingCurrencyEffectiveCost`, and `PricingCurrencyListUnitPrice`. +- `Tags` and `SkuPriceDetails` arrive as parquet `map` columns and are mapped as `String`. **Validated in a real deployment** (§11): the `TabularTranslator` serializes the map as JSON and ADX materializes it as `dynamic`. The type risk is closed. + +--- + +## 11. End-to-end validation in a real hub + +`focuscost_1.2-aws.json` was exercised against a deployed FinOps hub v14, using the real AWS FOCUS parquet file (19,827 rows, USD 82,264.63, May 2026, 16 sub-accounts, 57 services). + +**Method.** Because the only ETL trigger is a `manifest.json` landing in `msexports`, a temporary Data Factory pipeline published the schema to `config/schemas/`, copied the parquet into `msexports//`, and wrote a synthetic manifest with `exportConfig.type = FocusCost`, `dataVersion = 1.2-aws`, and `resourceId = /aws/-test`. The whole test was driven from Data Factory because the hub storage account has `publicNetworkAccess: Disabled`. + +**Result — the full chain succeeded:** + +| Pipeline | Status | What it proves | +| --- | --- | --- | +| `msexports_ExecuteETL` | Succeeded | The trigger accepts a synthetic manifest; routing by `exportDatasetType` works. | +| `msexports_ETL_ingestion` → `Load Schema Mappings` | Succeeded | `toLower('FocusCost_1.2-aws.json')` resolves to `focuscost_1.2-aws.json`. The extension point from §2.1 works as designed. | +| `msexports_ETL_ingestion` → `Convert to Parquet` | Succeeded | The 56 `TabularTranslator` mappings are valid against the real AWS parquet, including the `map` columns. | +| `ingestion_ExecuteETL` → `ingestion_ETL_dataExplorer` | Succeeded | ADX ingestion and the `Costs_raw` → `Costs_final_v1_2` transforms accept AWS data unchanged. | + +**ADX verification:** all 19,827 rows were ingested, with `ProviderName = "AWS"`, `Tags` materialized as `dynamic` (`{"map-migrated":"mig656O1TB0TE"}`), `ServiceCategory` correctly enriched, and non-empty cell counts identical to the native load. + +**Conclusion:** `focuscost_1.2-aws.json` is validated. The remaining work is exclusively collection — the `AmazonWebServices`/`GoogleCloud` modules that read S3/GCS and write the synthetic manifest (§5). Nothing in the existing ETL needs to change. + +**Note on `Get Existing Parquet Files`:** this activity fails with `PathNotFound` when the destination folder does not exist yet, and that is handled — the pipeline still completes successfully. It is not a symptom of a problem in the multicloud path. diff --git a/docs-wiki/design/Multicloud-FOCUS-plan.md b/docs-wiki/design/Multicloud-FOCUS-plan.md new file mode 100644 index 000000000..ed8e9f8f2 --- /dev/null +++ b/docs-wiki/design/Multicloud-FOCUS-plan.md @@ -0,0 +1,120 @@ +# Plan: make AWS and Google FOCUS ingestion an optional FinOps hub setup step + +> Scope, rationale, and high-level decisions for the feature. The technical detail — manifest contract, Bicep parameters, module structure, and file checklist — lives in [Multicloud-FOCUS-design](./Multicloud-FOCUS-design.md). +> Working branch: `arthursilvany/multicloud-focus`. + +## Problem + +Add optional support for ingesting FOCUS data from AWS and Google (GCP) during FinOps hub setup, without turning it into a requirement of the default deployment. The goal is to let an Azure hub deployment also receive multicloud data through a controlled ingestion flow, following the template's existing model for optional extensions. + +## References reviewed + +- +- +- Current repository: `src/templates/finops-hub/main.bicep`, `src/templates/finops-hub/modules/hub.bicep`, `src/templates/finops-hub/createUiDefinition.json`, `src/templates/finops-hub/modules/Microsoft.FinOpsHubs/RemoteHub/app.bicep` + +## Findings + +- The current template already uses an optional-flag pattern for setup extensions (`enableManagedExports`, `enableRecommendations`, `remoteHubStorageUri`, `remoteHubStorageKey`). +- `createUiDefinition.json` shows that the Azure portal UI already exposes optional settings in advanced sections for remote hub scenarios. +- `Microsoft.FinOpsHubs/RemoteHub/app.bicep` is the best reference pattern in the repository for: (1) connecting an external resource to the hub's Data Factory; (2) overriding datasets; and (3) keeping the hub working as centralized ingestion. +- Microsoft documentation covers multicloud/remote hub for Azure, but does not yet model AWS and Google as native, optional FinOps hub installer settings. There is a deployment-experience gap between "Azure-only defaults" and "multicloud custom ingestion". + +## Direction + +1. Keep the default FinOps hub deployment fully Azure-first, with no behavior changes by default. +2. Add an optional "Multicloud" step to the wizard, with separate toggles for AWS and Google. +3. Reuse the hub's Data Factory as the orchestrator for collecting external FOCUS files instead of creating a parallel deployment. +4. Treat AWS/Google as extra data sources, not as required hub resources. + +## Architectural revision after reading the code (key decision) + +The full analysis is in [Multicloud-FOCUS-design](./Multicloud-FOCUS-design.md). Its conclusion changed the original design: + +**Do not build a parallel ETL.** The hub already has the complete `msexports → Parquet → ingestion → ADX` chain, and it is fired by a `BlobEventsTrigger` that reacts **exclusively** to `manifest.json` (`Microsoft.CostManagement/Exports/app.bicep:1712-1728`). The FOCUS 1.0/1.0r2/1.2 schemas and the CSV, gzip, and Parquet datasets already exist. + +So the multicloud connector only needs to: + +1. copy the FOCUS files from the S3/GCS bucket into `msexports//...`; +2. write a Cost Management-compatible `manifest.json` **last**. + +Everything after that — conversion, retention, and analytical ingestion — already works unchanged. This drastically reduces new code and maintenance cost. + +Reference pattern to follow: `Microsoft.Billing/Invoices`, the newest and most complete optional app in the repository (`enableInvoiceDownload` parameter + conditional module + UI step + README + `.build.config` entry). + +## Spike results (complete) + +Run on the `arthursilvany/multicloud-focus` branch. Details in [Multicloud-FOCUS-design](./Multicloud-FOCUS-design.md) §8b. + +- **R1 — `exportConfig.resourceId`: disproved.** The ETL runs `split(toLower(resourceId), '/providers/microsoft.costmanagement/exports/')[0]` and uses the result only as a path segment. There is no resource ID parsing. Any string works — `/aws/` and `/gcp/` were adopted. Risk dropped from high to low. +- **R2 — FOCUS schema reuse: confirmed as a problem, and solved.** The hub's FOCUS schema is dominated by Azure-specific `x_*` columns and is not reusable. Solution: publish per-provider schemas and select them through the synthetic manifest's `exportConfig.dataVersion` field, since the schema file name is derived from `type` + `dataVersion` — both under our control. **Zero ETL changes.** +- **Bonus:** the schema file's `additionalColumns` field, empty today, looked like a ready-made mechanism for stamping `x_SourceProvider` to distinguish data origin in ADX. + +> **Superseded by the design doc.** Two of the statements above were later corrected: the real AWS export is FOCUS **1.2**, so the schema is `focuscost_1.2-aws.json` (not `1.0-aws`); and `additionalColumns` **cannot** stamp provenance, because the ETL intersects it with a fixed `Microsoft` / `Cost Management` array. Provenance turned out to require no work at all — ADX already classifies AWS and GCP from the shape of the data. See design §8/R2, §8b, and §8c. + +Conclusion: the synthetic manifest approach is validated. No blockers remain for starting implementation. + +## Proposed scope + +### 1) Setup configuration + +Add optional parameters to the main template and the portal UI: + +- `enableAwsFocusIngestion` (bool, default false) +- `enableGoogleFocusIngestion` (bool, default false) +- `awsFocusBucketName` / `awsFocusPrefix` / `awsFocusRegion` +- `awsFocusAccessKeySecretName` or `awsFocusCredentialsSecretName` +- `googleFocusBucketUri` / `googleFocusPrefix` / `googleProjectId` +- `googleFocusCredentialsSecretName` +- `focusIngestionSchedule` or `triggerFrequency` + +These inputs must be conditional: they are only visible when the matching source is enabled. + +### 2) Security and secrets + +- Use Key Vault to store AWS/Google credentials instead of exposing secrets in the template. +- Model the property as Key Vault references, similar to the `remoteHubStorageKey`/`AzureKeyVaultSecret` pattern. +- Validate that credentials are optional and that configuration fails explicitly when the flag is enabled but the secret or URI is missing. + +### 3) Data Factory extension + +Add Bicep modules to the hub flow to: + +- create the AWS and/or Google source `linkedService` +- create source and destination datasets +- create copy pipelines that move FOCUS files from external storage into the FinOps hub +- create a periodic trigger (daily/hourly, depending on the source) +- preserve the current `startTriggers` logic via `fx/hub-initialize.bicep` + +The extension must follow the same pattern already used in `Microsoft.FinOpsHubs/RemoteHub/app.bicep`. + +### 4) Ingestion and normalization + +- Ensure the final destination is compatible with the model the hub already uses for FOCUS/Parquet files and analytical ingestion. +- Define folder structure and naming rules to avoid collisions between AWS, Google, and Azure sources. +- Validate that the provider's FOCUS schema is consistent with what the hub expects, and whether a transformation layer is needed before final ingestion. + +### 5) Testing, validation, and documentation + +- Validate the template deployment with the flags off (baseline unchanged). +- Validate the deployment with each source enabled separately. +- Validate `bicep build`/deployment validation and the repository tests (`src/scripts/Test-PowerShell`). +- Update the FinOps hub and deployment portal documentation to explain the optional multicloud flow. + +## Risks and considerations + +- AWS/Google integration topology tends to depend on provider-specific services, APIs, and credentials; the solution must be a connector pattern, not a single hardcoded scenario. +- Ingestion cost and latency increase with external connections; the feature must be explicitly optional and documented. +- The hub assumes Azure-first data; the multicloud extension must be isolated so it does not affect the default installation. +- Google/AWS configurations may require distinct network and egress rules; this must be addressed in the template design. + +## Planned tasks + +- `multicloud-research`: confirm the technical scope and choose the extension model most compatible with the current hub. +- `multicloud-design`: define installer parameters, UI, and configuration object. +- `multicloud-iac`: specify the hub Bicep and the Data Factory pattern for AWS/Google. +- `multicloud-docs`: validate documentation, tests, and rollout. + +## Expected outcome + +A FinOps hub installation that keeps working as Azure-first by default, but offers the option to collect FOCUS data from AWS and Google as complementary sources, with security, isolation, and compatibility with the hub's current flow. diff --git a/src/powershell/Public/Add-FinOpsHubBillingReader.ps1 b/src/powershell/Public/Add-FinOpsHubBillingReader.ps1 new file mode 100644 index 000000000..35d318608 --- /dev/null +++ b/src/powershell/Public/Add-FinOpsHubBillingReader.ps1 @@ -0,0 +1,151 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Grants the Billing Reader role to a FinOps hub managed identity on a billing account. + + .DESCRIPTION + The Add-FinOpsHubBillingReader command grants the Billing Reader role to the Data Factory managed identity of a FinOps hub instance at the billing account scope. + + This role is required to download Microsoft invoice files and cannot be granted during deployment because billing account scopes are outside of any Azure subscription. + + .PARAMETER BillingAccountId + Required. ID of the billing account to grant access to. Find the ID in Cost Management + Billing > Properties. + + .PARAMETER HubName + Optional. Name of the FinOps hub instance. Supports wildcards. Default: * (all hubs in the selected subscription). + + .PARAMETER ResourceGroupName + Optional. Name of the resource group the FinOps hub was deployed to. Supports wildcards. Default: * (all resource groups). + + .EXAMPLE + Add-FinOpsHubBillingReader -BillingAccountId '12345678-1234-1234-1234-123456789012:87654321-4321-4321-4321-210987654321_2019-05-31' + + Grants the Billing Reader role to the managed identity of the only FinOps hub in the selected subscription. + + .EXAMPLE + Add-FinOpsHubBillingReader -BillingAccountId 1234567 -HubName foo -ResourceGroupName bar + + Grants the Billing Reader role to the managed identity of the 'foo' hub in the 'bar' resource group. + + .LINK + https://aka.ms/ftk/Add-FinOpsHubBillingReader +#> +function Add-FinOpsHubBillingReader +{ + [CmdletBinding(SupportsShouldProcess, ConfirmImpact = 'Medium')] + param + ( + [Parameter(Mandatory, Position = 1)] + [ValidateNotNullOrEmpty()] + [string] + $BillingAccountId, + + [Parameter()] + [string] + $HubName, + + [Parameter()] + [string] + $ResourceGroupName + ) + + $ErrorActionPreference = 'Stop' + + $context = Get-AzContext + if (-not $context) + { + throw $script:LocalizedData.Common_ContextNotFound + } + + # Billing account IDs are used verbatim in the scope, but tolerate a full scope being passed in + $accountId = $BillingAccountId.Trim('/') + if ($accountId -like '*/billingAccounts/*') + { + $accountId = $accountId.Substring($accountId.LastIndexOf('/billingAccounts/') + '/billingAccounts/'.Length) + } + + $scope = "/providers/Microsoft.Billing/billingAccounts/$accountId" + + # Find the Data Factory instances that belong to the hub + $hubs = Get-FinOpsHub -Name $HubName -ResourceGroupName $ResourceGroupName + $dataFactories = @($hubs.Resources | Where-Object { $_.ResourceType -eq 'Microsoft.DataFactory/factories' }) + + if ($dataFactories.Count -eq 0) + { + throw ($script:LocalizedData.HubBillingReader_Add_DataFactoryNotFound -f $(if ($HubName) { $HubName } else { '*' })) + } + + if ($dataFactories.Count -gt 1) + { + throw ($script:LocalizedData.HubBillingReader_Add_MultipleDataFactories -f $(if ($HubName) { $HubName } else { '*' })) + } + + $dataFactory = Get-AzDataFactoryV2 -ResourceGroupName $dataFactories[0].ResourceGroupName -Name $dataFactories[0].Name + $principalId = $dataFactory.Identity.PrincipalId + if (-not $principalId) + { + throw ($script:LocalizedData.HubBillingReader_Add_IdentityNotFound -f $dataFactories[0].Name) + } + + $apiVersion = '2024-04-01' + $billingAccountUri = "providers/Microsoft.Billing/billingAccounts/$accountId" + + # Billing account scopes are not part of Azure RBAC, so the billing role assignment API must be + # used instead of New-AzRoleAssignment. Role definition IDs differ per agreement type, so the + # reader role is looked up by name and only falls back to the documented MCA role definition. + $roleDefinitionId = $null + $roleDefinitions = Invoke-Rest -Method GET -Uri "$billingAccountUri/billingRoleDefinitions?api-version=$apiVersion" -CommandName 'Add-FinOpsHubBillingReader' + if ($roleDefinitions.Success) + { + $readerRole = $roleDefinitions.Content.value ` + | Where-Object { $_.properties.roleName -in @('Billing account reader', 'Billing Reader', 'Reader') } ` + | Select-Object -First 1 + if ($readerRole) + { + $roleDefinitionId = $readerRole.id + } + } + + if (-not $roleDefinitionId) + { + # Billing account reader for a Microsoft Customer Agreement + $roleDefinitionId = "/$billingAccountUri/billingRoleDefinitions/50000000-aaaa-bbbb-cccc-100000000002" + } + + $assignments = Invoke-Rest -Method GET -Uri "$billingAccountUri/billingRoleAssignments?api-version=$apiVersion" -CommandName 'Add-FinOpsHubBillingReader' + if ($assignments.Success) + { + $existing = $assignments.Content.value ` + | Where-Object { $_.properties.principalId -eq $principalId -and $_.properties.roleDefinitionId -eq $roleDefinitionId } ` + | Select-Object -First 1 + if ($existing) + { + Write-Verbose ($script:LocalizedData.HubBillingReader_Add_AlreadyAssigned -f $accountId) + return $existing + } + } + + if (-not $PSCmdlet.ShouldProcess($scope, 'Grant Billing Reader')) + { + return + } + + $body = [PSCustomObject]@{ + properties = [PSCustomObject]@{ + principalId = $principalId + principalTenantId = $context.Tenant.Id + roleDefinitionId = $roleDefinitionId + } + } + + $response = Invoke-Rest -Method PUT -Uri "$billingAccountUri/billingRoleAssignments/$((New-Guid).Guid)?api-version=$apiVersion" -Body $body -CommandName 'Add-FinOpsHubBillingReader' + if (-not $response.Success) + { + throw ($script:LocalizedData.HubBillingReader_Add_AssignFailed -f $accountId, $response.Content.error.message) + } + + Write-Verbose ($script:LocalizedData.HubBillingReader_Add_Assigned -f $accountId) + return $response.Content +} diff --git a/src/powershell/Public/Add-FinOpsHubResourceGraphReader.ps1 b/src/powershell/Public/Add-FinOpsHubResourceGraphReader.ps1 new file mode 100644 index 000000000..df7adc206 --- /dev/null +++ b/src/powershell/Public/Add-FinOpsHubResourceGraphReader.ps1 @@ -0,0 +1,128 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + .SYNOPSIS + Grants the Reader role to a FinOps hub managed identity on Resource Graph scopes. + + .DESCRIPTION + The Add-FinOpsHubResourceGraphReader command grants the Reader role to the Data Factory + managed identity of a FinOps hub instance. The role is required for the hub to run + Azure Resource Graph recommendation queries, including Azure Advisor cost recommendations. + + .PARAMETER Scope + Required. Subscription ID, subscription resource ID, or management group resource ID + where the hub managed identity should be granted Reader access. Multiple scopes are supported. + + .PARAMETER HubName + Optional. Name of the FinOps hub instance. Supports wildcards. Default: * (all hubs in the selected subscription). + + .PARAMETER ResourceGroupName + Optional. Name of the resource group the FinOps hub was deployed to. Supports wildcards. + Default: * (all resource groups). + + .EXAMPLE + Add-FinOpsHubResourceGraphReader -Scope '00000000-0000-0000-0000-000000000000' -HubName 'finops-hub14' + + Grants the Reader role at the subscription scope to the managed identity of the matching hub. + + .EXAMPLE + Add-FinOpsHubResourceGraphReader -Scope '/providers/Microsoft.Management/managementGroups/contoso' -HubName 'finops-hub' + + Grants the Reader role at the management group scope. + + .LINK + https://aka.ms/ftk/Add-FinOpsHubResourceGraphReader +#> +function Add-FinOpsHubResourceGraphReader +{ + [CmdletBinding(SupportsShouldProcess, ConfirmImpact = 'Medium')] + param + ( + [Parameter(Mandatory, Position = 1)] + [ValidateNotNullOrEmpty()] + [string[]] + $Scope, + + [Parameter()] + [string] + $HubName, + + [Parameter()] + [string] + $ResourceGroupName + ) + + $ErrorActionPreference = 'Stop' + + if (-not (Get-AzContext)) + { + throw $script:LocalizedData.Common_ContextNotFound + } + + $normalizedScopes = @( + foreach ($scopeValue in $Scope) + { + $scopeValue = $scopeValue.Trim().TrimEnd('/') + if ($scopeValue -match '^[0-9a-fA-F-]{36}$') + { + $scopeValue = "/subscriptions/$scopeValue" + } + elseif ($scopeValue -notmatch '^/subscriptions/[^/]+$' -and + $scopeValue -notmatch '^/providers/Microsoft\.Management/managementGroups/[^/]+$') + { + throw ($script:LocalizedData.HubResourceGraphReader_InvalidScope -f $scopeValue) + } + + $scopeValue + } + ) | Select-Object -Unique + + $hubs = @(Get-FinOpsHub -Name $HubName -ResourceGroupName $ResourceGroupName) + $dataFactories = @( + $hubs.Resources | Where-Object { $_.ResourceType -eq 'Microsoft.DataFactory/factories' } + ) + + if ($dataFactories.Count -eq 0) + { + throw ($script:LocalizedData.HubResourceGraphReader_DataFactoryNotFound -f $(if ($HubName) { $HubName } else { '*' })) + } + + if ($dataFactories.Count -gt 1) + { + throw ($script:LocalizedData.HubResourceGraphReader_MultipleDataFactories -f $(if ($HubName) { $HubName } else { '*' })) + } + + $dataFactory = Get-AzDataFactoryV2 -ResourceGroupName $dataFactories[0].ResourceGroupName -Name $dataFactories[0].Name + $principalId = $dataFactory.Identity.PrincipalId + if (-not $principalId) + { + throw ($script:LocalizedData.HubResourceGraphReader_IdentityNotFound -f $dataFactories[0].Name) + } + + foreach ($scopeValue in $normalizedScopes) + { + $existing = @(Get-AzRoleAssignment -ObjectId $principalId -RoleDefinitionName 'Reader' -Scope $scopeValue) + if ($existing.Count -gt 0) + { + Write-Verbose ($script:LocalizedData.HubResourceGraphReader_AlreadyAssigned -f $scopeValue) + $existing | Select-Object -First 1 + continue + } + + if ($PSCmdlet.ShouldProcess($scopeValue, 'Grant Reader to the FinOps hub managed identity')) + { + try + { + $assignment = New-AzRoleAssignment -ObjectId $principalId -RoleDefinitionName 'Reader' -Scope $scopeValue + } + catch + { + throw ($script:LocalizedData.HubResourceGraphReader_AssignFailed -f $scopeValue, $_.Exception.Message) + } + + Write-Verbose ($script:LocalizedData.HubResourceGraphReader_Assigned -f $scopeValue) + $assignment + } + } +} diff --git a/src/powershell/Public/Deploy-FinOpsHub.ps1 b/src/powershell/Public/Deploy-FinOpsHub.ps1 index 6140e0af8..dd0317783 100644 --- a/src/powershell/Public/Deploy-FinOpsHub.ps1 +++ b/src/powershell/Public/Deploy-FinOpsHub.ps1 @@ -84,6 +84,30 @@ .PARAMETER DataExplorerFinalRetentionInMonths Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. Default: 13. + .PARAMETER EnableAwsFocusIngestion + Optional. Enable ingestion of FOCUS cost data exported from Amazon Web Services. Requires an S3 bucket with a FOCUS 1.2 export and an access key. Default: false. + + .PARAMETER AwsBucketName + Optional. Name of the Amazon S3 bucket that contains the FOCUS export. Requires EnableAwsFocusIngestion. + + .PARAMETER AwsBucketPath + Optional. Path to the export root folder within the S3 bucket, without leading or trailing slashes. This is the folder that contains the "data" and "metadata" subfolders. Example: "reports/focus-export". Requires EnableAwsFocusIngestion. + + .PARAMETER AwsAccountId + Optional. Amazon Web Services account ID that owns the FOCUS export. Requires EnableAwsFocusIngestion. + + .PARAMETER AwsRegion + Optional. Amazon Web Services region of the S3 bucket. Leave empty to use the global S3 endpoint. Requires EnableAwsFocusIngestion. Default: "" (global). + + .PARAMETER AwsAccessKeyId + Optional. Amazon Web Services access key ID used to read the S3 bucket. Requires EnableAwsFocusIngestion. + + .PARAMETER AwsSecretAccessKey + Optional. Amazon Web Services secret access key used to read the S3 bucket. Stored in Key Vault. Requires EnableAwsFocusIngestion. + + .PARAMETER MultiCloudScheduleHour + Optional. Hour of the day (UTC) to collect multicloud FOCUS files. Default: 4. + .PARAMETER NetworkMode Optional. Network mode for the hub: 'public' (default), 'vnet' (private endpoints, default outbound), or 'private' (private endpoints + NAT Gateway for controlled outbound access - required when the 'Subnets should be private' policy is enforced). @@ -223,7 +247,40 @@ function Deploy-FinOpsHub [Parameter()] [ValidateRange(0, 999)] [int] - $IngestionRetentionInMonths = 13 + $IngestionRetentionInMonths = 13, + + [Parameter()] + [switch] + $EnableAwsFocusIngestion, + + [Parameter()] + [string] + $AwsBucketName, + + [Parameter()] + [string] + $AwsBucketPath, + + [Parameter()] + [string] + $AwsAccountId, + + [Parameter()] + [string] + $AwsRegion, + + [Parameter()] + [string] + $AwsAccessKeyId, + + [Parameter()] + [securestring] + $AwsSecretAccessKey, + + [Parameter()] + [ValidateRange(0, 23)] + [int] + $MultiCloudScheduleHour = 4 ) # Initialize toolkitPath before try block to ensure cleanup works even if early failure occurs @@ -329,6 +386,20 @@ function Deploy-FinOpsHub $parameterSplat.TemplateParameterObject.Add('enableNatGateway', $true) } + # Only pass the multicloud parameters when the feature is requested. Leaving them out + # keeps deployments compatible with template versions that predate the parameters. + if ($EnableAwsFocusIngestion -and ($Version -eq 'latest' -or [version]$Version -ge '15.0')) + { + $parameterSplat.TemplateParameterObject.Add('enableAwsFocusIngestion', $true) + $parameterSplat.TemplateParameterObject.Add('awsBucketName', $AwsBucketName) + $parameterSplat.TemplateParameterObject.Add('awsBucketPath', $AwsBucketPath) + $parameterSplat.TemplateParameterObject.Add('awsAccountId', $AwsAccountId) + $parameterSplat.TemplateParameterObject.Add('awsRegion', $AwsRegion) + $parameterSplat.TemplateParameterObject.Add('awsAccessKeyId', $AwsAccessKeyId) + $parameterSplat.TemplateParameterObject.Add('awsSecretAccessKey', $AwsSecretAccessKey) + $parameterSplat.TemplateParameterObject.Add('multiCloudScheduleHour', $MultiCloudScheduleHour) + } + if ($Tags -and $Tags.Keys.Count -gt 0) { $parameterSplat.TemplateParameterObject.Add('tags', $Tags) diff --git a/src/powershell/Tests/Lint/DataFactorySettings.Tests.ps1 b/src/powershell/Tests/Lint/DataFactorySettings.Tests.ps1 new file mode 100644 index 000000000..cb15535ca --- /dev/null +++ b/src/powershell/Tests/Lint/DataFactorySettings.Tests.ps1 @@ -0,0 +1,84 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + Lint rule: Data Factory settings that compile and deploy but fail at runtime. + + Both rules below were found by deploying the AWS FOCUS connector to a live hub. Neither is + caught by `bicep build`, and the first is not even caught by the ARM deployment of the + trigger itself - it only surfaces when the trigger is started, which happens in a separate + deployment script at the end of the hub deployment. A failure there aborts the whole + deployment and leaves every other trigger stopped. + + 1. Schedule trigger start times + When `timeZone` is 'UTC', Data Factory requires the zone designator on `startTime`: + 'yyyy-MM-ddTHH:mm:ssZ'. Without it the trigger deploys successfully but fails to start + with InvalidWorkflowTriggerRecurrence. Named Windows time zones (the timeZones module) + take the opposite form and must not carry the designator. + + 2. quoteAllText + Data Factory rejects `quoteAllText: false` on a Copy activity sink with + DelimitedTextInvalidSettings ("QuoteAllText cannot set to false for Copy activity + currently"). To write unquoted text, leave the property unset and set an empty + `quoteChar` on the dataset instead. +#> + +Describe 'DataFactorySettings' { + + BeforeDiscovery { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + + $scanFiles = @( + Get-ChildItem -Path (Join-Path $repoRoot 'src/templates') -Filter '*.bicep' -Recurse -File -ErrorAction SilentlyContinue | + Sort-Object FullName -Unique | + ForEach-Object { + @{ FullName = $_.FullName; RelPath = $_.FullName.Substring($repoRoot.Length + 1).Replace('\', '/') } + } + ) + } + + BeforeAll { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + + $scanFileCount = @(Get-ChildItem -Path (Join-Path $repoRoot 'src/templates') -Filter '*.bicep' -Recurse -File -ErrorAction SilentlyContinue).Count + + # A startTime literal plus whatever follows it up to the next property, so the paired + # timeZone can be read. Data Factory always emits the two adjacent within recurrence. + $startTimePattern = [regex]"startTime:\s*'([^']*)'\s*\r?\n\s*timeZone:\s*(.+)" + + $quoteAllTextPattern = [regex]'quoteAllText:\s*false' + } + + It 'Should scan the template tree' { + $scanFileCount | Should -BeGreaterThan 20 + } + + It 'Should use the zone designator on UTC schedule trigger start times: ' -ForEach $scanFiles { + $content = Get-Content -Path $FullName -Raw + + $offenders = @( + foreach ($match in $startTimePattern.Matches($content)) + { + $startTime = $match.Groups[1].Value + $timeZone = $match.Groups[2].Value.Trim() + + if ($timeZone -eq "'UTC'" -and -not $startTime.EndsWith('Z')) + { + "startTime '$startTime' with timeZone $timeZone" + } + elseif ($timeZone -ne "'UTC'" -and $timeZone.StartsWith("'") -and $startTime.EndsWith('Z')) + { + "startTime '$startTime' with timeZone $timeZone" + } + } + ) + + $offenders -join '; ' | Should -BeNullOrEmpty -Because ("Data Factory requires the 'yyyy-MM-ddTHH:mm:ssZ' form when timeZone is UTC, and the form without a designator for a named time zone. The wrong form deploys successfully but fails to start the trigger with InvalidWorkflowTriggerRecurrence, which aborts the hub deployment and leaves every trigger stopped.") + } + + It 'Should not set quoteAllText to false: ' -ForEach $scanFiles { + $content = Get-Content -Path $FullName -Raw + + @($quoteAllTextPattern.Matches($content)).Count | Should -Be 0 -Because ('Data Factory rejects quoteAllText: false with DelimitedTextInvalidSettings ("QuoteAllText cannot set to false for Copy activity currently"). Leave the property unset and set an empty quoteChar on the dataset to write unquoted text.') + } +} diff --git a/src/powershell/Tests/Unit/Add-FinOpsHubResourceGraphReader.Tests.ps1 b/src/powershell/Tests/Unit/Add-FinOpsHubResourceGraphReader.Tests.ps1 new file mode 100644 index 000000000..8055bff44 --- /dev/null +++ b/src/powershell/Tests/Unit/Add-FinOpsHubResourceGraphReader.Tests.ps1 @@ -0,0 +1,62 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +& "$PSScriptRoot/../Initialize-Tests.ps1" + +InModuleScope FinOpsToolkit { + Describe 'Add-FinOpsHubResourceGraphReader' { + BeforeAll { + function Get-AzContext {} + function Get-FinOpsHub {} + function Get-AzDataFactoryV2 {} + function Get-AzRoleAssignment {} + function New-AzRoleAssignment {} + } + + BeforeEach { + Mock Get-AzContext { @{ Tenant = @{ Id = 'tenant-id' } } } + Mock Get-FinOpsHub { + @{ + Resources = @( + @{ + ResourceType = 'Microsoft.DataFactory/factories' + ResourceGroupName = 'hub-rg' + Name = 'hub-engine' + } + ) + } + } + Mock Get-AzDataFactoryV2 { + @{ Identity = @{ PrincipalId = 'principal-id' } } + } + Mock New-AzRoleAssignment { @{ Id = 'assignment-id' } } + } + + It 'assigns Reader at a subscription scope when given a subscription ID' { + $result = Add-FinOpsHubResourceGraphReader -Scope '00000000-0000-0000-0000-000000000000' -Confirm:$false + + Should -Invoke New-AzRoleAssignment -Times 1 + $result.Id | Should -Be 'assignment-id' + } + + It 'does not create an assignment when Reader already exists' { + Mock Get-AzRoleAssignment { @{ Id = 'existing-id' } } + + $result = Add-FinOpsHubResourceGraphReader -Scope '/subscriptions/sub-id' -Confirm:$false + + Should -Invoke New-AzRoleAssignment -Times 0 + $result.Id | Should -Be 'existing-id' + } + + It 'supports management group scopes' { + Add-FinOpsHubResourceGraphReader -Scope '/providers/Microsoft.Management/managementGroups/contoso' -Confirm:$false + + Should -Invoke New-AzRoleAssignment -Times 1 + } + + It 'rejects unsupported scopes' { + { Add-FinOpsHubResourceGraphReader -Scope '/resourceGroups/example' -Confirm:$false } | Should -Throw + Should -Invoke Get-FinOpsHub -Times 0 + } + } +} diff --git a/src/powershell/Tests/Unit/Deploy-FinOpsHub.Tests.ps1 b/src/powershell/Tests/Unit/Deploy-FinOpsHub.Tests.ps1 index 1bb4512a0..bb35b547c 100644 --- a/src/powershell/Tests/Unit/Deploy-FinOpsHub.Tests.ps1 +++ b/src/powershell/Tests/Unit/Deploy-FinOpsHub.Tests.ps1 @@ -268,5 +268,85 @@ InModuleScope 'FinOpsToolkit' { } -Times 1 } } + + Context 'Multicloud' { + BeforeAll { + Mock -CommandName 'Get-AzResourceGroup' -MockWith { return @{ ResourceGroupName = $rgName } } + Mock -CommandName 'New-AzResourceGroup' + Mock -CommandName 'Save-FinOpsHubTemplate' + Mock -CommandName 'Initialize-FinOpsHubDeployment' + $templateFile = Join-Path -Path ([System.IO.Path]::GetTempPath()) -ChildPath 'FinOps/finops-hub-v1.0.0/main.bicep' + Mock -CommandName 'Get-ChildItem' -MockWith { return @{ FullName = $templateFile } } + Mock -CommandName 'New-AzResourceGroupDeployment' + + [Diagnostics.CodeAnalysis.SuppressMessageAttribute("PSUseDeclaredVarsMoreThanAssignments", "")] + $awsSecret = ConvertTo-SecureString -String 'ftk-test-secret' -AsPlainText -Force + } + + It 'Should not pass AWS parameters by default' { + { Deploy-FinOpsHub -Name $hubName -ResourceGroup $rgName -Location $location -Version 'latest' } | Should -Not -Throw + Should -Invoke -CommandName 'New-AzResourceGroupDeployment' -ParameterFilter { + -not $TemplateParameterObject.ContainsKey('enableAwsFocusIngestion') + } -Times 1 + } + + It 'Should pass AWS parameters when AWS FOCUS ingestion is enabled' { + { + Deploy-FinOpsHub -Name $hubName -ResourceGroup $rgName -Location $location -Version 'latest' ` + -EnableAwsFocusIngestion ` + -AwsBucketName 'ftk-test-bucket' ` + -AwsBucketPath 'reports/focus-export' ` + -AwsAccountId '123456789012' ` + -AwsRegion 'us-east-1' ` + -AwsAccessKeyId 'AKIAIOSFODNN7EXAMPLE' ` + -AwsSecretAccessKey $awsSecret + } | Should -Not -Throw + Should -Invoke -CommandName 'New-AzResourceGroupDeployment' -ParameterFilter { + $TemplateParameterObject.enableAwsFocusIngestion -eq $true -and + $TemplateParameterObject.awsBucketName -eq 'ftk-test-bucket' -and + $TemplateParameterObject.awsBucketPath -eq 'reports/focus-export' -and + $TemplateParameterObject.awsAccountId -eq '123456789012' -and + $TemplateParameterObject.awsRegion -eq 'us-east-1' -and + $TemplateParameterObject.awsAccessKeyId -eq 'AKIAIOSFODNN7EXAMPLE' + } -Times 1 + } + + It 'Should default the collection hour to 4' { + { + Deploy-FinOpsHub -Name $hubName -ResourceGroup $rgName -Location $location -Version 'latest' ` + -EnableAwsFocusIngestion -AwsBucketName 'ftk-test-bucket' -AwsSecretAccessKey $awsSecret + } | Should -Not -Throw + Should -Invoke -CommandName 'New-AzResourceGroupDeployment' -ParameterFilter { + $TemplateParameterObject.multiCloudScheduleHour -eq 4 + } -Times 1 + } + + It 'Should pass the requested collection hour' { + { + Deploy-FinOpsHub -Name $hubName -ResourceGroup $rgName -Location $location -Version 'latest' ` + -EnableAwsFocusIngestion -AwsBucketName 'ftk-test-bucket' -AwsSecretAccessKey $awsSecret -MultiCloudScheduleHour 20 + } | Should -Not -Throw + Should -Invoke -CommandName 'New-AzResourceGroupDeployment' -ParameterFilter { + $TemplateParameterObject.multiCloudScheduleHour -eq 20 + } -Times 1 + } + + It 'Should not pass AWS parameters when targeting a version older than 15.0' { + { + Deploy-FinOpsHub -Name $hubName -ResourceGroup $rgName -Location $location -Version '14.0' ` + -EnableAwsFocusIngestion -AwsBucketName 'ftk-test-bucket' -AwsSecretAccessKey $awsSecret + } | Should -Not -Throw + Should -Invoke -CommandName 'New-AzResourceGroupDeployment' -ParameterFilter { + -not $TemplateParameterObject.ContainsKey('enableAwsFocusIngestion') + } -Times 1 + } + + It 'Should reject a collection hour outside of 0-23' { + { + Deploy-FinOpsHub -Name $hubName -ResourceGroup $rgName -Location $location -Version 'latest' ` + -EnableAwsFocusIngestion -MultiCloudScheduleHour 24 + } | Should -Throw + } + } } } \ No newline at end of file diff --git a/src/powershell/en-US/FinOpsToolkit.strings.psd1 b/src/powershell/en-US/FinOpsToolkit.strings.psd1 index 0bba99f77..b512357d5 100644 --- a/src/powershell/en-US/FinOpsToolkit.strings.psd1 +++ b/src/powershell/en-US/FinOpsToolkit.strings.psd1 @@ -16,6 +16,21 @@ ConvertFrom-StringData -StringData @' Hub_Remove_Failed = FinOps hub could not be deleted. {0}. Hub_Remove_NotFound = FinOps hub '{0}' not found. + HubBillingReader_Add_AlreadyAssigned = The FinOps hub managed identity already has the Billing Reader role on billing account '{0}'. + HubBillingReader_Add_Assigned = Granted the Billing Reader role to the FinOps hub managed identity on billing account '{0}'. + HubBillingReader_Add_AssignFailed = Could not grant the Billing Reader role on billing account '{0}'. {1}. + HubBillingReader_Add_DataFactoryNotFound = Could not find a Data Factory for FinOps hub '{0}'. Confirm the hub is deployed and you are connected to the correct subscription. + HubBillingReader_Add_IdentityNotFound = Data Factory '{0}' does not have a managed identity. + HubBillingReader_Add_MultipleDataFactories = Found more than one Data Factory for FinOps hub '{0}'. Specify -ResourceGroupName to narrow the search. + + HubResourceGraphReader_AlreadyAssigned = The FinOps hub managed identity already has the Reader role on scope '{0}'. + HubResourceGraphReader_Assigned = Granted the Reader role to the FinOps hub managed identity on scope '{0}'. + HubResourceGraphReader_AssignFailed = Could not grant the Reader role on scope '{0}'. {1}. + HubResourceGraphReader_DataFactoryNotFound = Could not find a Data Factory for FinOps hub '{0}'. Confirm the hub is deployed and you are connected to the correct subscription. + HubResourceGraphReader_IdentityNotFound = Data Factory '{0}' does not have a managed identity. + HubResourceGraphReader_InvalidScope = Invalid Resource Graph scope '{0}'. Use a subscription ID, subscription resource ID, or management group resource ID. + HubResourceGraphReader_MultipleDataFactories = Found more than one Data Factory for FinOps hub '{0}'. Specify -ResourceGroupName to narrow the search. + HubLocal_Initialize_AssetEmpty = Downloaded asset '{0}' from '{1}' was empty. The release may be incomplete or the URI may not point to a valid FinOps toolkit release. HubLocal_Initialize_DownloadFailed = Could not download asset '{0}' from '{1}'. Check the release URI and network connectivity. HubLocal_Initialize_NotReachable = Could not reach the Kusto emulator at '{0}'. Start the local hub container before running this command. See https://aka.ms/finops/hubs/local. diff --git a/src/scripts/Deploy-Hub.ps1 b/src/scripts/Deploy-Hub.ps1 index c4741aa3a..149042201 100644 --- a/src/scripts/Deploy-Hub.ps1 +++ b/src/scripts/Deploy-Hub.ps1 @@ -76,6 +76,12 @@ .PARAMETER Recommendations Enable recommendations with all noisy recommendation types (AHB, Spot). Requires the hub template to have recommendation parameters. + .PARAMETER InvoiceDownload + Enable automatic download of Microsoft invoice files. Requires the hub template to have invoice parameters. + + .PARAMETER InvoiceBillingAccounts + Optional. Billing account IDs to download invoices for, separated by a new line, comma, or semicolon. Requires InvoiceDownload. Default: use the billing account scopes monitored by the hub. + .PARAMETER Remove Remove test environments. With a name, deletes the target resource group. Alone, lists all resource groups matching "{initials}-*". @@ -116,6 +122,8 @@ param( [string]$Fabric, [switch]$StorageOnly, [switch]$Recommendations, + [switch]$InvoiceDownload, + [string]$InvoiceBillingAccounts, [switch]$Remove, [string]$Scope, [switch]$ManagedExports, @@ -251,6 +259,16 @@ if ($Recommendations) $params.enableSpotRecommendations = $true } +# Invoice download (requires enableInvoiceDownload param in hub template) +if ($InvoiceDownload) +{ + $params.enableInvoiceDownload = $true + if ($InvoiceBillingAccounts) + { + $params.invoiceBillingAccounts = $InvoiceBillingAccounts + } +} + # Analytics backend if ($StorageOnly) { diff --git a/src/templates/finops-hub/.build.config b/src/templates/finops-hub/.build.config index 3b4f76c15..ea6e8b37a 100644 --- a/src/templates/finops-hub/.build.config +++ b/src/templates/finops-hub/.build.config @@ -5,6 +5,8 @@ "ignore": [ "errors.json", "modules/README.md", + "modules/Microsoft.Billing/Invoices/README.md", + "modules/Microsoft.FinOpsHubs/AmazonWebServices/README.md", "modules/scripts/README.md", "schemas/README.md", "test" diff --git a/src/templates/finops-hub/createUiDefinition.json b/src/templates/finops-hub/createUiDefinition.json index 52f5ae4e2..352c73993 100644 --- a/src/templates/finops-hub/createUiDefinition.json +++ b/src/templates/finops-hub/createUiDefinition.json @@ -836,6 +836,286 @@ } ] }, + { + "name": "invoices", + "label": "🆕 Invoices", + "elements": [ + { + "name": "invoicesIntro", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "Save your Microsoft invoice files in the hub data lake for reconciliation and audit. FinOps hubs can download invoices every month and organize them by billing period, billing profile, and purchase order number. Only supported for Microsoft Customer Agreement (MCA) and Microsoft Partner Agreement (MPA) billing accounts." + } + }, + { + "name": "enableInvoiceDownload", + "type": "Microsoft.Common.CheckBox", + "label": "Enable invoice download (preview)", + "toolTip": "Download Microsoft invoice files into the ingestion container every month. Invoices are saved in the invoices folder." + }, + { + "name": "billingAccounts", + "type": "Microsoft.Common.Section", + "label": "Billing accounts", + "visible": "[steps('invoices').enableInvoiceDownload]", + "elements": [ + { + "name": "billingAccountsIntro", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "Leave blank to download invoices for the billing account scopes this hub already monitors. To download invoices for specific billing accounts, enter one billing account ID per line. Find your billing account ID in Cost Management + Billing > Properties." + } + }, + { + "name": "invoiceBillingAccounts", + "type": "Microsoft.Common.TextBox", + "label": "Billing account IDs", + "defaultValue": "", + "multiLine": true, + "toolTip": "One billing account ID per line. Example: 12345678-1234-1234-1234-123456789012:87654321-4321-4321-4321-210987654321_2019-05-31", + "constraints": { + "required": false + }, + "visible": true + } + ] + }, + { + "name": "schedule", + "type": "Microsoft.Common.Section", + "label": "Schedule", + "visible": "[steps('invoices').enableInvoiceDownload]", + "elements": [ + { + "name": "scheduleIntro", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "Invoices for the previous month are downloaded once a month. Invoices are generally available within the first few days of the month, but the exact date varies by billing account." + } + }, + { + "name": "invoiceScheduleDay", + "type": "Microsoft.Common.DropDown", + "label": "Day of the month", + "defaultValue": "10", + "toolTip": "Day of the month to download invoices from the previous month.", + "constraints": { + "allowedValues": [ + { "label": "5", "value": 5 }, + { "label": "10", "value": 10 }, + { "label": "15", "value": 15 }, + { "label": "20", "value": 20 } + ], + "required": true + }, + "visible": true + } + ] + }, + { + "name": "permissions", + "type": "Microsoft.Common.Section", + "label": "Required permissions", + "visible": "[steps('invoices').enableInvoiceDownload]", + "elements": [ + { + "name": "permissionsNote", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "The Data Factory managed identity requires the Billing Reader role on the billing account. This role cannot be granted during deployment. After deployment, run Add-FinOpsHubBillingReader from the FinOpsToolkit PowerShell module or grant the role in Cost Management + Billing > Access control (IAM)." + } + } + ] + } + ] + }, + { + "name": "multicloud", + "label": "🆕 Multicloud", + "elements": [ + { + "name": "multicloudIntro", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "Ingest FOCUS cost data exported from other clouds so you can report on all of your costs together. FinOps hubs copy the exported files once a day and run them through the same normalization and ingestion pipeline used for Microsoft Cloud costs." + } + }, + { + "name": "enableAwsFocusIngestion", + "type": "Microsoft.Common.CheckBox", + "label": "Enable Amazon Web Services FOCUS ingestion (preview)", + "toolTip": "Collect a FOCUS 1.2 export from an Amazon S3 bucket every day and ingest it into the hub." + }, + { + "name": "awsExport", + "type": "Microsoft.Common.Section", + "label": "Amazon Web Services export", + "visible": "[steps('multicloud').enableAwsFocusIngestion]", + "elements": [ + { + "name": "awsExportIntro", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "Create a FOCUS 1.2 export in Billing and Cost Management > Data Exports and point it at an S3 bucket. Then enter the bucket details below. For an export delivered to s3://my-bucket/reports/focus-export/data/, the bucket name is my-bucket and the export path is reports/focus-export." + } + }, + { + "name": "awsBucketName", + "type": "Microsoft.Common.TextBox", + "label": "S3 bucket name", + "defaultValue": "", + "toolTip": "Name of the S3 bucket that receives the FOCUS export.", + "constraints": { + "required": "[steps('multicloud').enableAwsFocusIngestion]", + "regex": "^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$", + "validationMessage": "Must be a valid S3 bucket name: 3-63 characters, lowercase letters, numbers, dots, and hyphens." + }, + "visible": true + }, + { + "name": "awsBucketPath", + "type": "Microsoft.Common.TextBox", + "label": "Export path", + "defaultValue": "", + "toolTip": "Path to the export root folder within the bucket. This is the folder that contains the data and metadata subfolders.", + "constraints": { + "required": "[steps('multicloud').enableAwsFocusIngestion]", + "regex": "^[^/].*[^/]$", + "validationMessage": "Must not start or end with a slash. Do not include the data or metadata folder." + }, + "visible": true + }, + { + "name": "awsAccountId", + "type": "Microsoft.Common.TextBox", + "label": "AWS account ID", + "defaultValue": "", + "toolTip": "Account ID that owns the export. Used to isolate the data in the hub data lake.", + "constraints": { + "required": "[steps('multicloud').enableAwsFocusIngestion]", + "regex": "^[0-9]{12}$", + "validationMessage": "Must be a 12-digit AWS account ID." + }, + "visible": true + }, + { + "name": "awsRegion", + "type": "Microsoft.Common.TextBox", + "label": "Bucket region", + "defaultValue": "", + "toolTip": "Region of the bucket, for example us-east-1. Leave blank to use the global S3 endpoint.", + "constraints": { + "required": false, + "regex": "^$|^[a-z]{2}-[a-z]+-[0-9]$", + "validationMessage": "Must be a valid AWS region, for example us-east-1, or blank." + }, + "visible": true + } + ] + }, + { + "name": "awsCredentials", + "type": "Microsoft.Common.Section", + "label": "Amazon Web Services credentials", + "visible": "[steps('multicloud').enableAwsFocusIngestion]", + "elements": [ + { + "name": "awsCredentialsIntro", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "Create an IAM user with read access to the bucket and generate an access key for it. The user needs s3:ListBucket on the bucket and s3:GetObject on its contents. The secret access key is stored in the hub Key Vault and is never written to the connection definition." + } + }, + { + "name": "awsAccessKeyId", + "type": "Microsoft.Common.TextBox", + "label": "Access key ID", + "defaultValue": "", + "toolTip": "Access key ID for the IAM user that can read the bucket.", + "constraints": { + "required": "[steps('multicloud').enableAwsFocusIngestion]", + "regex": "^[A-Z0-9]{16,128}$", + "validationMessage": "Must be a valid AWS access key ID." + }, + "visible": true + }, + { + "name": "awsSecretAccessKey", + "type": "Microsoft.Common.PasswordBox", + "label": { + "password": "Secret access key" + }, + "toolTip": "Secret access key for the IAM user that can read the bucket.", + "constraints": { + "required": "[steps('multicloud').enableAwsFocusIngestion]" + }, + "options": { + "hideConfirmation": true + }, + "visible": true + } + ] + }, + { + "name": "schedule", + "type": "Microsoft.Common.Section", + "label": "Schedule", + "visible": "[steps('multicloud').enableAwsFocusIngestion]", + "elements": [ + { + "name": "scheduleIntro", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "Files are collected once a day. Each run collects the current and the previous billing period, because AWS can restate a closed period for up to two weeks after it ends." + } + }, + { + "name": "multiCloudScheduleHour", + "type": "Microsoft.Common.DropDown", + "label": "Hour of the day (UTC)", + "defaultValue": "04:00", + "toolTip": "Hour of the day, in UTC, to collect FOCUS files.", + "constraints": { + "allowedValues": [ + { "label": "00:00", "value": 0 }, + { "label": "04:00", "value": 4 }, + { "label": "08:00", "value": 8 }, + { "label": "12:00", "value": 12 }, + { "label": "16:00", "value": 16 }, + { "label": "20:00", "value": 20 } + ], + "required": true + }, + "visible": true + } + ] + }, + { + "name": "egress", + "type": "Microsoft.Common.Section", + "label": "Cost note", + "visible": "[steps('multicloud').enableAwsFocusIngestion]", + "elements": [ + { + "name": "egressNote", + "type": "Microsoft.Common.TextBlock", + "visible": true, + "options": { + "text": "Copying data out of Amazon S3 incurs egress charges billed by AWS, not Azure. Data Factory activity runs and staging storage are billed by Azure." + } + } + ] + } + ] + }, { "name": "advanced", "label": "Advanced", @@ -1002,6 +1282,17 @@ "enableRecommendations": "[steps('recommendations').enableRecommendations]", "enableAHBRecommendations": "[steps('recommendations').optional.enableAHBRecommendations]", "enableSpotRecommendations": "[steps('recommendations').optional.enableSpotRecommendations]", + "enableInvoiceDownload": "[steps('invoices').enableInvoiceDownload]", + "invoiceBillingAccounts": "[steps('invoices').billingAccounts.invoiceBillingAccounts]", + "invoiceScheduleDay": "[steps('invoices').schedule.invoiceScheduleDay]", + "enableAwsFocusIngestion": "[steps('multicloud').enableAwsFocusIngestion]", + "awsBucketName": "[steps('multicloud').awsExport.awsBucketName]", + "awsBucketPath": "[steps('multicloud').awsExport.awsBucketPath]", + "awsAccountId": "[steps('multicloud').awsExport.awsAccountId]", + "awsRegion": "[steps('multicloud').awsExport.awsRegion]", + "awsAccessKeyId": "[steps('multicloud').awsCredentials.awsAccessKeyId]", + "awsSecretAccessKey": "[steps('multicloud').awsCredentials.awsSecretAccessKey]", + "multiCloudScheduleHour": "[steps('multicloud').schedule.multiCloudScheduleHour]", "enablePublicAccess": "[steps('advanced').networking.enablePublicAccess]", "enableNatGateway": "[steps('advanced').networking.enableNatGateway]", "virtualNetworkAddressPrefix": "[steps('advanced').networking.virtualNetworkAddressPrefix]", diff --git a/src/templates/finops-hub/dashboard.json b/src/templates/finops-hub/dashboard.json index 67a7e1f94..f667a61d3 100644 --- a/src/templates/finops-hub/dashboard.json +++ b/src/templates/finops-hub/dashboard.json @@ -5,6 +5,354 @@ "title": "FinOps hub", "schema_version": "60", "tiles": [ + { + "id": "2a13803b-1f86-46d4-b506-86030f8b90de", + "title": "On this page", + "visualType": "markdownCard", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 0, "width": 22, "height": 2 }, + "markdownText": "  [Azure Advisor](?tile=3b45dfed-ae66-441f-8668-4cd1765fbe55)\r\n• [Where the savings are](?tile=930a193b-57b8-4377-a83b-aeda8a2cede9)\r\n• [Recommendation details](?tile=95fdc52f-bc42-4c16-ae17-b302a28e3b9d)\r\n• [FinOps hubs recommendations](?tile=01e44432-18b8-458f-b1c8-fec8a4a0327e)\r\n• [Data freshness](?tile=4206687d-7abf-4e40-9bce-c2eabe41a86b)\r\n• [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20the%20FinOps%20hubs%20Data%20Explorer%20dashboard%3F/cvaQuestion/How%20valuable%20the%20FinOps%20hubs%20Data%20Explorer%20dashboard%3F/surveyId/FTK$$ftkver$$/bladeName/Hubs.Dashboard/featureName/Optimize.UsageOptimization.Nav)\r\n", + "visualOptions": {} + }, + { + "id": "3b45dfed-ae66-441f-8668-4cd1765fbe55", + "title": "", + "visualType": "markdownCard", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 2, "width": 22, "height": 4 }, + "markdownText": "# Azure Advisor cost recommendations\r\nCost optimization recommendations collected from Azure Advisor for the scopes monitored by this hub, based on the latest daily collection. Values are recommender estimates, not realized savings. If this section is empty, confirm the recommendations app is deployed, its daily schedule is running, and Advisor has cost recommendations in scope. See [Data freshness](?tile=4206687d-7abf-4e40-9bce-c2eabe41a86b).\r\n\r\n⬆️ [Top](?tile=2a13803b-1f86-46d4-b506-86030f8b90de)     💜 [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20the%20FinOps%20hubs%20Data%20Explorer%20dashboard%3F/cvaQuestion/How%20valuable%20the%20FinOps%20hubs%20Data%20Explorer%20dashboard%3F/surveyId/FTK$$ftkver$$/bladeName/Hubs.Dashboard/featureName/Optimize.UsageOptimization.Advisor)", + "visualOptions": {} + }, + { + "id": "f78a8288-316d-4980-919a-3ec168a53d59", + "title": "Advisor cost recommendations (latest snapshot)", + "visualType": "multistat", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 6, "width": 15, "height": 5 }, + "queryRef": { "kind": "query", "queryId": "d7791e54-e5b1-4d2f-9891-07fc3d1c42c1" }, + "visualOptions": { + "multiStat__textSize": "auto", + "multiStat__valueColumn": "Value", + "colorRulesDisabled": false, + "colorStyle": "light", + "multiStat__displayOrientation": "horizontal", + "multiStat__labelColumn": "Label", + "multiStat__slot": { "width": 5, "height": 1 }, + "colorRules": [ + { + "id": "2bfaa60f-1716-4974-ba1d-c9b6232482f5", + "ruleType": "colorByCondition", + "applyToColumn": null, + "hideText": false, + "applyTo": "cells", + "conditions": [{ "operator": "==", "column": "Type", "values": ["Savings"] }], + "chainingOperator": "and", + "colorStyle": "bold", + "color": "green", + "tag": "Estimated savings", + "icon": null, + "ruleName": "Estimated savings", + "visualType": "multistat" + }, + { + "id": "e5081e50-2a08-463c-9774-4d4ad0362741", + "ruleType": "colorByCondition", + "applyToColumn": null, + "hideText": false, + "applyTo": "cells", + "conditions": [{ "operator": "==", "column": "Type", "values": ["Count"] }], + "chainingOperator": "and", + "colorStyle": "light", + "color": "blue", + "tag": "Count", + "icon": null, + "ruleName": "Count", + "visualType": "multistat" + } + ] + } + }, + { + "id": "0880afa6-5fce-43ce-8121-7f952fb1dbcd", + "title": "Recommendations by impact", + "visualType": "pie", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 15, "y": 6, "width": 7, "height": 5 }, + "queryRef": { "kind": "query", "queryId": "542437d4-1f4e-41d1-911e-254c796bcd23" }, + "visualOptions": { + "hideLegend": false, + "legendLocation": "bottom", + "xColumn": "Impact", + "yColumns": null, + "seriesColumns": null, + "crossFilterDisabled": false, + "drillthroughDisabled": false, + "labelDisabled": false, + "pie__label": ["name", "percentage"], + "tooltipDisabled": false, + "pie__tooltip": ["name", "percentage", "value"], + "pie__orderBy": "size", + "pie__kind": "pie", + "pie__topNSlices": null, + "crossFilter": [], + "drillthrough": [] + } + }, + { + "id": "930a193b-57b8-4377-a83b-aeda8a2cede9", + "title": "", + "visualType": "markdownCard", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 11, "width": 22, "height": 2 }, + "markdownText": "## Where the savings are", + "visualOptions": {} + }, + { + "id": "4c3d06e5-ef4e-40d3-82d0-c5173504c41a", + "title": "Top recommendations by estimated monthly savings", + "visualType": "bar", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 13, "width": 11, "height": 7 }, + "queryRef": { "kind": "query", "queryId": "cf085d1b-ae69-4b44-b54d-e43acf4001b5" }, + "visualOptions": { + "multipleYAxes": { + "base": { "id": "-1", "label": "", "columns": [], "yAxisMaximumValue": null, "yAxisMinimumValue": null, "yAxisScale": "linear", "horizontalLines": [] }, + "additional": [], + "showMultiplePanels": false + }, + "hideLegend": true, + "legendLocation": "bottom", + "xColumnTitle": "", + "xColumn": "Recommendation", + "yColumns": ["Savings"], + "seriesColumns": null, + "xAxisScale": "linear", + "verticalLine": "", + "crossFilterDisabled": false, + "drillthroughDisabled": false, + "crossFilter": [], + "drillthrough": [], + "selectedDataOnLoad": { "all": true, "limit": 10 }, + "dataPointsTooltip": { "all": false, "limit": 1 } + } + }, + { + "id": "ccb8da89-24c3-423c-a083-e223b834736a", + "title": "Estimated monthly savings by resource type", + "visualType": "bar", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 11, "y": 13, "width": 11, "height": 7 }, + "queryRef": { "kind": "query", "queryId": "d56a9ad3-e68b-4eae-a32b-bdf0c3509414" }, + "visualOptions": { + "multipleYAxes": { + "base": { "id": "-1", "label": "", "columns": [], "yAxisMaximumValue": null, "yAxisMinimumValue": null, "yAxisScale": "linear", "horizontalLines": [] }, + "additional": [], + "showMultiplePanels": false + }, + "hideLegend": true, + "legendLocation": "bottom", + "xColumnTitle": "", + "xColumn": "ResourceType", + "yColumns": ["Savings"], + "seriesColumns": null, + "xAxisScale": "linear", + "verticalLine": "", + "crossFilterDisabled": false, + "drillthroughDisabled": false, + "crossFilter": [], + "drillthrough": [], + "selectedDataOnLoad": { "all": true, "limit": 10 }, + "dataPointsTooltip": { "all": false, "limit": 1 } + } + }, + { + "id": "f87ce7e8-b246-45ee-a3a1-af1698751b69", + "title": "Estimated monthly savings by subscription", + "visualType": "bar", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 20, "width": 22, "height": 6 }, + "queryRef": { "kind": "query", "queryId": "3c7816aa-8038-440b-b635-fda289580e55" }, + "visualOptions": { + "multipleYAxes": { + "base": { "id": "-1", "label": "", "columns": [], "yAxisMaximumValue": null, "yAxisMinimumValue": null, "yAxisScale": "linear", "horizontalLines": [] }, + "additional": [], + "showMultiplePanels": false + }, + "hideLegend": true, + "legendLocation": "bottom", + "xColumnTitle": "", + "xColumn": "Subscription", + "yColumns": ["Savings"], + "seriesColumns": null, + "xAxisScale": "linear", + "verticalLine": "", + "crossFilterDisabled": false, + "drillthroughDisabled": false, + "crossFilter": [], + "drillthrough": [], + "selectedDataOnLoad": { "all": true, "limit": 10 }, + "dataPointsTooltip": { "all": false, "limit": 1 } + } + }, + { + "id": "95fdc52f-bc42-4c16-ae17-b302a28e3b9d", + "title": "", + "visualType": "markdownCard", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 26, "width": 22, "height": 2 }, + "markdownText": "## Recommendation details", + "visualOptions": {} + }, + { + "id": "992ad351-bd07-4be6-b71c-ef4755f0402f", + "title": "Advisor cost recommendations", + "visualType": "table", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 28, "width": 22, "height": 9 }, + "queryRef": { "kind": "query", "queryId": "6de2078e-9402-4e08-b977-5cf2d8560e96" }, + "visualOptions": { + "table__enableRenderLinks": true, + "colorRulesDisabled": false, + "colorStyle": "light", + "crossFilterDisabled": false, + "drillthroughDisabled": false, + "crossFilter": [], + "drillthrough": [], + "table__renderLinks": [], + "colorRules": [] + } + }, + { + "id": "01e44432-18b8-458f-b1c8-fec8a4a0327e", + "title": "", + "visualType": "markdownCard", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 37, "width": 22, "height": 4 }, + "markdownText": "# FinOps hubs recommendations\r\nBuilt-in recommendations generated by the hub itself from Azure Resource Graph, covering idle, orphaned, and legacy resources that Advisor doesn't report. Savings estimates are only available for some recommendation types.\r\n\r\n⬆️ [Top](?tile=2a13803b-1f86-46d4-b506-86030f8b90de)     💜 [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20the%20FinOps%20hubs%20Data%20Explorer%20dashboard%3F/cvaQuestion/How%20valuable%20the%20FinOps%20hubs%20Data%20Explorer%20dashboard%3F/surveyId/FTK$$ftkver$$/bladeName/Hubs.Dashboard/featureName/Optimize.UsageOptimization.Hubs)", + "visualOptions": {} + }, + { + "id": "18a495a3-1235-4e2e-af05-128e2470c029", + "title": "FinOps hubs recommendations (latest snapshot)", + "visualType": "multistat", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 41, "width": 15, "height": 5 }, + "queryRef": { "kind": "query", "queryId": "58cee477-2b9c-4916-9150-17ef57be2f47" }, + "visualOptions": { + "multiStat__textSize": "auto", + "multiStat__valueColumn": "Value", + "colorRulesDisabled": false, + "colorStyle": "light", + "multiStat__displayOrientation": "horizontal", + "multiStat__labelColumn": "Label", + "multiStat__slot": { "width": 5, "height": 1 }, + "colorRules": [ + { + "id": "bc532449-59d0-4838-83e7-59e328110d42", + "ruleType": "colorByCondition", + "applyToColumn": null, + "hideText": false, + "applyTo": "cells", + "conditions": [{ "operator": "==", "column": "Type", "values": ["Savings"] }], + "chainingOperator": "and", + "colorStyle": "bold", + "color": "green", + "tag": "Estimated savings", + "icon": null, + "ruleName": "Estimated savings", + "visualType": "multistat" + }, + { + "id": "bd947a63-52d3-4cb6-9816-039a8426d505", + "ruleType": "colorByCondition", + "applyToColumn": null, + "hideText": false, + "applyTo": "cells", + "conditions": [{ "operator": "==", "column": "Type", "values": ["Count"] }], + "chainingOperator": "and", + "colorStyle": "light", + "color": "blue", + "tag": "Count", + "icon": null, + "ruleName": "Count", + "visualType": "multistat" + } + ] + } + }, + { + "id": "169d0739-f2e7-41d4-bf3a-2fa68f571ba4", + "title": "FinOps hubs recommendations by type", + "visualType": "pie", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 15, "y": 41, "width": 7, "height": 5 }, + "queryRef": { "kind": "query", "queryId": "4e754e2a-08d8-4b4b-bf60-7b692d09e977" }, + "visualOptions": { + "hideLegend": false, + "legendLocation": "bottom", + "xColumn": "Type", + "yColumns": null, + "seriesColumns": null, + "crossFilterDisabled": false, + "drillthroughDisabled": false, + "labelDisabled": false, + "pie__label": ["name", "percentage"], + "tooltipDisabled": false, + "pie__tooltip": ["name", "percentage", "value"], + "pie__orderBy": "size", + "pie__kind": "pie", + "pie__topNSlices": null, + "crossFilter": [], + "drillthrough": [] + } + }, + { + "id": "beaae601-2077-44cf-b439-5c522f3f2647", + "title": "FinOps hubs recommendation details", + "visualType": "table", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 46, "width": 22, "height": 8 }, + "queryRef": { "kind": "query", "queryId": "dda8c8c5-4561-442e-957d-e2968f6e9bf0" }, + "visualOptions": { + "table__enableRenderLinks": true, + "colorRulesDisabled": false, + "colorStyle": "light", + "crossFilterDisabled": false, + "drillthroughDisabled": false, + "crossFilter": [], + "drillthrough": [], + "table__renderLinks": [], + "colorRules": [] + } + }, + { + "id": "4206687d-7abf-4e40-9bce-c2eabe41a86b", + "title": "", + "visualType": "markdownCard", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 54, "width": 22, "height": 2 }, + "markdownText": "## Data freshness", + "visualOptions": {} + }, + { + "id": "362857bf-c166-408b-b83d-46a94032333a", + "title": "Recommendation collection by source", + "visualType": "table", + "pageId": "33596cbb-e820-44b3-9afd-998da101e36b", + "layout": { "x": 0, "y": 56, "width": 22, "height": 6 }, + "queryRef": { "kind": "query", "queryId": "88413f07-af1a-4ba9-afb8-61fe20eab29c" }, + "visualOptions": { + "table__enableRenderLinks": true, + "colorRulesDisabled": false, + "colorStyle": "light", + "crossFilterDisabled": false, + "drillthroughDisabled": false, + "crossFilter": [], + "drillthrough": [], + "table__renderLinks": [], + "colorRules": [] + } + }, { "id": "acf2ca3d-9d8f-477f-b259-f3937bf938d4", "title": "Cost last month", @@ -2175,7 +2523,7 @@ "visualType": "markdownCard", "pageId": "d01a8154-8a60-4d22-96ea-54b45b1417fe", "layout": { "x": 5, "y": 5, "width": 5, "height": 8 }, - "markdownText": "### Usage optimization\nUsage optimization refers to the process of ensuring cloud services are utilized and tuned to maximize business value and minimize wasteful usage and spending. With this capability, you analyze cost, usage, and carbon emissions for cloud workloads to identify opportunities to maximize efficiency. This capability usually starts with recommendations and expands into more nuanced optimization efforts based on detailed resource utilization analysis. This capability can be time and effort intensive as each cloud service has its different optimization opportunities.\n\n📊 Coming soon\n   \n📗 [Learn more](http://aka.ms/ftk/fx/workloads)", + "markdownText": "### Usage optimization\nUsage optimization refers to the process of ensuring cloud services are utilized and tuned to maximize business value and minimize wasteful usage and spending. With this capability, you analyze cost, usage, and carbon emissions for cloud workloads to identify opportunities to maximize efficiency. This capability usually starts with recommendations and expands into more nuanced optimization efforts based on detailed resource utilization analysis. This capability can be time and effort intensive as each cloud service has its different optimization opportunities.\n\n📊 [View report](#33596cbb-e820-44b3-9afd-998da101e36b)\n   \n📗 [Learn more](http://aka.ms/ftk/fx/workloads)", "visualOptions": {} }, { @@ -2738,14 +3086,169 @@ "colorRules": [] } } - ], + , + { + "id": "f6a7b8c9-d0e1-4f2a-3b4c-5d6e7f809102", + "title": "", + "visualType": "markdownCard", + "pageId": "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d", + "layout": { + "x": 0, + "y": 0, + "width": 22, + "height": 2 + }, + "markdownText": "On this page:   [KPIs](?tile=b8c9d0e1-f2a3-4b4c-5d6e-7f8091021324)  •  [Trend](?tile=c9d0e1f2-a3b4-4c5d-6e7f-8091021324a5)  •  [Services](?tile=d0e1f2a3-b4c5-4d6e-7f80-91021324a5b6)  •  [Top Resources](?tile=e1f2a3b4-c5d6-4e7f-8091-021324a5b6c7)     💜 [Give feedback](https://portal.azure.com/#view/HubsExtension/InProductFeedbackBlade/extensionName/FinOpsToolkit/cesQuestion/How%20easy%20or%20hard%20is%20it%20to%20use%20the%20FinOps%20hubs%20Data%20Explorer%20dashboard%3F/cvaQuestion/How%20valuable%20the%20FinOps%20hubs%20Data%20Explorer%20dashboard%3F/surveyId/FTK$$ftkver$$/bladeName/Hubs.Dashboard/featureName/Understand.AIBilling)", + "visualOptions": {} + }, + { + "id": "a7b8c9d0-e1f2-4a3b-4c5d-6e7f80910213", + "title": "", + "visualType": "markdownCard", + "pageId": "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d", + "layout": { + "x": 0, + "y": 2, + "width": 22, + "height": 3 + }, + "markdownText": "# AI billing\nBilling-level costs for Azure AI services from FinOps Hub data, filtered by `ServiceCategory == 'AI and Machine Learning'` plus `Azure AI Search`. Covers Azure AI Services, Azure Machine Learning, and Azure AI Search.\n\n⬆️ [Top](?tile=f6a7b8c9-d0e1-4f2a-3b4c-5d6e7f809102)", + "visualOptions": {} + }, + { + "id": "b8c9d0e1-f2a3-4b4c-5d6e-7f8091021324", + "title": "AI cost KPIs", + "visualType": "multistat", + "pageId": "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d", + "layout": { + "x": 0, + "y": 5, + "width": 6, + "height": 8 + }, + "queryRef": { + "kind": "query", + "queryId": "b2c3d4e5-f6a7-4b8c-9d0e-1f2a3b4c5d6e" + }, + "visualOptions": { + "multiStat__textSize": "large", + "multiStat__slot": { + "width": 1, + "height": 2 + }, + "crossFilterDisabled": true, + "drillthroughDisabled": true, + "crossFilter": [], + "drillthrough": [], + "colorRules": [] + } + }, + { + "id": "c9d0e1f2-a3b4-4c5d-6e7f-8091021324a5", + "title": "AI cost per day", + "visualType": "column", + "pageId": "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d", + "layout": { + "x": 6, + "y": 5, + "width": 16, + "height": 8 + }, + "queryRef": { + "kind": "query", + "queryId": "c3d4e5f6-a7b8-4c9d-0e1f-2a3b4c5d6e7f" + }, + "visualOptions": { + "yAxisMinimumValue": 0, + "yColumnTitle": "Effective cost", + "xColumnTitle": "", + "xColumn": "ChargePeriodStart", + "yColumns": [ + "EffectiveCost" + ], + "crossFilterDisabled": false, + "drillthroughDisabled": false, + "crossFilter": [], + "drillthrough": [], + "colorRules": [] + } + }, + { + "id": "f2a3b4c5-d6e7-4f80-9102-1324a5b6c7d8", + "title": "", + "visualType": "markdownCard", + "pageId": "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d", + "layout": { + "x": 0, + "y": 13, + "width": 22, + "height": 2 + }, + "markdownText": "## Services\nAI cost breakdown by service and top cost-driving resources.\n\n⬆️ [Top](?tile=f6a7b8c9-d0e1-4f2a-3b4c-5d6e7f809102)", + "visualOptions": {} + }, + { + "id": "d0e1f2a3-b4c5-4d6e-7f80-91021324a5b6", + "title": "AI cost by service", + "visualType": "stackedcolumn", + "pageId": "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d", + "layout": { + "x": 0, + "y": 15, + "width": 14, + "height": 5 + }, + "queryRef": { + "kind": "query", + "queryId": "d4e5f6a7-b8c9-4d0e-1f2a-3b4c5d6e7f80" + }, + "visualOptions": { + "yColumnTitle": "Effective cost", + "xColumn": "ServiceName", + "yColumns": [ + "EffectiveCost" + ], + "crossFilterDisabled": false, + "drillthroughDisabled": false, + "crossFilter": [], + "drillthrough": [], + "colorRules": [] + } + }, + { + "id": "e1f2a3b4-c5d6-4e7f-8091-021324a5b6c7", + "title": "Top 5 AI resources", + "visualType": "table", + "pageId": "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d", + "layout": { + "x": 14, + "y": 15, + "width": 8, + "height": 5 + }, + "queryRef": { + "kind": "query", + "queryId": "e5f6a7b8-c9d0-4e1f-2a3b-4c5d6e7f8091" + }, + "visualOptions": { + "table__enableRenderLinks": true, + "colorRulesDisabled": true, + "crossFilterDisabled": false, + "drillthroughDisabled": false, + "crossFilter": [], + "drillthrough": [], + "table__renderLinks": [], + "colorRules": [] + } + }], "baseQueries": [ { "id": "58764bcb-2ba0-4c7f-b018-e5d7f6cff688", "queryId": "43612ae4-c475-4f22-bb50-ce9d995abb8f", "variableName": "CostsThisMonth" }, { "id": "21512220-154c-4646-a188-72e8c437d8ed", "queryId": "cb1f5404-c0b1-42fd-99fb-3cff7b08daaa", "variableName": "CostsLastMonth" }, { "id": "5fa73857-4e92-4a16-8179-f94563e5f605", "queryId": "4ce0f587-2d45-436c-8f79-102c6b382439", "variableName": "CostsByMonth" }, { "id": "4880346a-0a24-48f9-bf25-b7427df29d69", "queryId": "6b598467-8c31-4693-b1eb-7ed683fcfc3a", "variableName": "CostsByDay" }, { "id": "8ca40660-3fbc-4a08-b12e-92e7382d9449", "queryId": "4a1973bf-08e9-4e82-b8e6-6edff81cf0a5", "variableName": "CostsByDayAHB" }, - { "id": "48ebd897-d085-490f-8b4b-9b43d4fb2efc", "queryId": "eb9259cc-05b7-4441-a66d-a29026fe371b", "variableName": "CostsPlus" } + { "id": "48ebd897-d085-490f-8b4b-9b43d4fb2efc", "queryId": "eb9259cc-05b7-4441-a66d-a29026fe371b", "variableName": "CostsPlus" }, + { "id": "d4461bd4-a159-4478-9caa-2f2581ddcf95", "queryId": "ede6c9ce-4b33-4c2e-b572-07e3361b242e", "variableName": "CostRecommendations" } ], "parameters": [ { @@ -2803,6 +3306,11 @@ { "name": "- Anomaly management", "id": "5838c918-4541-44fe-90d2-77306ef1e241" }, { "name": "- Data ingestion", "id": "9e099251-9658-48da-b416-80422a2a47c7" }, { "name": "OPTIMIZE", "id": "d01a8154-8a60-4d22-96ea-54b45b1417fe" }, + { "name": "- Usage optimization", "id": "33596cbb-e820-44b3-9afd-998da101e36b" }, + { + "name": "- AI billing", + "id": "a1b2c3d4-e5f6-4a7b-8c9d-0e1f2a3b4c5d" + }, { "name": "- Rate optimization", "id": "306fef9a-c760-4559-a326-7c25d196b616" }, { "id": "a4ec1d55-5b6e-49af-bb9e-4f3d136cdf05", "name": "- Licensing + SaaS" }, { "name": "QUANTIFY", "id": "8beab65c-f5ec-4661-bc67-37b10baffb16" }, @@ -2811,6 +3319,72 @@ { "name": "- Invoicing + chargeback", "id": "f416685d-f559-4514-8e45-5e0e09aec286" } ], "queries": [ + { + "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, + "text": "Recommendations()\n//\n// Only cost recommendations. Reservation and savings plan recommendations carry an empty\n// category and are covered by the Rate optimization page.\n| where x_RecommendationCategory =~ 'Cost'\n//\n// Recommendations are re-collected and appended every day, so reduce to the latest collection\n// window before aggregating; otherwise savings multiply by the number of snapshots.\n// x_RecommendationDate is commonly null, so x_IngestionTime is the only reliable time anchor.\n| where x_IngestionTime > toscalar(Recommendations() | where x_RecommendationCategory =~ 'Cost' | summarize max(x_IngestionTime)) - 1d\n//\n// Guard the dedupe key: x_RecommendationId can be empty, which would collapse unrelated rows.\n| extend RecKey = iff(isnotempty(x_RecommendationId), x_RecommendationId, strcat(ResourceId, '|', x_RecommendationDescription))\n| summarize arg_max(x_IngestionTime, *) by RecKey\n//\n// Recommendation details are normalized to x_PascalCase keys during ingestion.\n| extend\n Source = iff(x_SourceName =~ 'Azure Advisor', 'Azure Advisor', 'FinOps hubs'),\n Impact = tostring(x_RecommendationDetails.x_RecommendationImpact),\n Solution = tostring(x_RecommendationDetails.x_RecommendationSolution),\n SavingsCurrency = tostring(x_RecommendationDetails.x_SavingsCurrency)\n| extend\n RecommendationType = coalesce(x_RecommendationDescription, x_SourceType, '(unknown)'),\n ResourceTypeName = coalesce(ResourceType, tostring(x_RecommendationDetails.x_ResourceType), '(unknown)'),\n SubscriptionName = coalesce(SubAccountName, SubAccountId, '(unassigned)')\n//\n// Savings are monthly estimates; rows without an estimate count as 0 instead of null.\n| extend MonthlySavings = coalesce(x_EffectiveCostSavings, todouble(0))\n//\n// Recommendations have no BillingCurrency column, so apply the currency filter defensively and\n// never drop rows that don't declare a savings currency.\n| where isempty(selectedBillingCurrency) or isempty(SavingsCurrency) or SavingsCurrency =~ selectedBillingCurrency", + "id": "ede6c9ce-4b33-4c2e-b572-07e3361b242e", + "usedVariables": ["selectedBillingCurrency"] + }, + { + "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, + "text": "let data = materialize(CostRecommendations | where Source =~ 'Azure Advisor');\nlet monthlySavings = todouble(coalesce(toscalar(data | summarize sum(MonthlySavings)), 0.0));\nunion\n (data | summarize Value = todouble(count()) | extend Order = 1, Label = 'Recommendations', Type = 'Count'),\n (data | summarize Value = todouble(dcountif(ResourceId, isnotempty(ResourceId))) | extend Order = 2, Label = 'Impacted resources', Type = 'Count'),\n (data | summarize Value = todouble(dcountif(SubAccountId, isnotempty(SubAccountId))) | extend Order = 3, Label = 'Subscriptions', Type = 'Count'),\n (print Value = monthlySavings | extend Order = 4, Label = 'Est. monthly savings', Type = 'Savings'),\n (print Value = monthlySavings * 12 | extend Order = 5, Label = 'Est. annual savings', Type = 'Savings')\n| order by Order asc\n| project Label, Value = numberstring(round(Value, 2)), Type", + "id": "d7791e54-e5b1-4d2f-9891-07fc3d1c42c1", + "usedVariables": ["CostRecommendations"] + }, + { + "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, + "text": "CostRecommendations\n| where Source =~ 'Azure Advisor'\n| summarize Recommendations = count() by Impact = iff(isempty(Impact), 'Unspecified', Impact)\n| order by Recommendations desc", + "id": "542437d4-1f4e-41d1-911e-254c796bcd23", + "usedVariables": ["CostRecommendations"] + }, + { + "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, + "text": "CostRecommendations\n| where Source =~ 'Azure Advisor'\n| summarize Savings = round(sum(MonthlySavings), 2) by RecommendationType\n| where Savings > 0\n| order by Savings desc\n| limit maxGroupCount\n| project Recommendation = RecommendationType, Savings", + "id": "cf085d1b-ae69-4b44-b54d-e43acf4001b5", + "usedVariables": ["CostRecommendations", "maxGroupCount"] + }, + { + "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, + "text": "CostRecommendations\n| where Source =~ 'Azure Advisor'\n| summarize Savings = round(sum(MonthlySavings), 2) by ResourceTypeName\n| where Savings > 0\n| order by Savings desc\n| limit maxGroupCount\n| project ResourceType = ResourceTypeName, Savings", + "id": "d56a9ad3-e68b-4eae-a32b-bdf0c3509414", + "usedVariables": ["CostRecommendations", "maxGroupCount"] + }, + { + "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, + "text": "CostRecommendations\n| where Source =~ 'Azure Advisor'\n| summarize Savings = round(sum(MonthlySavings), 2) by SubscriptionName\n| where Savings > 0\n| order by Savings desc\n| limit maxGroupCount\n| project Subscription = SubscriptionName, Savings", + "id": "3c7816aa-8038-440b-b635-fda289580e55", + "usedVariables": ["CostRecommendations", "maxGroupCount"] + }, + { + "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, + "text": "CostRecommendations\n| where Source =~ 'Azure Advisor'\n| project\n Recommendation = RecommendationType,\n Impact = iff(isempty(Impact), 'Unspecified', Impact),\n Resource = ResourceName,\n ResourceType = ResourceTypeName,\n ResourceGroup = x_ResourceGroupName,\n Subscription = SubscriptionName,\n Solution,\n Currency = SavingsCurrency,\n MonthlySavings = round(MonthlySavings, 2),\n LastSeen = x_IngestionTime\n| order by MonthlySavings desc, Recommendation asc", + "id": "6de2078e-9402-4e08-b977-5cf2d8560e96", + "usedVariables": ["CostRecommendations"] + }, + { + "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, + "text": "let data = materialize(CostRecommendations | where Source !~ 'Azure Advisor');\nlet monthlySavings = todouble(coalesce(toscalar(data | summarize sum(MonthlySavings)), 0.0));\nunion\n (data | summarize Value = todouble(count()) | extend Order = 1, Label = 'Recommendations', Type = 'Count'),\n (data | summarize Value = todouble(dcountif(ResourceId, isnotempty(ResourceId))) | extend Order = 2, Label = 'Impacted resources', Type = 'Count'),\n (data | summarize Value = todouble(dcountif(x_SourceType, isnotempty(x_SourceType))) | extend Order = 3, Label = 'Recommendation types', Type = 'Count'),\n (print Value = monthlySavings | extend Order = 4, Label = 'Est. monthly savings', Type = 'Savings'),\n (print Value = monthlySavings * 12 | extend Order = 5, Label = 'Est. annual savings', Type = 'Savings')\n| order by Order asc\n| project Label, Value = numberstring(round(Value, 2)), Type", + "id": "58cee477-2b9c-4916-9150-17ef57be2f47", + "usedVariables": ["CostRecommendations"] + }, + { + "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, + "text": "CostRecommendations\n| where Source !~ 'Azure Advisor'\n| summarize Recommendations = count() by Type = coalesce(x_SourceType, RecommendationType, '(unknown)')\n| order by Recommendations desc", + "id": "4e754e2a-08d8-4b4b-bf60-7b692d09e977", + "usedVariables": ["CostRecommendations"] + }, + { + "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, + "text": "CostRecommendations\n| where Source !~ 'Azure Advisor'\n| project\n Recommendation = RecommendationType,\n Type = x_SourceType,\n Resource = ResourceName,\n ResourceType = ResourceTypeName,\n ResourceGroup = x_ResourceGroupName,\n Subscription = SubscriptionName,\n MonthlySavings = round(MonthlySavings, 2),\n LastSeen = x_IngestionTime\n| order by MonthlySavings desc, Recommendation asc", + "id": "dda8c8c5-4561-442e-957d-e2968f6e9bf0", + "usedVariables": ["CostRecommendations"] + }, + { + "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, + "text": "// Reports every collection, not just the latest snapshot, so gaps and stale sources are visible.\nRecommendations()\n| summarize\n Snapshots = dcount(bin(x_IngestionTime, 1h)),\n Rows = count(),\n FirstCollected = min(x_IngestionTime),\n LastCollected = max(x_IngestionTime)\n by Source = x_SourceName, Type = x_SourceType\n| extend AgeInDays = round((now() - LastCollected) / 1d, 1)\n| order by Source asc, Type asc", + "id": "88413f07-af1a-4ba9-afb8-61fe20eab29c", + "usedVariables": [] + }, { "dataSource": { "kind": "inline", "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" }, "text": "let monthname = dynamic(['(ignore)', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']);\nlet costs = materialize(\n CostsLastMonth\n | summarize BilledCost = round(sum(BilledCost), 2), EffectiveCost = round(sum(EffectiveCost), 2) by BillingPeriodStart = startofmonth(BillingPeriodStart)\n | extend json = todynamic(strcat('[{\"type\":\"Billed cost\", \"Cost\":', BilledCost, '}, {\"type\":\"Effective cost\", \"Cost\":', EffectiveCost, '}]'))\n | mv-expand json\n | project Type = strcat(json.type, ' (', monthname[monthofyear(BillingPeriodStart)], ' ', format_datetime(BillingPeriodStart, 'yyyy'), ')'), Cost = todouble(json.Cost)\n);\ncosts", @@ -3328,5 +3902,50 @@ "id": "f2a8c4d6-3b5e-4a7f-9c2d-8e5b1f4a7d9c", "usedVariables": [] } - ] + , + { + "dataSource": { + "kind": "inline", + "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" + }, + "text": "let _isAI = (svcCat: string, svcName: string) { svcCat == 'AI and Machine Learning' or svcName == 'Azure AI Search' };\nlet aiCosts = CostsByDay | where _isAI(ServiceCategory, ServiceName);\nlet totalAI = toscalar(aiCosts | summarize round(sum(EffectiveCost), 2));\nlet totalCloud = toscalar(CostsByDay | summarize sum(EffectiveCost));\nlet aiPercent = iff(totalCloud > 0, round(totalAI / totalCloud * 100, 1), 0.0);\nlet resourceCount = toscalar(aiCosts | summarize dcount(ResourceName));\nlet currentMonthStart = startofmonth(now());\nlet priorMonthStart = startofmonth(datetime_add('month', -1, now()));\nlet currentMonthAI = toscalar(\n CostsPlus\n | where (ServiceCategory == 'AI and Machine Learning' or ServiceName == 'Azure AI Search')\n | where ChargePeriodStart >= currentMonthStart\n | summarize sum(EffectiveCost)\n);\nlet priorMonthAI = toscalar(\n CostsPlus\n | where (ServiceCategory == 'AI and Machine Learning' or ServiceName == 'Azure AI Search')\n | where ChargePeriodStart >= priorMonthStart and ChargePeriodStart < currentMonthStart\n | summarize sum(EffectiveCost)\n);\nlet momChange = iff(priorMonthAI > 0, round((currentMonthAI - priorMonthAI) / priorMonthAI * 100, 1), 0.0);\nprint\n ['Total AI Cost'] = totalAI,\n ['AI % of Cloud'] = aiPercent,\n ['MoM Change %'] = momChange,\n ['AI Resources'] = resourceCount", + "id": "b2c3d4e5-f6a7-4b8c-9d0e-1f2a3b4c5d6e", + "usedVariables": [ + "CostsByDay", + "CostsPlus" + ] + }, + { + "dataSource": { + "kind": "inline", + "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" + }, + "text": "CostsByDay\n| where (ServiceCategory == 'AI and Machine Learning' or ServiceName == 'Azure AI Search')\n| summarize EffectiveCost = round(sum(EffectiveCost), 2) by bin(ChargePeriodStart, 1d)\n| order by ChargePeriodStart asc", + "id": "c3d4e5f6-a7b8-4c9d-0e1f-2a3b4c5d6e7f", + "usedVariables": [ + "CostsByDay" + ] + }, + { + "dataSource": { + "kind": "inline", + "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" + }, + "text": "let aiCosts = CostsByMonth | where (ServiceCategory == 'AI and Machine Learning' or ServiceName == 'Azure AI Search');\nlet totalAI = toscalar(aiCosts | summarize sum(EffectiveCost));\naiCosts\n| summarize\n EffectiveCost = round(sum(EffectiveCost), 2),\n ResourceCount = dcount(ResourceName)\n by ServiceName\n| extend SharePercent = iff(totalAI > 0, round(EffectiveCost / totalAI * 100, 1), 0.0)\n| order by EffectiveCost desc", + "id": "d4e5f6a7-b8c9-4d0e-1f2a-3b4c5d6e7f80", + "usedVariables": [ + "CostsByMonth" + ] + }, + { + "dataSource": { + "kind": "inline", + "dataSourceId": "23540be2-ffc9-4b61-8c4c-05e493e682a6" + }, + "text": "CostsByDay\n| where (ServiceCategory == 'AI and Machine Learning' or ServiceName == 'Azure AI Search')\n| summarize\n TotalCost = round(sum(EffectiveCost), 2),\n DailyAvg = round(avg(EffectiveCost), 2)\n by ResourceName, ServiceName\n| top 5 by TotalCost desc", + "id": "e5f6a7b8-c9d0-4e1f-2a3b-4c5d6e7f8091", + "usedVariables": [ + "CostsByDay" + ] + }] } diff --git a/src/templates/finops-hub/main.bicep b/src/templates/finops-hub/main.bicep index d93fe98c4..d8c65a191 100644 --- a/src/templates/finops-hub/main.bicep +++ b/src/templates/finops-hub/main.bicep @@ -48,6 +48,50 @@ param enableAHBRecommendations bool = false @description('Optional. Enable non-Spot AKS cluster recommendations that flag AKS clusters with autoscaling but not using Spot VMs. May generate noise since Spot VMs are only appropriate for interruptible workloads. Requires enableRecommendations. Default: false.') param enableSpotRecommendations bool = false +@description('Optional. Enable automatic download of Microsoft invoice files into the hub data lake. Only supported for Microsoft Customer Agreement (MCA) and Microsoft Partner Agreement (MPA) billing accounts. The Data Factory managed identity requires Billing Reader role on the billing account. Default: false.') +param enableInvoiceDownload bool = false + +@description('Optional. Billing account IDs to download invoices for, separated by a new line, comma, or semicolon. Requires enableInvoiceDownload. Leave empty to use the billing account scopes monitored by this hub. Default: "" (none).') +param invoiceBillingAccounts string = '' + +@description('Optional. Day of the month to download invoices from the previous month. Requires enableInvoiceDownload. Default: 10.') +@minValue(1) +@maxValue(28) +param invoiceScheduleDay int = 10 + +@description('Optional. Enable ingestion of FOCUS cost data exported from Amazon Web Services. Requires an S3 bucket with a FOCUS 1.2 export and an access key provided during deployment. Default: false.') +param enableAwsFocusIngestion bool = false + +@description('Optional. Name of the Amazon S3 bucket that contains the FOCUS export. Requires enableAwsFocusIngestion.') +param awsBucketName string = '' + +@description('Optional. Path to the export root folder within the S3 bucket, without leading or trailing slashes. This is the folder that contains the "data" and "metadata" subfolders. Example: "reports/focus-export". Requires enableAwsFocusIngestion.') +param awsBucketPath string = '' + +@description('Optional. Amazon Web Services account ID that owns the FOCUS export. Requires enableAwsFocusIngestion.') +param awsAccountId string = '' + +@description('Optional. Amazon Web Services region of the S3 bucket. Leave empty to use the global S3 endpoint. Requires enableAwsFocusIngestion. Default: "" (global).') +param awsRegion string = '' + +@description('Optional. Amazon Web Services access key ID used to read the S3 bucket. Requires enableAwsFocusIngestion.') +param awsAccessKeyId string = '' + +@description('Optional. Amazon Web Services secret access key used to read the S3 bucket. Stored in Key Vault. Requires enableAwsFocusIngestion.') +@secure() +param awsSecretAccessKey string = '' + +@description('Optional. FOCUS version of the Amazon Web Services export. Requires enableAwsFocusIngestion. Default: "1.2".') +@allowed([ + '1.2' +]) +param awsFocusVersion string = '1.2' + +@description('Optional. Hour of the day (UTC) to collect multicloud FOCUS files. Default: 4.') +@minValue(0) +@maxValue(23) +param multiCloudScheduleHour int = 4 + @description('Optional. Name of the Azure Data Explorer cluster to use for advanced analytics. If empty, Azure Data Explorer will not be deployed. Required to use with Power BI if you have more than $2-5M/mo in costs being monitored. Default: "" (do not use).') param dataExplorerName string = '' @@ -166,12 +210,22 @@ param enableNatGateway bool = false param virtualNetworkAddressPrefix string = '10.20.30.0/26' +//============================================================================== +// Variables +//============================================================================== + +// Accept new lines, commas, and semicolons as separators for billing account IDs. +var invoiceBillingAccountArray = filter( + map(split(replace(replace(replace(invoiceBillingAccounts, '\r\n', '\n'), ',', '\n'), ';', '\n'), '\n'), id => trim(id)), + id => !empty(id) +) + + //============================================================================== // Resources //============================================================================== -module hub 'modules/hub.bicep' = { - name: 'hub' +module hub 'modules/hub.bicep' = { name: 'hub' params: { hubName: hubName location: location @@ -183,6 +237,18 @@ module hub 'modules/hub.bicep' = { enableRecommendations: enableRecommendations enableAHBRecommendations: enableAHBRecommendations enableSpotRecommendations: enableSpotRecommendations + enableInvoiceDownload: enableInvoiceDownload + invoiceBillingAccounts: invoiceBillingAccountArray + invoiceScheduleDay: invoiceScheduleDay + enableAwsFocusIngestion: enableAwsFocusIngestion + awsBucketName: awsBucketName + awsBucketPath: awsBucketPath + awsAccountId: awsAccountId + awsRegion: awsRegion + awsAccessKeyId: awsAccessKeyId + awsSecretAccessKey: awsSecretAccessKey + awsFocusVersion: awsFocusVersion + multiCloudScheduleHour: multiCloudScheduleHour dataExplorerName: dataExplorerName dataExplorerSku: dataExplorerSku dataExplorerCapacity: dataExplorerCapacity diff --git a/src/templates/finops-hub/modules/Microsoft.Billing/Invoices/README.md b/src/templates/finops-hub/modules/Microsoft.Billing/Invoices/README.md new file mode 100644 index 000000000..f745d8834 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.Billing/Invoices/README.md @@ -0,0 +1,84 @@ +# Microsoft.Billing/Invoices + +Downloads Microsoft invoice files into the FinOps hub data lake so you can reconcile invoices against your cost data and keep an auditable archive. + +## What gets deployed + +| Resource | Name | Description | +| -------- | ---- | ----------- | +| Linked service | `invoices_download` | Generic HTTP linked service used to download invoice files. | +| Dataset | `invoices_download` | Binary source for the short-lived SAS URL returned by the Billing API. | +| Dataset | `invoices_file` | Binary sink in the ingestion container. | +| Pipeline | `invoices_DownloadInvoices` | Resolves the billing accounts to process and runs the download pipeline for each one. | +| Pipeline | `invoices_DownloadBillingAccountInvoices` | Downloads all invoices for a single billing account. | +| Pipeline | `invoices_DownloadInvoiceFile` | Requests a download URL for a single invoice and saves the file. Polls the Billing API while the request is still running. | +| Trigger | `invoices_MonthlySchedule` | Runs once a month to download invoices from the previous month. | + +Invoice files are saved in the **ingestion** container using the following hierarchy: + +```text +ingestion/ +└── invoices/ + └── {YYYY-MM}/ + └── {billingProfileId}/ + └── {purchaseOrderNumber}/ + └── {invoiceNumber}.pdf +``` + +The billing profile ID is used instead of the display name to avoid spaces and special characters in path names. Invoices without a purchase order number are saved in a `no-po` folder. + +## Requirements + +| # | Requirement | Notes | +| - | ----------- | ----- | +| 1 | Microsoft Customer Agreement (MCA) or Microsoft Partner Agreement (MPA) billing account | Legacy Enterprise Agreement (EA) billing accounts do not support invoice downloads. The pipeline returns an empty list and completes successfully. | +| 2 | Billing account ID | Find it in **Cost Management + Billing** > **Properties**. | +| 3 | `Billing Reader` role for the hub managed identity | Must be granted after deployment. See below. | + +## Configuration + +Enable the feature with the **Invoices** step in the deployment wizard, or set the following template parameters: + +| Parameter | Description | +| --------- | ----------- | +| `enableInvoiceDownload` | Set to `true` to deploy the app. Default: `false`. | +| `invoiceBillingAccounts` | Billing account IDs to download invoices for, separated by a new line, comma, or semicolon. Leave empty to use the billing account scopes monitored by the hub. | +| `invoiceScheduleDay` | Day of the month to download invoices from the previous month. Default: `10`. | + +Billing accounts are stored in the `invoices.billingAccounts` array in `settings.json` in the **config** container. You can update that array directly, but the value is overwritten on the next deployment. + +## Grant the Billing Reader role + +Billing account scopes exist outside of any Azure subscription and are not part of Azure RBAC, so the role assignment cannot be created during deployment and `az role assignment create` doesn't work. Grant it after the hub is deployed: + +```powershell +Add-FinOpsHubBillingReader -BillingAccountId '' +``` + +Or grant it in the Azure portal under **Cost Management + Billing** > your billing account > **Access control (IAM)**, assigning the **Billing account reader** role to the Data Factory managed identity. + +## Validate the deployment + +1. Open the hub Data Factory and run the `invoices_DownloadInvoices` pipeline in debug mode. +2. Confirm each activity succeeds: + - `Load Settings` returns the hub settings. + - `List Invoices` returns a populated `value` array. + - `Request Download URL` returns a download URL for each invoice, or a 202 status followed by `Until Download URL Is Ready` completing. + - `Save Invoice File` reports more than 0 bytes written. +3. Confirm the files exist in the `invoices` folder of the ingestion container. + +## Troubleshooting + +| Symptom | Cause | Resolution | +| ------- | ----- | ---------- | +| `List Invoices` returns 401 or 403 | The managed identity is missing the `Billing Reader` role, or it was granted at the wrong scope. | Run `Add-FinOpsHubBillingReader` and confirm the scope is the billing account, not the resource group. | +| `List Invoices` returns an empty array | There are no invoices for the period, or the billing account is a legacy EA account. | Run the pipeline with `periodOffsetMonths` set to `-2`. Confirm the account type in **Cost Management + Billing** > **Properties**. | +| `Download Invoices Per Billing Account` iterates 0 times | No billing accounts are configured and no billing account scopes are monitored. | Set `invoiceBillingAccounts`, or add a billing account scope to the hub. | +| `Request Download URL` returns 404 | The invoice ID is malformed. | Check the `List Invoices` output and confirm each `id` starts with `/providers/Microsoft.Billing/`. | +| `Missing Download URL` fails the pipeline | The Billing API accepted the request but never returned a download URL within 30 minutes. | Confirm the invoice is available for download in the portal. Increase the `Until Download URL Is Ready` timeout if the account consistently takes longer. | +| `Save Invoice File` fails with an expired URL | Too much time elapsed between requesting the URL and copying the file. | Download URLs expire in about an hour. Reduce the `batchCount` on the `Download Invoices` loop. | +| `Save Invoice File` fails with a permission error | The managed identity is missing `Storage Blob Data Contributor` on the hub storage account. | Redeploy the hub. This role is granted automatically. | + +## Cost + +Only Data Factory activity runs and a small amount of storage are added. For a typical account with about 30 invoices a month, expect roughly 1 USD per month. diff --git a/src/templates/finops-hub/modules/Microsoft.Billing/Invoices/app.bicep b/src/templates/finops-hub/modules/Microsoft.Billing/Invoices/app.bicep new file mode 100644 index 000000000..2ac069c0e --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.Billing/Invoices/app.bicep @@ -0,0 +1,913 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { finOpsToolkitVersion, HubAppProperties, isSupportedVersion } from '../../fx/hub-types.bicep' +import { AppMetadata as CoreMetadata } from '../../Microsoft.FinOpsHubs/Core/metadata.bicep' +import { AppMetadata as InvoicesMetadata } from './metadata.bicep' + +metadata hubApp = { + id: 'Microsoft.Billing.Invoices' + version: '$$ftkver$$' + dependencies: [ + 'Microsoft.FinOpsHubs.Core' + ] + metadata: 'https://microsoft.github.io/finops-toolkit/deploy/finops-hub/$$ftkver$$/Microsoft.Billing/Invoices/metadata.bicep' +} + + +//============================================================================== +// Parameters +//============================================================================== + +@description('Required. FinOps hub app getting deployed.') +param app HubAppProperties + +@description('Required. Metadata describing shared resources from the Core app. Must be v13 or higher.') +@validate(x => isSupportedVersion(x.version, '13.0', ''), 'Core app version must be 13.0 or higher.') +param core CoreMetadata + +@description('Optional. Day of the month to download invoices from the previous month. Invoices are generally available within the first few days of the month. Default: 10.') +@minValue(1) +@maxValue(28) +param scheduleDay int = 10 + + +//============================================================================== +// Variables +//============================================================================== + +var INVOICES = 'invoices' + +// API version used for all Microsoft.Billing invoice operations. +var billingApiVersion = '2024-04-01' + + +//============================================================================== +// Resources +//============================================================================== + +// Register app +module appRegistration '../../fx/hub-app.bicep' = { + name: 'Microsoft.Billing.Invoices_Register' + params: { + app: app + version: finOpsToolkitVersion + features: [ + 'DataFactory' + ] + } +} + +// Get data factory instance +resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = { + name: app.dataFactory + dependsOn: [appRegistration] + + resource dataset_config 'datasets@2018-06-01' existing = { + name: core.datasets.config + } +} + +//------------------------------------------------------------------------------ +// Linked services +//------------------------------------------------------------------------------ + +// Generic HTTP linked service used to download the short-lived SAS URL returned by the Billing API. +resource linkedService_invoiceDownload 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = { + name: '${INVOICES}_download' + parent: dataFactory + properties: { + annotations: [] + type: 'HttpServer' + parameters: { + baseUrl: { + type: 'String' + } + } + typeProperties: { + url: '@{linkedService().baseUrl}' + enableServerCertificateValidation: true + authenticationType: 'Anonymous' + } + } +} + +//------------------------------------------------------------------------------ +// Datasets +//------------------------------------------------------------------------------ + +// Binary source pointing at the short-lived SAS URL returned by the Billing API. +resource dataset_invoiceDownload 'Microsoft.DataFactory/factories/datasets@2018-06-01' = { + name: '${INVOICES}_download' + parent: dataFactory + properties: { + annotations: [] + type: 'Binary' + parameters: { + downloadUrl: { + type: 'String' + } + } + linkedServiceName: { + referenceName: linkedService_invoiceDownload.name + type: 'LinkedServiceReference' + parameters: { + baseUrl: { + value: '@dataset().downloadUrl' + type: 'Expression' + } + } + } + typeProperties: { + location: { + type: 'HttpServerLocation' + relativeUrl: '' + } + } + } +} + +// Binary sink in the hub data lake. Files are saved in the ingestion container. +resource dataset_invoiceFile 'Microsoft.DataFactory/factories/datasets@2018-06-01' = { + name: '${INVOICES}_file' + parent: dataFactory + properties: { + annotations: [] + type: 'Binary' + parameters: { + folderPath: { + type: 'String' + } + fileName: { + type: 'String' + } + } + linkedServiceName: { + parameters: {} + referenceName: app.storage + type: 'LinkedServiceReference' + } + typeProperties: { + location: { + type: 'AzureBlobFSLocation' + fileSystem: core.containers.ingestion + folderPath: { + value: '@dataset().folderPath' + type: 'Expression' + } + fileName: { + value: '@dataset().fileName' + type: 'Expression' + } + } + } + } +} + +//------------------------------------------------------------------------------ +// Pipelines +//------------------------------------------------------------------------------ + +// Downloads a single invoice file. Split out from the per-billing-account pipeline because +// Data Factory does not support an Until activity nested inside a ForEach activity, and because +// a child pipeline gives each invoice its own variable scope, which keeps the parent ForEach parallel. +resource pipeline_DownloadInvoiceFile 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: '${INVOICES}_DownloadInvoiceFile' + parent: dataFactory + properties: { + description: 'Requests a download URL for a single invoice and saves the file in the hub data lake.' + parameters: { + invoiceId: { + type: 'String' + } + folderPath: { + type: 'String' + } + fileName: { + type: 'String' + } + } + variables: { + downloadUrl: { + type: 'String' + } + pollUrl: { + type: 'String' + } + } + activities: [ + { // Request Download URL + name: 'Request Download URL' + description: 'Request a short-lived SAS URL for the invoice file. This is a long-running operation: the API may return 200 with the URL or 202 with a Location header to poll.' + type: 'WebActivity' + dependsOn: [] + policy: { + timeout: '0.00:05:00' + retry: 3 + retryIntervalInSeconds: 30 + secureOutput: true + secureInput: false + } + userProperties: [] + typeProperties: { + // invoiceId starts with a slash, so strip it before appending to the ARM endpoint. + url: { + value: '@concat(\'${environment().resourceManager}\', substring(pipeline().parameters.invoiceId, 1, sub(length(pipeline().parameters.invoiceId), 1)), \'/download?api-version=${billingApiVersion}\')' + type: 'Expression' + } + method: 'POST' + body: '{}' + authentication: { + type: 'MSI' + resource: environment().resourceManager + } + } + } + { // Set Download URL + name: 'Set Download URL' + description: 'Capture the download URL when the API completed synchronously.' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Request Download URL' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + secureOutput: true + secureInput: true + } + userProperties: [] + typeProperties: { + variableName: 'downloadUrl' + value: { + value: '@if(contains(activity(\'Request Download URL\').output, \'url\'), activity(\'Request Download URL\').output.url, \'\')' + type: 'Expression' + } + } + } + { // Set Poll URL + name: 'Set Poll URL' + description: 'Capture the async operation URL when the API returned 202 Accepted. Data Factory does not follow long-running operations automatically.' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Set Download URL' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + secureOutput: true + secureInput: true + } + userProperties: [] + typeProperties: { + variableName: 'pollUrl' + value: { + value: '@if(contains(activity(\'Request Download URL\').output, \'ADFWebActivityResponseHeaders\'), if(contains(activity(\'Request Download URL\').output.ADFWebActivityResponseHeaders, \'Location\'), activity(\'Request Download URL\').output.ADFWebActivityResponseHeaders.Location, if(contains(activity(\'Request Download URL\').output.ADFWebActivityResponseHeaders, \'Azure-AsyncOperation\'), activity(\'Request Download URL\').output.ADFWebActivityResponseHeaders[\'Azure-AsyncOperation\'], \'\')), \'\')' + type: 'Expression' + } + } + } + { // Until Download URL Is Ready + name: 'Until Download URL Is Ready' + description: 'Poll the async operation until it returns the download URL. Exits immediately when the URL is already known or there is nothing to poll.' + type: 'Until' + dependsOn: [ + { + activity: 'Set Poll URL' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + expression: { + value: '@or(not(empty(variables(\'downloadUrl\'))), empty(variables(\'pollUrl\')))' + type: 'Expression' + } + timeout: '0.00:30:00' + activities: [ + { // If Download Is Pending + name: 'If Download Is Pending' + type: 'IfCondition' + dependsOn: [] + userProperties: [] + typeProperties: { + expression: { + value: '@and(empty(variables(\'downloadUrl\')), not(empty(variables(\'pollUrl\'))))' + type: 'Expression' + } + ifTrueActivities: [ + { // Wait For Download + name: 'Wait For Download' + type: 'Wait' + dependsOn: [] + userProperties: [] + typeProperties: { + waitTimeInSeconds: 15 + } + } + { // Check Download Status + name: 'Check Download Status' + description: 'Check whether the invoice document is ready. Returns 202 while running and 200 with the download URL when complete.' + type: 'WebActivity' + dependsOn: [ + { + activity: 'Wait For Download' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + timeout: '0.00:05:00' + retry: 3 + retryIntervalInSeconds: 30 + secureOutput: true + secureInput: true + } + userProperties: [] + typeProperties: { + url: { + value: '@variables(\'pollUrl\')' + type: 'Expression' + } + method: 'GET' + authentication: { + type: 'MSI' + resource: environment().resourceManager + } + } + } + { // Update Download URL + name: 'Update Download URL' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Check Download Status' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + secureOutput: true + secureInput: true + } + userProperties: [] + typeProperties: { + variableName: 'downloadUrl' + value: { + value: '@if(contains(activity(\'Check Download Status\').output, \'url\'), activity(\'Check Download Status\').output.url, \'\')' + type: 'Expression' + } + } + } + ] + } + } + ] + } + } + { // Verify Download URL + name: 'Verify Download URL' + description: 'Fail with a clear error when the download URL was never returned, instead of letting the copy fail on an empty URL.' + type: 'IfCondition' + dependsOn: [ + { + activity: 'Until Download URL Is Ready' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + expression: { + value: '@empty(variables(\'downloadUrl\'))' + type: 'Expression' + } + ifTrueActivities: [ + { + name: 'Missing Download URL' + type: 'Fail' + dependsOn: [] + userProperties: [] + typeProperties: { + message: { + value: '@concat(\'The Billing API did not return a download URL for invoice \', pipeline().parameters.fileName, \'. The request may still be in progress or the invoice may not be available for download.\')' + type: 'Expression' + } + errorCode: 'InvoiceDownloadUrlNotAvailable' + } + } + ] + } + } + { // Save Invoice File + name: 'Save Invoice File' + description: 'Copy the invoice file from the SAS URL into the hub data lake.' + type: 'Copy' + dependsOn: [ + { + activity: 'Verify Download URL' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + timeout: '0.00:15:00' + retry: 2 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: true + } + userProperties: [] + typeProperties: { + source: { + type: 'BinarySource' + storeSettings: { + type: 'HttpReadSettings' + requestMethod: 'GET' + } + formatSettings: { + type: 'BinaryReadSettings' + } + } + sink: { + type: 'BinarySink' + storeSettings: { + type: 'AzureBlobFSWriteSettings' + } + } + enableStaging: false + } + inputs: [ + { + referenceName: dataset_invoiceDownload.name + type: 'DatasetReference' + parameters: { + downloadUrl: { + value: '@variables(\'downloadUrl\')' + type: 'Expression' + } + } + } + ] + outputs: [ + { + referenceName: dataset_invoiceFile.name + type: 'DatasetReference' + parameters: { + folderPath: { + value: '@pipeline().parameters.folderPath' + type: 'Expression' + } + fileName: { + value: '@pipeline().parameters.fileName' + type: 'Expression' + } + } + } + ] + } + ] + policy: { + elapsedTimeMetric: {} + } + annotations: [] + } +} + +// Downloads all invoices for a single billing account. Split out from the orchestrator +// pipeline because Data Factory does not support nested ForEach activities. +resource pipeline_DownloadBillingAccountInvoices 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: '${INVOICES}_DownloadBillingAccountInvoices' + parent: dataFactory + properties: { + description: 'Downloads invoice files for a single billing account and saves them in the hub data lake.' + parameters: { + billingAccountId: { + type: 'String' + } + periodOffsetMonths: { + type: 'Int' + defaultValue: -1 + } + } + variables: { + periodStart: { + type: 'String' + } + periodEnd: { + type: 'String' + } + } + activities: [ + { // Set Period Start + name: 'Set Period Start' + description: 'First day of the month being downloaded.' + type: 'SetVariable' + dependsOn: [] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'periodStart' + value: { + value: '@formatDateTime(startOfMonth(addToTime(utcNow(), pipeline().parameters.periodOffsetMonths, \'Month\')), \'yyyy-MM-dd\')' + type: 'Expression' + } + } + } + { // Set Period End + name: 'Set Period End' + description: 'Last day of the month being downloaded.' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Set Period Start' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'periodEnd' + value: { + value: '@formatDateTime(addDays(addToTime(startOfMonth(addToTime(utcNow(), pipeline().parameters.periodOffsetMonths, \'Month\')), 1, \'Month\'), -1), \'yyyy-MM-dd\')' + type: 'Expression' + } + } + } + { // List Invoices + name: 'List Invoices' + description: 'List all invoices for the billing account within the requested period. Returns an empty list for billing accounts that do not support invoice downloads (for example, legacy Enterprise Agreement accounts).' + type: 'WebActivity' + dependsOn: [ + { + activity: 'Set Period End' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + timeout: '0.00:10:00' + retry: 3 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + url: { + value: '@concat(\'${environment().resourceManager}providers/Microsoft.Billing/billingAccounts/\', pipeline().parameters.billingAccountId, \'/invoices?api-version=${billingApiVersion}&periodStartDate=\', variables(\'periodStart\'), \'&periodEndDate=\', variables(\'periodEnd\'))' + type: 'Expression' + } + method: 'GET' + authentication: { + type: 'MSI' + resource: environment().resourceManager + } + } + } + { // Download Invoices + name: 'Download Invoices' + description: 'Download each invoice file. The download URL is a short-lived SAS URL, so keep concurrency low to avoid expiration.' + type: 'ForEach' + dependsOn: [ + { + activity: 'List Invoices' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@coalesce(activity(\'List Invoices\').output.value, json(\'[]\'))' + type: 'Expression' + } + isSequential: false + batchCount: 3 + activities: [ + { // Download Invoice File + name: 'Download Invoice File' + type: 'ExecutePipeline' + dependsOn: [] + policy: { + secureInput: false + } + userProperties: [] + typeProperties: { + pipeline: { + referenceName: pipeline_DownloadInvoiceFile.name + type: 'PipelineReference' + } + waitOnCompletion: true + parameters: { + invoiceId: { + value: '@item().id' + type: 'Expression' + } + folderPath: { + // invoices/// + // billingProfileId and purchaseOrderNumber are optional. Referencing a property that + // is missing from the response fails to evaluate, so guard each one with contains(). + value: '@concat(\'${INVOICES}/\', formatDateTime(item().properties.invoicePeriodStartDate, \'yyyy-MM\'), \'/\', if(contains(item().properties, \'billingProfileId\'), last(split(item().properties.billingProfileId, \'/\')), \'unknown\'), \'/\', if(and(contains(item().properties, \'purchaseOrderNumber\'), not(empty(item().properties.purchaseOrderNumber))), item().properties.purchaseOrderNumber, \'no-po\'))' + type: 'Expression' + } + fileName: { + value: '@concat(item().name, \'.pdf\')' + type: 'Expression' + } + } + } + } + ] + } + } + ] + policy: { + elapsedTimeMetric: {} + } + annotations: [] + } +} + +// Resolves the billing accounts to download invoices for and runs the download pipeline for each one. +resource pipeline_DownloadInvoices 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: '${INVOICES}_DownloadInvoices' + parent: dataFactory + properties: { + description: 'Downloads Microsoft invoice files for all configured billing accounts and saves them in the hub data lake.' + parameters: { + periodOffsetMonths: { + type: 'Int' + defaultValue: -1 + } + } + variables: { + billingAccounts: { + type: 'Array' + } + } + activities: [ + { // Load Settings + name: 'Load Settings' + description: 'Read hub settings to determine which billing accounts to download invoices for.' + type: 'Lookup' + dependsOn: [] + policy: { + timeout: '0.00:05:00' + retry: 2 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + source: { + type: 'JsonSource' + storeSettings: { + type: 'AzureBlobFSReadSettings' + recursive: false + enablePartitionDiscovery: false + } + formatSettings: { + type: 'JsonReadSettings' + } + } + dataset: { + referenceName: dataFactory::dataset_config.name + type: 'DatasetReference' + parameters: { + fileName: core.settings.file + folderPath: core.settings.container + } + } + firstRowOnly: true + } + } + { // Find Monitored Billing Accounts + name: 'Find Monitored Billing Accounts' + description: 'Fall back to the billing account scopes monitored by this hub when no billing accounts are explicitly configured.' + type: 'Filter' + dependsOn: [ + { + activity: 'Load Settings' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@coalesce(activity(\'Load Settings\').output.firstRow.scopes, json(\'[]\'))' + type: 'Expression' + } + condition: { + // Billing account scopes only: /providers/Microsoft.Billing/billingAccounts/ + value: '@and(startswith(toLower(item().scope), \'/providers/microsoft.billing/billingaccounts/\'), equals(length(split(item().scope, \'/\')), 5))' + type: 'Expression' + } + } + } + { // Set Configured Billing Accounts + // Data Factory does not allow a container activity (ForEach) inside another container + // activity (If), so the configured and monitored billing accounts are resolved with two + // top-level loops instead. Each loop iterates over an empty array when it does not apply. + name: 'Set Configured Billing Accounts' + description: 'Add the explicitly configured billing accounts, if any.' + type: 'ForEach' + dependsOn: [ + { + activity: 'Find Monitored Billing Accounts' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@coalesce(activity(\'Load Settings\').output.firstRow.invoices.billingAccounts, json(\'[]\'))' + type: 'Expression' + } + isSequential: true + activities: [ + { + name: 'Append Configured Billing Account' + type: 'AppendVariable' + dependsOn: [] + userProperties: [] + typeProperties: { + variableName: 'billingAccounts' + value: { + value: '@item()' + type: 'Expression' + } + } + } + ] + } + } + { // Set Monitored Billing Accounts + name: 'Set Monitored Billing Accounts' + description: 'Extract the billing account ID from each monitored billing account scope. Skipped when billing accounts are explicitly configured.' + type: 'ForEach' + dependsOn: [ + { + activity: 'Set Configured Billing Accounts' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@if(greater(length(coalesce(activity(\'Load Settings\').output.firstRow.invoices.billingAccounts, json(\'[]\'))), 0), json(\'[]\'), activity(\'Find Monitored Billing Accounts\').output.Value)' + type: 'Expression' + } + isSequential: true + activities: [ + { + name: 'Append Monitored Billing Account' + type: 'AppendVariable' + dependsOn: [] + userProperties: [] + typeProperties: { + variableName: 'billingAccounts' + value: { + value: '@last(split(item().scope, \'/\'))' + type: 'Expression' + } + } + } + ] + } + } + { // Download Invoices Per Billing Account + name: 'Download Invoices Per Billing Account' + description: 'Run the download pipeline for each billing account. Executed sequentially to keep the number of concurrent Billing API calls low.' + type: 'ForEach' + dependsOn: [ + { + activity: 'Set Monitored Billing Accounts' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@variables(\'billingAccounts\')' + type: 'Expression' + } + isSequential: true + activities: [ + { + name: 'Download Billing Account Invoices' + type: 'ExecutePipeline' + dependsOn: [] + policy: { + secureInput: false + } + userProperties: [] + typeProperties: { + pipeline: { + referenceName: pipeline_DownloadBillingAccountInvoices.name + type: 'PipelineReference' + } + waitOnCompletion: true + parameters: { + billingAccountId: { + value: '@item()' + type: 'Expression' + } + periodOffsetMonths: { + value: '@pipeline().parameters.periodOffsetMonths' + type: 'Expression' + } + } + } + } + ] + } + } + ] + policy: { + elapsedTimeMetric: {} + } + annotations: [] + } +} + +//------------------------------------------------------------------------------ +// Scheduling +//------------------------------------------------------------------------------ + +module timeZones '../../Microsoft.CostManagement/ManagedExports/timeZones.bicep' = { + name: 'Microsoft.Billing.Invoices_TimeZones' + params: { + location: app.hub.location + } +} + +resource trigger_MonthlySchedule 'Microsoft.DataFactory/factories/triggers@2018-06-01' = { + name: '${INVOICES}_MonthlySchedule' + parent: dataFactory + properties: { + description: 'Downloads invoices from the previous month.' + pipelines: [ + { + pipelineReference: { + referenceName: pipeline_DownloadInvoices.name + type: 'PipelineReference' + } + parameters: { + periodOffsetMonths: -1 + } + } + ] + type: 'ScheduleTrigger' + typeProperties: { + recurrence: { + frequency: 'Month' + interval: 1 + startTime: '2023-01-10T06:00:00' + timeZone: timeZones.outputs.Timezone + schedule: { + monthDays: [ + scheduleDay + ] + hours: [ + 6 + ] + minutes: [ + 0 + ] + } + } + } + } +} + + +//============================================================================== +// Outputs +//============================================================================== + +@description('The app properties for the Invoices app.') +output app HubAppProperties = app + +@description('Metadata describing resources created by the Invoices app.') +output metadata InvoicesMetadata = { + id: 'Microsoft.Billing.Invoices' + version: finOpsToolkitVersion + storage: { + container: core.containers.ingestion + folder: INVOICES + } + datasets: { + invoiceDownload: dataset_invoiceDownload.name + invoiceFile: dataset_invoiceFile.name + } + linkedServices: { + invoiceDownload: linkedService_invoiceDownload.name + } + pipelines: { + downloadInvoices: pipeline_DownloadInvoices.name + } +} diff --git a/src/templates/finops-hub/modules/Microsoft.Billing/Invoices/metadata.bicep b/src/templates/finops-hub/modules/Microsoft.Billing/Invoices/metadata.bicep new file mode 100644 index 000000000..f03ab7f97 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.Billing/Invoices/metadata.bicep @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//============================================================================== +// App metadata definition +//============================================================================== + +@export() +@description('Metadata for resources created by the Invoices app.') +type AppMetadata = { + @description('Fully-qualified app identifier.') + id: string + @description('App version.') + version: string + @description('Storage container and folder where invoice files are saved.') + storage: { + @description('Container where invoice files are saved.') + container: string + @description('Root folder within the container where invoice files are saved.') + folder: string + } + @description('Data Factory dataset names.') + datasets: { + @description('Binary dataset for the invoice download URL.') + invoiceDownload: string + @description('Binary dataset for the invoice file saved in storage.') + invoiceFile: string + } + @description('Data Factory linked service names.') + linkedServices: { + @description('HTTP linked service used to download invoice files.') + invoiceDownload: string + } + @description('Data Factory pipeline names.') + pipelines: { + @description('Pipeline that downloads invoices for all configured billing accounts.') + downloadInvoices: string + } +} diff --git a/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep index b372acac7..bc5c86ba7 100644 --- a/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/app.bicep @@ -70,6 +70,7 @@ module schemaFiles '../../fx/hub-storage.bicep' = { 'schemas/amortizedcost_c360-2025-04.json': loadTextContent('./schemas/amortizedcost_c360-2025-04.json') 'schemas/focuscost_1.2.json': loadTextContent('./schemas/focuscost_1.2.json') 'schemas/focuscost_1.2-preview.json': loadTextContent('./schemas/focuscost_1.2-preview.json') + 'schemas/focuscost_1.2-aws.json': loadTextContent('./schemas/focuscost_1.2-aws.json') 'schemas/focuscost_1.0r2.json': loadTextContent('./schemas/focuscost_1.0r2.json') 'schemas/focuscost_1.0.json': loadTextContent('./schemas/focuscost_1.0.json') 'schemas/focuscost_1.0-preview(v1).json': loadTextContent('./schemas/focuscost_1.0-preview(v1).json') diff --git a/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.2-aws.json b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.2-aws.json new file mode 100644 index 000000000..19183e437 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas/focuscost_1.2-aws.json @@ -0,0 +1,232 @@ +{ + "additionalColumns": [], + "translator": { + "type": "TabularTranslator", + "mappings": [ + { + "source": { "name": "AvailabilityZone", "type": "String" }, + "sink": { "name": "AvailabilityZone" } + }, + { + "source": { "name": "BilledCost", "type": "Decimal" }, + "sink": { "name": "BilledCost" } + }, + { + "source": { "name": "BillingAccountId", "type": "String" }, + "sink": { "name": "BillingAccountId" } + }, + { + "source": { "name": "BillingAccountName", "type": "String" }, + "sink": { "name": "BillingAccountName" } + }, + { + "source": { "name": "BillingAccountType", "type": "String" }, + "sink": { "name": "BillingAccountType" } + }, + { + "source": { "name": "BillingCurrency", "type": "String" }, + "sink": { "name": "BillingCurrency" } + }, + { + "source": { "name": "BillingPeriodEnd", "type": "DateTimeOffset" }, + "sink": { "name": "BillingPeriodEnd" } + }, + { + "source": { "name": "BillingPeriodStart", "type": "DateTimeOffset" }, + "sink": { "name": "BillingPeriodStart" } + }, + { + "source": { "name": "CapacityReservationId", "type": "String" }, + "sink": { "name": "CapacityReservationId" } + }, + { + "source": { "name": "CapacityReservationStatus", "type": "String" }, + "sink": { "name": "CapacityReservationStatus" } + }, + { + "source": { "name": "ChargeCategory", "type": "String" }, + "sink": { "name": "ChargeCategory" } + }, + { + "source": { "name": "ChargeClass", "type": "String" }, + "sink": { "name": "ChargeClass" } + }, + { + "source": { "name": "ChargeDescription", "type": "String" }, + "sink": { "name": "ChargeDescription" } + }, + { + "source": { "name": "ChargeFrequency", "type": "String" }, + "sink": { "name": "ChargeFrequency" } + }, + { + "source": { "name": "ChargePeriodEnd", "type": "DateTimeOffset" }, + "sink": { "name": "ChargePeriodEnd" } + }, + { + "source": { "name": "ChargePeriodStart", "type": "DateTimeOffset" }, + "sink": { "name": "ChargePeriodStart" } + }, + { + "source": { "name": "CommitmentDiscountCategory", "type": "String" }, + "sink": { "name": "CommitmentDiscountCategory" } + }, + { + "source": { "name": "CommitmentDiscountId", "type": "String" }, + "sink": { "name": "CommitmentDiscountId" } + }, + { + "source": { "name": "CommitmentDiscountName", "type": "String" }, + "sink": { "name": "CommitmentDiscountName" } + }, + { + "source": { "name": "CommitmentDiscountQuantity", "type": "Decimal" }, + "sink": { "name": "CommitmentDiscountQuantity" } + }, + { + "source": { "name": "CommitmentDiscountStatus", "type": "String" }, + "sink": { "name": "CommitmentDiscountStatus" } + }, + { + "source": { "name": "CommitmentDiscountType", "type": "String" }, + "sink": { "name": "CommitmentDiscountType" } + }, + { + "source": { "name": "CommitmentDiscountUnit", "type": "String" }, + "sink": { "name": "CommitmentDiscountUnit" } + }, + { + "source": { "name": "ConsumedQuantity", "type": "Decimal" }, + "sink": { "name": "ConsumedQuantity" } + }, + { + "source": { "name": "ConsumedUnit", "type": "String" }, + "sink": { "name": "ConsumedUnit" } + }, + { + "source": { "name": "ContractedCost", "type": "Decimal" }, + "sink": { "name": "ContractedCost" } + }, + { + "source": { "name": "ContractedUnitPrice", "type": "Decimal" }, + "sink": { "name": "ContractedUnitPrice" } + }, + { + "source": { "name": "EffectiveCost", "type": "Decimal" }, + "sink": { "name": "EffectiveCost" } + }, + { + "source": { "name": "InvoiceId", "type": "String" }, + "sink": { "name": "InvoiceId" } + }, + { + "source": { "name": "InvoiceIssuerName", "type": "String" }, + "sink": { "name": "InvoiceIssuerName" } + }, + { + "source": { "name": "ListCost", "type": "Decimal" }, + "sink": { "name": "ListCost" } + }, + { + "source": { "name": "ListUnitPrice", "type": "Decimal" }, + "sink": { "name": "ListUnitPrice" } + }, + { + "source": { "name": "PricingCategory", "type": "String" }, + "sink": { "name": "PricingCategory" } + }, + { + "source": { "name": "PricingCurrency", "type": "String" }, + "sink": { "name": "PricingCurrency" } + }, + { + "source": { "name": "PricingQuantity", "type": "Decimal" }, + "sink": { "name": "PricingQuantity" } + }, + { + "source": { "name": "PricingUnit", "type": "String" }, + "sink": { "name": "PricingUnit" } + }, + { + "source": { "name": "ProviderName", "type": "String" }, + "sink": { "name": "ProviderName" } + }, + { + "source": { "name": "PublisherName", "type": "String" }, + "sink": { "name": "PublisherName" } + }, + { + "source": { "name": "RegionId", "type": "String" }, + "sink": { "name": "RegionId" } + }, + { + "source": { "name": "RegionName", "type": "String" }, + "sink": { "name": "RegionName" } + }, + { + "source": { "name": "ResourceId", "type": "String" }, + "sink": { "name": "ResourceId" } + }, + { + "source": { "name": "ResourceName", "type": "String" }, + "sink": { "name": "ResourceName" } + }, + { + "source": { "name": "ResourceType", "type": "String" }, + "sink": { "name": "ResourceType" } + }, + { + "source": { "name": "ServiceCategory", "type": "String" }, + "sink": { "name": "ServiceCategory" } + }, + { + "source": { "name": "ServiceName", "type": "String" }, + "sink": { "name": "ServiceName" } + }, + { + "source": { "name": "ServiceSubcategory", "type": "String" }, + "sink": { "name": "ServiceSubcategory" } + }, + { + "source": { "name": "SkuId", "type": "String" }, + "sink": { "name": "SkuId" } + }, + { + "source": { "name": "SkuMeter", "type": "String" }, + "sink": { "name": "SkuMeter" } + }, + { + "source": { "name": "SkuPriceDetails", "type": "String" }, + "sink": { "name": "SkuPriceDetails" } + }, + { + "source": { "name": "SkuPriceId", "type": "String" }, + "sink": { "name": "SkuPriceId" } + }, + { + "source": { "name": "SubAccountId", "type": "String" }, + "sink": { "name": "SubAccountId" } + }, + { + "source": { "name": "SubAccountName", "type": "String" }, + "sink": { "name": "SubAccountName" } + }, + { + "source": { "name": "SubAccountType", "type": "String" }, + "sink": { "name": "SubAccountType" } + }, + { + "source": { "name": "Tags", "type": "String" }, + "sink": { "name": "Tags" } + }, + { + "source": { "name": "x_Operation", "type": "String" }, + "sink": { "name": "x_Operation" } + }, + { + "source": { "name": "x_ServiceCode", "type": "String" }, + "sink": { "name": "x_ServiceCode" } + } + ] + } +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AmazonWebServices/README.md b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AmazonWebServices/README.md new file mode 100644 index 000000000..3cc6524ea --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AmazonWebServices/README.md @@ -0,0 +1,133 @@ +# Microsoft.FinOpsHubs/AmazonWebServices + +Collects FOCUS cost data exported from Amazon Web Services into the FinOps hub so AWS costs are ingested, normalized, and reported alongside Microsoft Cloud costs. + +The app only handles **collection**. Once the files are staged in the export container, the existing FinOps hub ETL converts, normalizes, and ingests them with no changes: the generated manifest points at the `focuscost_1.2-aws.json` schema, which the ingestion pipeline loads the same way it loads a Cost Management schema. + +## What gets deployed + +| Resource | Name | Description | +| -------- | ---- | ----------- | +| Key Vault secret | `aws-secret-access-key` | AWS secret access key. Never stored in the linked service definition. | +| Linked service | `aws_s3` | Amazon S3 connection using access key authentication. | +| Dataset | `aws_focus_manifest_folder` | Binary folder used to discover the export manifest. | +| Dataset | `aws_focus_manifest` | JSON export manifest read in place from Amazon S3. | +| Dataset | `aws_focus_source` | Binary FOCUS data file in Amazon S3. | +| Dataset | `aws_focus_landing` | Binary FOCUS data file staged in the export container. | +| Dataset | `aws_focus_manifest_landing` | Text sink used to write the generated manifest. | +| Pipeline | `aws_CollectFocusExport` | Entry point. Collects the current and previous billing periods. | +| Pipeline | `aws_CollectFocusExportPeriod` | Locates the export manifest for one billing period. | +| Pipeline | `aws_CollectFocusExportManifest` | Copies the files listed in one manifest and publishes the generated manifest. | +| Trigger | `aws_DailySchedule` | Runs once a day. | + +Files are staged in the **msexports** container using the following hierarchy: + +```text +msexports/ +└── aws/ + └── {accountId}/ + └── {YYYY-MM}/ + └── {runId}/ + ├── {export-name}-00001.snappy.parquet + └── manifest.json +``` + +After ingestion, the data lands in `ingestion/Costs/{YYYY}/{MM}/aws/{accountId}/`. + +## Requirements + +| # | Requirement | Notes | +| - | ----------- | ----- | +| 1 | A FOCUS 1.2 export in AWS Data Exports | Create it in **Billing and Cost Management** > **Data Exports**. Export type must be **FOCUS 1.2**. | +| 2 | An S3 bucket that receives the export | Parquet and gzipped CSV are both supported. | +| 3 | An IAM user with read access to the bucket | Needs `s3:GetObject` and `s3:ListBucket` on the bucket and its contents. | +| 4 | An access key ID and secret access key for that user | The secret is stored in the hub Key Vault during deployment. | +| 5 | The AWS account ID | Used to isolate the data in the hub data lake. | + +## Configuration + +Enable the feature with the **Multicloud** step in the deployment wizard, or set the following template parameters: + +| Parameter | Description | +| --------- | ----------- | +| `enableAwsFocusIngestion` | Set to `true` to deploy the app. Default: `false`. | +| `awsBucketName` | Name of the S3 bucket that contains the export. | +| `awsBucketPath` | Path to the export root folder within the bucket. This is the folder that contains the `data` and `metadata` subfolders. Example: `reports/focus-export`. | +| `awsAccountId` | AWS account ID that owns the export. | +| `awsRegion` | Region of the bucket. Leave empty to use the global S3 endpoint. | +| `awsAccessKeyId` | Access key ID used to read the bucket. | +| `awsSecretAccessKey` | Secret access key used to read the bucket. | +| `awsFocusVersion` | FOCUS version of the export. Only `1.2` is supported. | +| `multiCloudScheduleHour` | Hour of the day (UTC) to collect files. Default: `4`. | + +### Finding the bucket path + +For an export delivered to `s3://my-bucket/reports/focus-export/data/billing_period=2026-05/`, set `awsBucketName` to `my-bucket` and `awsBucketPath` to `reports/focus-export`. Do not include leading or trailing slashes, and do not include `data` or `metadata`. + +### Minimum IAM policy + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": "arn:aws:s3:::my-bucket" + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject"], + "Resource": "arn:aws:s3:::my-bucket/reports/focus-export/*" + } + ] +} +``` + +## How collection works + +1. The daily trigger runs `aws_CollectFocusExport`, which iterates the current and previous billing period. AWS may restate a closed period for up to two weeks, so the previous period is always re-collected. +2. For each period, `aws_CollectFocusExportPeriod` lists `{bucketPath}/metadata/billing_period={YYYY-MM}/` and looks for the export manifest. When the period has not been exported yet, the folder does not exist and the pipeline completes without doing anything. +3. `aws_CollectFocusExportManifest` reads the manifest, copies only the files listed in its `dataFiles` array, then writes a generated `manifest.json` in the same staging folder. +4. Writing that manifest fires the existing `msexports_ManifestAdded` trigger, which starts the normal ingestion pipeline. + +Three details of this flow matter and should not be changed casually: + +- **Only the files listed in `dataFiles` are copied.** When an export is configured to create a new file on every refresh, the `data/billing_period={YYYY-MM}/` folder accumulates one subfolder per day. Listing the folder recursively would copy every refresh and multiply the month's costs. +- **The AWS manifest is never copied into the export container.** Its schema is incompatible with the manifest contract the ETL expects. It is read in place and left in Amazon S3. +- **The generated manifest is written last**, and only after every copy has succeeded, so ingestion never starts on a partial set of files. + +## Idempotency + +Every run generates a new `runId`, which the ETL uses as the ingestion ID. Data Explorer replaces all data tagged with a previous ingestion ID for the same destination folder, so re-collecting a period replaces it rather than adding to it. + +This depends on the destination path being stable and lowercase. The account ID is lowercased before it is used, because the Data Explorer `drop-by` tag is case-sensitive: ingesting the same data under `aws/123456789012` and `AWS/123456789012` produces two tags that coexist and silently double the reported cost. + +## Validate the deployment + +1. Open the hub Data Factory and run the `aws_CollectFocusExport` pipeline in debug mode. +2. Confirm each activity succeeds: + - `Find Manifest` reports `exists: true` for the current period. + - `Filter Manifest Files` returns exactly one item. + - `Copy FOCUS Files` reports more than 0 bytes written for each file. + - `Write Manifest` completes. +3. Confirm the staged files exist under `aws/{accountId}/{YYYY-MM}/` in the **msexports** container, and that `manifest.json` is valid JSON. +4. Confirm the `msexports_ExecuteETL` pipeline started on its own within a couple of minutes. +5. Query the ingestion table and confirm rows arrived with `x_SourceProvider` set for AWS. + +## Troubleshooting + +| Symptom | Cause | Resolution | +| ------- | ----- | ---------- | +| `Find Manifest` fails with an access error | The access key is wrong, expired, or the IAM user cannot list the bucket. | Confirm the key in Key Vault and the IAM policy above. | +| `Find Manifest` reports `exists: false` every run | `awsBucketPath` is wrong, or the export has not run yet. | Confirm the path contains the `data` and `metadata` folders. Do not include `data` or `metadata` in the value. | +| `Filter Manifest Files` returns 0 items | The metadata partition exists but holds no manifest. | Confirm the export completed in AWS for that period. | +| `Copy FOCUS Files` fails with a not found error | The object key was built incorrectly. | Confirm `awsBucketName` matches the bucket in the `s3://` URIs in the manifest. A mismatch leaves the prefix in the key. | +| `msexports_ExecuteETL` never starts | The generated manifest is not valid JSON, or it was not written to the export container. | Download `manifest.json` from the staging folder and validate it. | +| Ingestion runs but no rows appear | The generated manifest reported no files. | Confirm `blobCount` is greater than zero and that `dataRowCount` is absent. A `dataRowCount` of `0` makes the ETL treat the export as empty. | +| Costs are doubled | The same period was ingested under two different paths. | Confirm `awsAccountId` is lowercase and has not changed between runs. | +| Ingestion fails on a column mapping | The export schema drifted, or the export is not FOCUS 1.2. | Compare the manifest's `columns` array against `focuscost_1.2-aws.json`. | + +## Cost + +Data Factory activity runs, the data movement itself, and staging storage are added. Cross-cloud egress is billed by AWS, not Azure. For a single account with a few hundred megabytes of FOCUS data a month, expect a few USD per month on the Azure side. diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AmazonWebServices/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AmazonWebServices/app.bicep new file mode 100644 index 000000000..d839a55b8 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AmazonWebServices/app.bicep @@ -0,0 +1,1127 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { finOpsToolkitVersion, HubAppProperties, privateRoutingForLinkedServices, isSupportedVersion } from '../../fx/hub-types.bicep' +import { AppMetadata as CoreMetadata } from '../Core/metadata.bicep' +import { AppMetadata as ExportsMetadata } from '../../Microsoft.CostManagement/Exports/metadata.bicep' +import { AppMetadata as AwsMetadata } from './metadata.bicep' + +metadata hubApp = { + id: 'Microsoft.FinOpsHubs.AmazonWebServices' + version: '$$ftkver$$' + dependencies: [ + 'Microsoft.FinOpsHubs.Core' + 'Microsoft.CostManagement.Exports' + ] + metadata: 'https://microsoft.github.io/finops-toolkit/deploy/finops-hub/$$ftkver$$/Microsoft.FinOpsHubs/AmazonWebServices/metadata.bicep' +} + + +//============================================================================== +// Parameters +//============================================================================== + +@description('Required. FinOps hub app getting deployed.') +param app HubAppProperties + +@description('Required. Metadata describing shared resources from the Core app. Must be v13 or higher.') +@validate(x => isSupportedVersion(x.version, '13.0', ''), 'AWS FOCUS ingestion requires FinOps hubs version 13.0 or higher.') +param core CoreMetadata + +@description('Required. Metadata describing shared resources from the Cost Management Exports app. Owns the export container the collected files are staged in.') +param exports ExportsMetadata + +@description('Required. Name of the Amazon S3 bucket that contains the FOCUS export.') +@minLength(3) +@maxLength(63) +param bucketName string + +@description('Required. Path to the export root folder within the bucket. This is the folder that contains the "data" and "metadata" subfolders, without leading or trailing slashes. Example: "reports/focus-export".') +param bucketPath string + +@description('Required. Amazon Web Services account ID that owns the export. Used to isolate the data in the hub data lake. Must be lowercase to avoid duplicate ingestion.') +param accountId string + +@description('Optional. Amazon Web Services region of the bucket. Used to build the S3 service URL. Leave empty to use the global endpoint. Default: "" (global).') +param region string = '' + +@description('Required. Amazon Web Services access key ID used to read the bucket.') +param accessKeyId string + +@description('Required. Amazon Web Services secret access key used to read the bucket. Stored in Key Vault.') +@secure() +param secretAccessKey string + +@description('Optional. FOCUS version of the export. Only 1.2 is supported today because it is the only version with a validated AWS schema file. Default: "1.2".') +@allowed([ + '1.2' +]) +param focusVersion string = '1.2' + +@description('Optional. Hour of the day (UTC) to collect FOCUS files. Default: 4.') +@minValue(0) +@maxValue(23) +param scheduleHour int = 4 + + +//============================================================================== +// Variables +//============================================================================== + +var AWS = 'aws' + +// Name of the Key Vault secret that holds the AWS secret access key. +var secretAccessKeyName = '${AWS}-secret-access-key' + +// Amazon Web Services may restate a closed billing period for up to two weeks, so every run +// collects the current and the previous period. Offsets are in months, relative to today. +var collectionOffsets = [0, -1] + +// Lowercase account ID. The ETL lowercases the scope before building the destination path, and the +// Data Explorer drop-by tag is case-sensitive, so a mixed-case value silently duplicates the data. +var accountFolder = toLower(accountId) + +// Value written to exportConfig.resourceId in the generated manifest. The ETL splits this into the +// scope segment of the destination path: Costs///aws/. +var exportResourceId = '/${AWS}/${accountFolder}' + +// Schema file the ETL loads is derived from exportConfig.type and exportConfig.dataVersion, so this +// value must match a published schema file: focuscost_.json. +var exportDataVersion = '${focusVersion}-${AWS}' + + +//============================================================================== +// Resources +//============================================================================== + +// Register app +module appRegistration '../../fx/hub-app.bicep' = { + name: 'Microsoft.FinOpsHubs.AmazonWebServices_Register' + params: { + app: app + version: finOpsToolkitVersion + features: [ + 'DataFactory' + 'KeyVault' + 'Storage' + ] + } +} + +// Store the AWS secret access key so it is never exposed in the linked service definition. +module keyVault_secret '../../fx/hub-vault.bicep' = { + name: 'Microsoft.FinOpsHubs.AmazonWebServices_Vault.SecretAccessKey' + dependsOn: [appRegistration] // Wait for the Key Vault to be created + params: { + vaultName: app.keyVault + secretName: secretAccessKeyName + secretValue: secretAccessKey + } +} + +// Get data factory instance +resource dataFactory 'Microsoft.DataFactory/factories@2018-06-01' existing = { + name: app.dataFactory + dependsOn: [appRegistration] + + resource dataset_config 'datasets@2018-06-01' existing = { + name: core.datasets.config + } +} + +//------------------------------------------------------------------------------ +// Linked services +//------------------------------------------------------------------------------ + +// Amazon S3 connection. The secret access key is resolved from Key Vault at runtime, matching the +// pattern used by the RemoteHub app. +resource linkedService_amazonS3 'Microsoft.DataFactory/factories/linkedservices@2018-06-01' = { + name: '${AWS}_s3' + parent: dataFactory + dependsOn: [keyVault_secret] + properties: { + annotations: [] + parameters: {} + type: 'AmazonS3' + typeProperties: union( + { + authenticationType: 'AccessKey' + accessKeyId: accessKeyId + secretAccessKey: { + type: 'AzureKeyVaultSecret' + store: { + referenceName: app.keyVault + type: 'LinkedServiceReference' + } + secretName: secretAccessKeyName + } + }, + empty(region) ? {} : { serviceUrl: 'https://s3.${region}.amazonaws.com' } + ) + // Required for the linked service to use the managed virtual network when private routing is enabled. + ...privateRoutingForLinkedServices(app.hub) + } +} + +//------------------------------------------------------------------------------ +// Datasets +//------------------------------------------------------------------------------ + +// Folder in Amazon S3 used to discover the export manifest. Only used by Get Metadata. +resource dataset_focusManifestFolder 'Microsoft.DataFactory/factories/datasets@2018-06-01' = { + name: '${AWS}_focus_manifest_folder' + parent: dataFactory + properties: { + annotations: [] + type: 'Binary' + parameters: { + folderPath: { + type: 'String' + } + } + linkedServiceName: { + parameters: {} + referenceName: linkedService_amazonS3.name + type: 'LinkedServiceReference' + } + typeProperties: { + location: { + type: 'AmazonS3Location' + bucketName: bucketName + folderPath: { + value: '@dataset().folderPath' + type: 'Expression' + } + } + } + } +} + +// Export manifest published by Amazon Web Services. This file is read in place and is never copied +// into the export container: its schema is incompatible with the manifest contract the ETL expects. +resource dataset_focusManifest 'Microsoft.DataFactory/factories/datasets@2018-06-01' = { + name: '${AWS}_focus_manifest' + parent: dataFactory + properties: { + annotations: [] + type: 'Json' + parameters: { + folderPath: { + type: 'String' + } + fileName: { + type: 'String' + } + } + linkedServiceName: { + parameters: {} + referenceName: linkedService_amazonS3.name + type: 'LinkedServiceReference' + } + typeProperties: { + location: { + type: 'AmazonS3Location' + bucketName: bucketName + folderPath: { + value: '@dataset().folderPath' + type: 'Expression' + } + fileName: { + value: '@dataset().fileName' + type: 'Expression' + } + } + } + } +} + +// FOCUS data file in Amazon S3. Copied as-is so the file keeps its original format and compression. +resource dataset_focusSource 'Microsoft.DataFactory/factories/datasets@2018-06-01' = { + name: '${AWS}_focus_source' + parent: dataFactory + properties: { + annotations: [] + type: 'Binary' + parameters: { + folderPath: { + type: 'String' + } + fileName: { + type: 'String' + } + } + linkedServiceName: { + parameters: {} + referenceName: linkedService_amazonS3.name + type: 'LinkedServiceReference' + } + typeProperties: { + location: { + type: 'AmazonS3Location' + bucketName: bucketName + folderPath: { + value: '@dataset().folderPath' + type: 'Expression' + } + fileName: { + value: '@dataset().fileName' + type: 'Expression' + } + } + } + } +} + +// FOCUS data file staged in the export container, where the existing ETL picks it up. +resource dataset_focusLanding 'Microsoft.DataFactory/factories/datasets@2018-06-01' = { + name: '${AWS}_focus_landing' + parent: dataFactory + properties: { + annotations: [] + type: 'Binary' + parameters: { + folderPath: { + type: 'String' + } + fileName: { + type: 'String' + } + } + linkedServiceName: { + parameters: {} + referenceName: app.storage + type: 'LinkedServiceReference' + } + typeProperties: { + location: { + type: 'AzureBlobFSLocation' + fileSystem: exports.containers.msexports + folderPath: { + value: '@dataset().folderPath' + type: 'Expression' + } + fileName: { + value: '@dataset().fileName' + type: 'Expression' + } + } + } + } +} + +// Writes the generated manifest as raw text. Data Factory cannot build an arbitrary nested JSON +// document with a JSON sink, and a Web activity cannot reach the storage account when private +// routing is enabled, so the manifest is assembled as a string and written verbatim through a +// single-column text sink. Quoting is disabled so the file contains exactly the JSON that was built. +resource dataset_focusManifestLanding 'Microsoft.DataFactory/factories/datasets@2018-06-01' = { + name: '${AWS}_focus_manifest_landing' + parent: dataFactory + properties: { + annotations: [] + type: 'DelimitedText' + parameters: { + folderPath: { + type: 'String' + } + fileName: { + type: 'String' + } + } + linkedServiceName: { + parameters: {} + referenceName: app.storage + type: 'LinkedServiceReference' + } + typeProperties: { + location: { + type: 'AzureBlobFSLocation' + fileSystem: exports.containers.msexports + folderPath: { + value: '@dataset().folderPath' + type: 'Expression' + } + fileName: { + value: '@dataset().fileName' + type: 'Expression' + } + } + // A single column is written, so the delimiter is never emitted. It is set to a character that + // does not occur in the generated manifest so the value is never quoted. + columnDelimiter: '~' + quoteChar: '' + escapeChar: '' + firstRowAsHeader: false + encodingName: 'UTF-8' + } + schema: [ + { + name: 'manifest' + type: 'String' + } + ] + } +} + +//------------------------------------------------------------------------------ +// Pipelines +//------------------------------------------------------------------------------ + +// Stages the files listed in a single export manifest and publishes the generated manifest that +// starts the existing ETL. Split into its own pipeline because Data Factory does not allow a +// container activity (ForEach) inside another container activity, and because a child pipeline +// gives each manifest its own variable scope. +resource pipeline_CollectFocusExportManifest 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: '${AWS}_CollectFocusExportManifest' + parent: dataFactory + properties: { + description: 'Copies the FOCUS files listed in an Amazon Web Services export manifest into the export container and writes the manifest that starts the ingestion pipeline.' + parameters: { + billingPeriod: { + type: 'String' + } + manifestFolder: { + type: 'String' + } + manifestFile: { + type: 'String' + } + } + variables: { + runId: { + type: 'String' + } + dataFiles: { + type: 'Array' + } + blobs: { + type: 'Array' + } + destinationFolder: { + type: 'String' + } + manifestJson: { + type: 'String' + } + } + activities: [ + { // Set Run Id + name: 'Set Run Id' + description: 'Generate the ingestion ID for this run. Data Explorer replaces all data tagged with a previous ingestion ID, which is what makes a daily refresh idempotent.' + type: 'SetVariable' + dependsOn: [] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'runId' + value: { + value: '@guid()' + type: 'Expression' + } + } + } + { // Load Settings + name: 'Load Settings' + description: 'Read hub settings to determine how long staged export files are retained.' + type: 'Lookup' + dependsOn: [] + policy: { + timeout: '0.00:05:00' + retry: 2 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + source: { + type: 'JsonSource' + storeSettings: { + type: 'AzureBlobFSReadSettings' + recursive: false + enablePartitionDiscovery: false + } + formatSettings: { + type: 'JsonReadSettings' + } + } + dataset: { + referenceName: dataFactory::dataset_config.name + type: 'DatasetReference' + parameters: { + fileName: core.settings.file + folderPath: core.settings.container + } + } + firstRowOnly: true + } + } + { // Read Source Manifest + name: 'Read Source Manifest' + description: 'Read the Amazon Web Services export manifest. Only the files it lists are copied: the source folder accumulates one subfolder per refresh, so copying everything would duplicate the period.' + type: 'Lookup' + dependsOn: [] + policy: { + timeout: '0.00:30:00' + retry: 2 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + source: { + type: 'JsonSource' + storeSettings: { + type: 'AmazonS3ReadSettings' + recursive: false + enablePartitionDiscovery: false + } + formatSettings: { + type: 'JsonReadSettings' + } + } + dataset: { + referenceName: dataset_focusManifest.name + type: 'DatasetReference' + parameters: { + folderPath: { + value: '@pipeline().parameters.manifestFolder' + type: 'Expression' + } + fileName: { + value: '@pipeline().parameters.manifestFile' + type: 'Expression' + } + } + } + firstRowOnly: true + } + } + { // Set Data Files + name: 'Set Data Files' + description: 'Save the list of files to copy. The manifest field is dataFiles and each item is a full s3:// URI.' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Read Source Manifest' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'dataFiles' + value: { + value: '@coalesce(activity(\'Read Source Manifest\').output.firstRow.dataFiles, json(\'[]\'))' + type: 'Expression' + } + } + } + { // Set Destination Folder + name: 'Set Destination Folder' + description: 'Build the staging folder for this run. The run ID keeps repeated or concurrent runs of the same period from overwriting each other mid-copy.' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Set Run Id' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'destinationFolder' + value: { + value: '@toLower(concat(\'${AWS}/${accountFolder}/\', pipeline().parameters.billingPeriod, \'/\', variables(\'runId\')))' + type: 'Expression' + } + } + } + { // Copy FOCUS Files + name: 'Copy FOCUS Files' + description: 'Copy each file listed in the manifest into the export container without changing its format.' + type: 'ForEach' + dependsOn: [ + { + activity: 'Set Data Files' + dependencyConditions: ['Succeeded'] + } + { + activity: 'Set Destination Folder' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@variables(\'dataFiles\')' + type: 'Expression' + } + isSequential: false + batchCount: 4 + activities: [ + { + name: 'Copy FOCUS File' + type: 'Copy' + dependsOn: [] + policy: { + timeout: '0.12:00:00' + retry: 2 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + source: { + type: 'BinarySource' + storeSettings: { + type: 'AmazonS3ReadSettings' + recursive: false + } + formatSettings: { + type: 'BinaryReadSettings' + } + } + sink: { + type: 'BinarySink' + storeSettings: { + type: 'AzureBlobFSWriteSettings' + } + } + enableStaging: false + } + inputs: [ + { + referenceName: dataset_focusSource.name + type: 'DatasetReference' + parameters: { + // Strip the s3:/// prefix to get the object key, then split it into the + // folder and file name the dataset expects. + folderPath: { + value: '@join(take(split(replace(item(), \'s3://${bucketName}/\', \'\'), \'/\'), sub(length(split(replace(item(), \'s3://${bucketName}/\', \'\'), \'/\')), 1)), \'/\')' + type: 'Expression' + } + fileName: { + value: '@last(split(item(), \'/\'))' + type: 'Expression' + } + } + } + ] + outputs: [ + { + referenceName: dataset_focusLanding.name + type: 'DatasetReference' + parameters: { + folderPath: { + value: '@variables(\'destinationFolder\')' + type: 'Expression' + } + fileName: { + value: '@last(split(item(), \'/\'))' + type: 'Expression' + } + } + } + ] + } + ] + } + } + { // Build Blob List + name: 'Build Blob List' + description: 'Build the blobs array for the generated manifest. Sequential because AppendVariable is not safe inside a parallel loop.' + type: 'ForEach' + dependsOn: [ + { + activity: 'Copy FOCUS Files' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@variables(\'dataFiles\')' + type: 'Expression' + } + isSequential: true + activities: [ + { + name: 'Append Blob' + type: 'AppendVariable' + dependsOn: [] + userProperties: [] + typeProperties: { + variableName: 'blobs' + value: { + // blobName is the path within the export container, which is how the ETL passes it + // to the parquet dataset. + value: '@json(concat(\'{"blobName":"\', variables(\'destinationFolder\'), \'/\', last(split(item(), \'/\')), \'"}\'))' + type: 'Expression' + } + } + } + ] + } + } + { // Build Manifest + name: 'Build Manifest' + description: 'Assemble the manifest the ingestion pipeline expects. dataRowCount is intentionally omitted: the source manifest carries no row count, and writing zero would make the ETL treat the export as empty.' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Build Blob List' + dependencyConditions: ['Succeeded'] + } + { + activity: 'Load Settings' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'manifestJson' + value: { + value: '@concat(\'{"exportConfig":{"type":"FocusCost","dataVersion":"${exportDataVersion}","exportName":"${AWS}-focus","resourceId":"${exportResourceId}"},"runInfo":{"runId":"\', variables(\'runId\'), \'","startDate":"\', pipeline().parameters.billingPeriod, \'-01T00:00:00Z"},"blobCount":\', string(length(variables(\'blobs\'))), \',"blobs":\', string(variables(\'blobs\')), \',"retention":{"msexports":{"days":\', string(coalesce(activity(\'Load Settings\').output.firstRow.retention.msexports.days, 0)), \'}}}\')' + type: 'Expression' + } + } + } + { // Write Manifest + name: 'Write Manifest' + description: 'Publish the manifest, which starts the ingestion pipeline. Depends on a successful copy so a manifest is never published over a partial set of files.' + type: 'Copy' + dependsOn: [ + { + activity: 'Build Manifest' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + timeout: '0.00:30:00' + retry: 2 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + source: { + // The settings file is only used to produce a single row. Its columns are dropped by the + // translator below and replaced with the generated manifest. + type: 'JsonSource' + storeSettings: { + type: 'AzureBlobFSReadSettings' + recursive: false + enablePartitionDiscovery: false + } + formatSettings: { + type: 'JsonReadSettings' + } + additionalColumns: [ + { + name: 'manifest' + value: { + value: '@variables(\'manifestJson\')' + type: 'Expression' + } + } + ] + } + sink: { + type: 'DelimitedTextSink' + storeSettings: { + type: 'AzureBlobFSWriteSettings' + } + formatSettings: { + type: 'DelimitedTextWriteSettings' + // quoteAllText must not be set: Data Factory rejects false with + // DelimitedTextInvalidSettings. Quoting is already disabled by the empty quoteChar + // on the dataset. + fileExtension: '.json' + } + } + enableStaging: false + translator: { + type: 'TabularTranslator' + mappings: [ + { + source: { + name: 'manifest' + type: 'String' + } + sink: { + name: 'manifest' + type: 'String' + } + } + ] + } + } + inputs: [ + { + referenceName: dataFactory::dataset_config.name + type: 'DatasetReference' + parameters: { + fileName: core.settings.file + folderPath: core.settings.container + } + } + ] + outputs: [ + { + referenceName: dataset_focusManifestLanding.name + type: 'DatasetReference' + parameters: { + folderPath: { + value: '@variables(\'destinationFolder\')' + type: 'Expression' + } + fileName: 'manifest.json' + } + } + ] + } + ] + policy: { + elapsedTimeMetric: {} + } + annotations: [] + } +} + +// Finds the export manifest for a single billing period. The manifest is published by Amazon Web +// Services only after every data file has landed, so it doubles as the completeness signal. +resource pipeline_CollectFocusExportPeriod 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: '${AWS}_CollectFocusExportPeriod' + parent: dataFactory + properties: { + description: 'Locates the Amazon Web Services export manifest for one billing period and stages the files it lists.' + parameters: { + periodOffsetMonths: { + type: 'Int' + defaultValue: 0 + } + } + variables: { + billingPeriod: { + type: 'String' + } + manifestFolder: { + type: 'String' + } + } + activities: [ + { // Set Billing Period + name: 'Set Billing Period' + description: 'Resolve the billing period to collect, formatted the way the source path partitions it.' + type: 'SetVariable' + dependsOn: [] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'billingPeriod' + value: { + value: '@formatDateTime(addToTime(utcNow(), pipeline().parameters.periodOffsetMonths, \'Month\'), \'yyyy-MM\')' + type: 'Expression' + } + } + } + { // Set Manifest Folder + name: 'Set Manifest Folder' + description: 'Build the path to the metadata partition for the billing period. The partition key is lowercase in the delivered export.' + type: 'SetVariable' + dependsOn: [ + { + activity: 'Set Billing Period' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + variableName: 'manifestFolder' + value: { + value: '@concat(\'${bucketPath}/metadata/billing_period=\', variables(\'billingPeriod\'))' + type: 'Expression' + } + } + } + { // Find Manifest + name: 'Find Manifest' + description: 'List the metadata partition. Requesting the exists field keeps the activity from failing when the period has not been exported yet.' + type: 'GetMetadata' + dependsOn: [ + { + activity: 'Set Manifest Folder' + dependencyConditions: ['Succeeded'] + } + ] + policy: { + timeout: '0.00:30:00' + retry: 2 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + dataset: { + referenceName: dataset_focusManifestFolder.name + type: 'DatasetReference' + parameters: { + folderPath: { + value: '@variables(\'manifestFolder\')' + type: 'Expression' + } + } + } + fieldList: [ + 'exists' + 'childItems' + ] + storeSettings: { + type: 'AmazonS3ReadSettings' + recursive: false + enablePartitionDiscovery: false + } + formatSettings: { + type: 'BinaryReadSettings' + } + } + } + { // Filter Manifest Files + name: 'Filter Manifest Files' + description: 'Keep only the export manifest. The partition may also contain other metadata files.' + type: 'Filter' + dependsOn: [ + { + activity: 'Find Manifest' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@if(activity(\'Find Manifest\').output.exists, activity(\'Find Manifest\').output.childItems, json(\'[]\'))' + type: 'Expression' + } + condition: { + value: '@and(equals(item().type, \'File\'), endswith(toLower(item().name), \'manifest.json\'))' + type: 'Expression' + } + } + } + { // Collect Manifest + name: 'Collect Manifest' + description: 'Stage the files listed in the manifest. The loop body does not run when the period has not been exported yet.' + type: 'ForEach' + dependsOn: [ + { + activity: 'Filter Manifest Files' + dependencyConditions: ['Succeeded'] + } + ] + userProperties: [] + typeProperties: { + items: { + value: '@activity(\'Filter Manifest Files\').output.Value' + type: 'Expression' + } + isSequential: true + activities: [ + { + name: 'Collect Manifest Files' + type: 'ExecutePipeline' + dependsOn: [] + policy: { + secureInput: false + } + userProperties: [] + typeProperties: { + pipeline: { + referenceName: pipeline_CollectFocusExportManifest.name + type: 'PipelineReference' + } + waitOnCompletion: true + parameters: { + billingPeriod: { + value: '@variables(\'billingPeriod\')' + type: 'Expression' + } + manifestFolder: { + value: '@variables(\'manifestFolder\')' + type: 'Expression' + } + manifestFile: { + value: '@item().name' + type: 'Expression' + } + } + } + } + ] + } + } + ] + policy: { + elapsedTimeMetric: {} + } + annotations: [] + } +} + +// Entry point. Amazon Web Services may restate a closed period for up to two weeks, so each run +// collects the current and the previous billing period. +resource pipeline_CollectFocusExport 'Microsoft.DataFactory/factories/pipelines@2018-06-01' = { + name: '${AWS}_CollectFocusExport' + parent: dataFactory + properties: { + description: 'Collects FOCUS cost data exported from Amazon Web Services for the current and previous billing periods.' + parameters: { + periodOffsetMonths: { + type: 'Array' + defaultValue: collectionOffsets + } + } + activities: [ + { // Collect Billing Periods + name: 'Collect Billing Periods' + description: 'Collect each billing period in turn. Sequential to keep the number of concurrent requests to Amazon S3 low.' + type: 'ForEach' + dependsOn: [] + userProperties: [] + typeProperties: { + items: { + value: '@pipeline().parameters.periodOffsetMonths' + type: 'Expression' + } + isSequential: true + activities: [ + { + name: 'Collect Billing Period' + type: 'ExecutePipeline' + dependsOn: [] + policy: { + secureInput: false + } + userProperties: [] + typeProperties: { + pipeline: { + referenceName: pipeline_CollectFocusExportPeriod.name + type: 'PipelineReference' + } + waitOnCompletion: true + parameters: { + periodOffsetMonths: { + value: '@item()' + type: 'Expression' + } + } + } + } + ] + } + } + ] + policy: { + elapsedTimeMetric: {} + } + annotations: [] + } +} + +//------------------------------------------------------------------------------ +// Scheduling +//------------------------------------------------------------------------------ + +resource trigger_DailySchedule 'Microsoft.DataFactory/factories/triggers@2018-06-01' = { + name: '${AWS}_DailySchedule' + parent: dataFactory + properties: { + description: 'Collects FOCUS cost data exported from Amazon Web Services once a day.' + pipelines: [ + { + pipelineReference: { + referenceName: pipeline_CollectFocusExport.name + type: 'PipelineReference' + } + parameters: { + periodOffsetMonths: collectionOffsets + } + } + ] + type: 'ScheduleTrigger' + typeProperties: { + recurrence: { + frequency: 'Day' + interval: 1 + // The zone designator is required when timeZone is UTC. Without it, the trigger deploys but + // fails to start with InvalidWorkflowTriggerRecurrence. + startTime: '2023-01-01T00:00:00Z' + timeZone: 'UTC' + schedule: { + hours: [ + scheduleHour + ] + minutes: [ + 0 + ] + } + } + } + } +} + + +//============================================================================== +// Outputs +//============================================================================== + +@description('The app properties for the Amazon Web Services app.') +output app HubAppProperties = app + +@description('Metadata describing resources created by the Amazon Web Services app.') +output metadata AwsMetadata = { + id: 'Microsoft.FinOpsHubs.AmazonWebServices' + version: finOpsToolkitVersion + storage: { + container: exports.containers.msexports + folder: '${AWS}/${accountFolder}' + } + datasets: { + focusManifestFolder: dataset_focusManifestFolder.name + focusManifest: dataset_focusManifest.name + focusSource: dataset_focusSource.name + focusLanding: dataset_focusLanding.name + focusManifestLanding: dataset_focusManifestLanding.name + } + linkedServices: { + amazonS3: linkedService_amazonS3.name + } + pipelines: { + collectFocusExport: pipeline_CollectFocusExport.name + collectFocusExportPeriod: pipeline_CollectFocusExportPeriod.name + collectFocusExportManifest: pipeline_CollectFocusExportManifest.name + } +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AmazonWebServices/metadata.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AmazonWebServices/metadata.bicep new file mode 100644 index 000000000..581a56560 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/AmazonWebServices/metadata.bicep @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//============================================================================== +// App metadata definition +//============================================================================== + +@export() +@description('Metadata for resources created by the Amazon Web Services app.') +type AppMetadata = { + @description('Fully-qualified app identifier.') + id: string + @description('App version.') + version: string + @description('Storage container and folder where collected FOCUS files are staged.') + storage: { + @description('Container where collected FOCUS files are staged for the ETL pipeline.') + container: string + @description('Root folder within the container where collected FOCUS files are staged.') + folder: string + } + @description('Data Factory dataset names.') + datasets: { + @description('Binary dataset used to list the export manifest folder in Amazon S3.') + focusManifestFolder: string + @description('JSON dataset for the Amazon Web Services export manifest read from Amazon S3.') + focusManifest: string + @description('Binary dataset for FOCUS files read from Amazon S3.') + focusSource: string + @description('Binary dataset for FOCUS files staged in the export container.') + focusLanding: string + @description('Text dataset used to write the generated export manifest to the export container.') + focusManifestLanding: string + } + @description('Data Factory linked service names.') + linkedServices: { + @description('Amazon S3 linked service used to read the FOCUS export.') + amazonS3: string + } + @description('Data Factory pipeline names.') + pipelines: { + @description('Pipeline that collects FOCUS files for all configured billing periods.') + collectFocusExport: string + @description('Pipeline that resolves the export manifest for a single billing period.') + collectFocusExportPeriod: string + @description('Pipeline that stages the files listed in a single export manifest.') + collectFocusExportManifest: string + } +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 index ffea9d454..1b8d5fc61 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 @@ -69,6 +69,9 @@ if (!$json) version = '' learnMore = 'https://aka.ms/finops/hubs' scopes = @() + invoices = @{ + billingAccounts = @() + } retention = @{ 'msexports' = @{ days = 0 @@ -154,6 +157,24 @@ else $json.retention.final.months = [Int32]::Parse($env:finalRetentionInMonths) } +# Set or update the billing accounts to download invoices for +$invoiceBillingAccounts = @() +if ($env:invoiceBillingAccounts) +{ + $invoiceBillingAccounts = @($env:invoiceBillingAccounts.Split('|') | ForEach-Object { $_.Trim() } | Where-Object { $_ } | Select-Object -Unique) +} + +if (!($json.invoices)) +{ + $json | Add-Member -Name invoices -Value (ConvertFrom-Json '{ "billingAccounts": [] }') -MemberType NoteProperty +} +elseif ($null -eq $json.invoices.billingAccounts) +{ + $json.invoices | Add-Member -Name billingAccounts -Value @() -MemberType NoteProperty -Force +} + +$json.invoices.billingAccounts = $invoiceBillingAccounts + # Updating settings Write-Output "Updating version to $env:ftkVersion..." $json.version = $env:ftkVersion diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/app.bicep index 9e1a694c0..46dffee3b 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/app.bicep @@ -36,6 +36,9 @@ param rawRetentionInDays int = 0 @description('Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. Default: 13.') param finalRetentionInMonths int = 13 +@description('Optional. List of billing account IDs to download invoices for. Only used when the Invoices app is enabled. Leave empty to use the billing account scopes monitored by this hub. Default: [] (none).') +param invoiceBillingAccounts string[] = [] + //============================================================================== // Variables @@ -140,6 +143,10 @@ module uploadSettings '../../fx/hub-deploymentScript.bicep' = { name: 'finalRetentionInMonths' value: string(finalRetentionInMonths) } + { + name: 'invoiceBillingAccounts' + value: join(invoiceBillingAccounts, '|') + } { name: 'storageAccountName' value: app.storage diff --git a/src/templates/finops-hub/modules/hub.bicep b/src/templates/finops-hub/modules/hub.bicep index 0b18ee35e..0758ecd5a 100644 --- a/src/templates/finops-hub/modules/hub.bicep +++ b/src/templates/finops-hub/modules/hub.bicep @@ -56,6 +56,50 @@ param enableAHBRecommendations bool = false @description('Optional. Enable non-Spot AKS cluster recommendations that flag AKS clusters with autoscaling but not using Spot VMs. May generate noise since Spot VMs are only appropriate for interruptible workloads. Requires enableRecommendations. Default: false.') param enableSpotRecommendations bool = false +@description('Optional. Enable automatic download of Microsoft invoice files into the hub data lake. Only supported for Microsoft Customer Agreement (MCA) and Microsoft Partner Agreement (MPA) billing accounts. The Data Factory managed identity requires Billing Reader role on the billing account. Default: false.') +param enableInvoiceDownload bool = false + +@description('Optional. List of billing account IDs to download invoices for. Requires enableInvoiceDownload. Leave empty to use the billing account scopes monitored by this hub. Default: [] (none).') +param invoiceBillingAccounts string[] = [] + +@description('Optional. Day of the month to download invoices from the previous month. Requires enableInvoiceDownload. Default: 10.') +@minValue(1) +@maxValue(28) +param invoiceScheduleDay int = 10 + +@description('Optional. Enable ingestion of FOCUS cost data exported from Amazon Web Services. Requires an S3 bucket with a FOCUS 1.2 export and an access key provided during deployment. Default: false.') +param enableAwsFocusIngestion bool = false + +@description('Optional. Name of the Amazon S3 bucket that contains the FOCUS export. Requires enableAwsFocusIngestion.') +param awsBucketName string = '' + +@description('Optional. Path to the export root folder within the S3 bucket. This is the folder that contains the "data" and "metadata" subfolders. Requires enableAwsFocusIngestion.') +param awsBucketPath string = '' + +@description('Optional. Amazon Web Services account ID that owns the FOCUS export. Requires enableAwsFocusIngestion.') +param awsAccountId string = '' + +@description('Optional. Amazon Web Services region of the S3 bucket. Leave empty to use the global S3 endpoint. Requires enableAwsFocusIngestion. Default: "" (global).') +param awsRegion string = '' + +@description('Optional. Amazon Web Services access key ID used to read the S3 bucket. Requires enableAwsFocusIngestion.') +param awsAccessKeyId string = '' + +@description('Optional. Amazon Web Services secret access key used to read the S3 bucket. Stored in Key Vault. Requires enableAwsFocusIngestion.') +@secure() +param awsSecretAccessKey string = '' + +@description('Optional. FOCUS version of the Amazon Web Services export. Requires enableAwsFocusIngestion. Default: "1.2".') +@allowed([ + '1.2' +]) +param awsFocusVersion string = '1.2' + +@description('Optional. Hour of the day (UTC) to collect multicloud FOCUS files. Default: 4.') +@minValue(0) +@maxValue(23) +param multiCloudScheduleHour int = 4 + // cSpell:ignore eventhouse @description('Optional. Microsoft Fabric eventhouse query URI. Default: "" (do not use).') param fabricQueryUri string = '' @@ -223,6 +267,8 @@ var telemetryString = join([ !useAzureDataExplorer || dataExplorerCapacity == 1 ? '' : 'x${dataExplorerCapacity}' // P = private endpoints enabled enablePublicAccess ? '' : 'P' + // A = AWS FOCUS ingestion enabled + enableAwsFocusIngestion ? 'A' : '' ], '') @@ -266,6 +312,7 @@ module core 'Microsoft.FinOpsHubs/Core/app.bicep' = { ingestionRetentionInMonths: ingestionRetentionInMonths rawRetentionInDays: dataExplorerRawRetentionInDays finalRetentionInMonths: dataExplorerFinalRetentionInMonths + invoiceBillingAccounts: enableInvoiceDownload ? invoiceBillingAccounts : [] } } @@ -352,6 +399,40 @@ module recommendations 'Microsoft.FinOpsHubs/Recommendations/app.bicep' = if (en } } +//------------------------------------------------------------------------------ +// Invoices +//------------------------------------------------------------------------------ + +module invoices 'Microsoft.Billing/Invoices/app.bicep' = if (enableInvoiceDownload) { + name: 'Microsoft.Billing.Invoices' + params: { + app: newApp(hub, 'Microsoft.Billing', 'Invoices') + core: core.outputs.metadata + scheduleDay: invoiceScheduleDay + } +} + +//------------------------------------------------------------------------------ +// Multicloud FOCUS ingestion +//------------------------------------------------------------------------------ + +module awsFocus 'Microsoft.FinOpsHubs/AmazonWebServices/app.bicep' = if (enableAwsFocusIngestion) { + name: 'Microsoft.FinOpsHubs.AmazonWebServices' + params: { + app: newApp(hub, 'Microsoft.FinOpsHubs', 'AmazonWebServices') + core: core.outputs.metadata + exports: cmExports.outputs.metadata + bucketName: awsBucketName + bucketPath: awsBucketPath + accountId: awsAccountId + region: awsRegion + accessKeyId: awsAccessKeyId + secretAccessKey: awsSecretAccessKey + focusVersion: awsFocusVersion + scheduleHour: multiCloudScheduleHour + } +} + //------------------------------------------------------------------------------ // Remote hub app //------------------------------------------------------------------------------ @@ -403,6 +484,8 @@ module startTriggers 'fx/hub-initialize.bicep' = { deleteOldResources remoteHub cmManagedExports + invoices + awsFocus ] params: { app: core.outputs.app