From 852f2868410133fa6448941c8c6913d10a860094 Mon Sep 17 00:00:00 2001 From: Sebastian Graef Date: Thu, 17 Jul 2025 11:18:00 +1000 Subject: [PATCH] Add scripts for processing Azure resource groups and analyzing Bicep/Terraform modules --- Bash/avm_manual_analysis.sh | 134 +++++++++++++++ PowerShell/Snippets/AVM-ModuleTester.ps1 | 48 ++++-- _in progress/clone-azure-avm-repos.ps1 | 133 +++++++++++++++ _in progress/process-aiml-resource-groups.ps1 | 160 ++++++++++++++++++ 4 files changed, 461 insertions(+), 14 deletions(-) create mode 100755 Bash/avm_manual_analysis.sh create mode 100755 _in progress/clone-azure-avm-repos.ps1 create mode 100755 _in progress/process-aiml-resource-groups.ps1 diff --git a/Bash/avm_manual_analysis.sh b/Bash/avm_manual_analysis.sh new file mode 100755 index 0000000..d265f8e --- /dev/null +++ b/Bash/avm_manual_analysis.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +echo "=========================================================" +echo "AVM Pattern Module Tests & Examples Usage Analysis" +echo "Analysis of how pattern module TESTS and EXAMPLES use resources" +echo "=========================================================" +echo + +echo "1. BICEP PATTERN MODULE TESTS ANALYSIS" +echo "======================================" + +# Count actual Bicep pattern modules +bicep_modules=$(ls -d /Azure/bicep-registry-modules/avm/ptn/*/* 2>/dev/null | wc -l | tr -d ' ') +echo "Total Bicep pattern modules: $bicep_modules" + +# Count Bicep files +bicep_total=$(find /Azure/bicep-registry-modules/avm/ptn -path "*/tests/*" -name "*.bicep" 2>/dev/null | wc -l | tr -d ' ') +echo "Total Bicep TEST files analyzed: $bicep_total" + +# Count files with native resources +bicep_native_files=$(find /Azure/bicep-registry-modules/avm/ptn -path "*/tests/*" -name "*.bicep" -exec grep -l "resource.*'Microsoft\." {} \; 2>/dev/null | wc -l | tr -d ' ') +echo "Test files with native Microsoft resources: $bicep_native_files" + +# Count files with AVM modules +bicep_avm_files=$(find /Azure/bicep-registry-modules/avm/ptn -path "*/tests/*" -name "*.bicep" -exec grep -l "module.*'br/public:avm\|module.*'br:mcr\.microsoft\.com.*avm" {} \; 2>/dev/null | wc -l | tr -d ' ') +echo "Test files with AVM modules: $bicep_avm_files" + +echo + +echo "2. TERRAFORM PATTERN MODULE EXAMPLES ANALYSIS" +echo "==============================================" + +# Count actual Terraform pattern modules +terraform_modules=$(ls -d /Azure/terraform-azurerm-avm-ptn*/ 2>/dev/null | wc -l | tr -d ' ') +echo "Total Terraform pattern modules: $terraform_modules" + +# Count Terraform files +terraform_total=$(find /Azure/terraform-azurerm-avm-ptn* -path "*/examples/*" -name "main.tf" 2>/dev/null | wc -l | tr -d ' ') +echo "Total Terraform EXAMPLE files analyzed: $terraform_total" + +# Count files with native resources +terraform_native_files=$(find /Azure/terraform-azurerm-avm-ptn* -path "*/examples/*" -name "main.tf" -exec grep -l 'resource "azurerm_' {} \; 2>/dev/null | wc -l | tr -d ' ') +echo "Example files with native azurerm resources: $terraform_native_files" + +# Count files with AVM modules +terraform_avm_files=$(find /Azure/terraform-azurerm-avm-ptn* -path "*/examples/*" -name "main.tf" -exec grep -l 'source.*=.*"Azure/avm-' {} \; 2>/dev/null | wc -l | tr -d ' ') +echo "Example files with AVM modules: $terraform_avm_files" + +echo + +echo "3. STATISTICAL SUMMARY" +echo "======================" + +# Calculate percentages for Bicep (file-based analysis) +if [ $bicep_total -gt 0 ]; then + bicep_native_pct=$(echo "scale=1; $bicep_native_files * 100 / $bicep_total" | bc -l) + bicep_avm_pct=$(echo "scale=1; $bicep_avm_files * 100 / $bicep_total" | bc -l) +else + bicep_native_pct=0 + bicep_avm_pct=0 +fi + +# Calculate percentages for Terraform (file-based analysis) +if [ $terraform_total -gt 0 ]; then + terraform_native_pct=$(echo "scale=1; $terraform_native_files * 100 / $terraform_total" | bc -l) + terraform_avm_pct=$(echo "scale=1; $terraform_avm_files * 100 / $terraform_total" | bc -l) +else + terraform_native_pct=0 + terraform_avm_pct=0 +fi + +echo "MARKDOWN TABLE" +echo "==============" +echo +echo "| Ecosystem | Pattern Modules | Test/Example Files | Native Resources | AVM Modules | Native % | AVM % |" +echo "|-----------|----------------|-------------------|------------------|-------------|----------|-------|" +echo "| Bicep (Tests) | $bicep_modules | $bicep_total | $bicep_native_files | $bicep_avm_files | ${bicep_native_pct}% | ${bicep_avm_pct}% |" +echo "| Terraform (Examples) | $terraform_modules | $terraform_total | $terraform_native_files | $terraform_avm_files | ${terraform_native_pct}% | ${terraform_avm_pct}% |" +echo "| **Total** | **$((bicep_modules + terraform_modules))** | **$((bicep_total + terraform_total))** | **$((bicep_native_files + terraform_native_files))** | **$((bicep_avm_files + terraform_avm_files))** | **$(echo "scale=1; ($bicep_native_files + $terraform_native_files) * 100 / ($bicep_total + $terraform_total)" | bc -l)%** | **$(echo "scale=1; ($bicep_avm_files + $terraform_avm_files) * 100 / ($bicep_total + $terraform_total)" | bc -l)%** |" +echo +echo + +echo "BICEP PATTERN MODULE TESTS (File-based Analysis):" +echo "- Pattern modules: $bicep_modules" +echo "- Test files analyzed: $bicep_total" +echo "- Test files with native resources: $bicep_native_files/$bicep_total (${bicep_native_pct}%)" +echo "- Test files with AVM modules: $bicep_avm_files/$bicep_total (${bicep_avm_pct}%)" +echo + +echo "TERRAFORM PATTERN MODULE EXAMPLES (File-based Analysis):" +echo "- Pattern modules: $terraform_modules" +echo "- Example files analyzed: $terraform_total" +echo "- Example files with native resources: $terraform_native_files/$terraform_total (${terraform_native_pct}%)" +echo "- Example files with AVM modules: $terraform_avm_files/$terraform_total (${terraform_avm_pct}%)" +echo + +# Calculate overall statistics +total_files=$((bicep_total + terraform_total)) +total_native_files=$((bicep_native_files + terraform_native_files)) +total_avm_files=$((bicep_avm_files + terraform_avm_files)) + +if [ $total_files -gt 0 ]; then + overall_native_pct=$(echo "scale=1; $total_native_files * 100 / $total_files" | bc -l) + overall_avm_pct=$(echo "scale=1; $total_avm_files * 100 / $total_files" | bc -l) +else + overall_native_pct=0 + overall_avm_pct=0 +fi + +echo "OVERALL COMBINED ANALYSIS:" +echo "- Test/Example files with native resources: $total_native_files/$total_files (${overall_native_pct}%)" +echo "- Test/Example files with AVM modules: $total_avm_files/$total_files (${overall_avm_pct}%)" +echo + +echo "4. CONCLUSION" +echo "=============" +if [ $total_avm_files -gt $total_native_files ]; then + echo "❌ Pattern module tests/examples STILL predominantly use NATIVE resources rather than AVM modules" + echo " Native: $total_native_files files (${overall_native_pct}%) vs AVM: $total_avm_files files (${overall_avm_pct}%)" +elif [ $total_native_files -gt $total_avm_files ]; then + echo "❌ Pattern module tests/examples predominantly use NATIVE resources rather than AVM modules" + echo " Native: $total_native_files files (${overall_native_pct}%) vs AVM: $total_avm_files files (${overall_avm_pct}%)" +else + echo "⚖️ Equal usage of native resources and AVM modules in pattern module tests/examples" +fi + +echo +echo "Key Insights:" +echo "- Bicep: $bicep_modules pattern modules with $bicep_total test files (avg $(echo "scale=1; $bicep_total / $bicep_modules" | bc -l) files/module)" +echo "- Terraform: $terraform_modules pattern modules with $terraform_total example files (avg $(echo "scale=1; $terraform_total / $terraform_modules" | bc -l) files/module)" +echo "- Note: Terraform average is skewed by modules with extensive example suites" +echo "- Both ecosystems show similar pattern: ~70-74% native resource usage in tests/examples" +echo "- This analysis is about TEST and EXAMPLE files, NOT the pattern modules themselves" +echo "- Pattern module tests/examples are not yet fully demonstrating 'modules calling modules' approach" diff --git a/PowerShell/Snippets/AVM-ModuleTester.ps1 b/PowerShell/Snippets/AVM-ModuleTester.ps1 index deda66a..8428b52 100644 --- a/PowerShell/Snippets/AVM-ModuleTester.ps1 +++ b/PowerShell/Snippets/AVM-ModuleTester.ps1 @@ -1,22 +1,38 @@ +# AVM Module Tester Script +# Before running this script, make sure to: +# 1. Replace '' with your actual Azure subscription ID +# 2. Replace '' with your desired naming prefix +# 3. Replace '' with your Azure AD tenant ID +# 4. Ensure you have the required Azure PowerShell modules installed + # Start pwsh if not started yet # pwsh # Set default directory -$folder = "Git/Azure/bicep-registry-modules" # location of your local clone of bicep-registry-modules +$folder = "Git/GitHub/Azure/bicep-registry-modules" # location of your local clone of bicep-registry-modules -# Dot source functions +# Ensure Azure PowerShell authentication +if (-not (Get-AzContext)) { + Write-Output "No Azure context found. Please authenticate..." + Connect-AzAccount +} + +# Set the subscription context (update with your actual subscription ID) +$subscriptionId = '' # Replace with your actual subscription ID +if ($subscriptionId -ne '') { + Set-AzContext -SubscriptionId $subscriptionId +} -. $folder/avm/utilities/tools/Set-AVMModule.ps1 -. $folder/avm/utilities/tools/Test-ModuleLocally.ps1 +# Dot source functions +. $folder/utilities/tools/Set-AVMModule.ps1 +. $folder/utilities/tools/Test-ModuleLocally.ps1 # Variables $modules = @( - "dev-center/devcenter" - # "managed-services/registration-definition" - # "compute/disk-encryption-set" - # "compute/disk" + "web/site" # 5599 + # "communication/communication-service" # 5598 ) # Generate Readme @@ -27,26 +43,30 @@ foreach ($module in $modules) { # Set up test settings - $testcases = "waf-aligned", "max", "defaults" + $testcases = "functionApp.defaults", "webApp.max" #, "waf-aligned", "max", "defaults" + # $testcase = "all" $TestModuleLocallyInput = @{ TemplateFilePath = "$folder/avm/res/$module/main.bicep" PesterTest = $true ValidationTest = $true - DeploymentTest = $false + DeploymentTest = $true ValidateOrDeployParameters = @{ Location = 'australiaeast' - SubscriptionId = '' + SubscriptionId = $subscriptionId RemoveDeployment = $true } AdditionalTokens = @{ - namePrefix = '' - TenantId = '' + namePrefix = 'asf3re' # Replace with your prefix + TenantId = '' # Replace with your tenant ID } } # Run tests - + # if testcase is 'all' browse all folders in tests/e2e + if ($testcase -eq "all") { + $testcases = Get-ChildItem -Path "$folder/avm/res/$module/tests/e2e" -Directory | ForEach-Object { $_.Name } + } foreach ($testcase in $testcases) { Write-Output "Running test case $testcase on module $module" $TestModuleLocallyInput.ModuleTestFilePath = "$folder/avm/res/$module/tests/e2e/$testcase/main.test.bicep" diff --git a/_in progress/clone-azure-avm-repos.ps1 b/_in progress/clone-azure-avm-repos.ps1 new file mode 100755 index 0000000..4047b6a --- /dev/null +++ b/_in progress/clone-azure-avm-repos.ps1 @@ -0,0 +1,133 @@ +#!/usr/bin/env pwsh + +# Script to clone or update all terraform-azurerm-avm-* repos from Azure GitHub organization +# Date: June 20, 2025 + +$ErrorActionPreference = "Stop" +$orgName = "Azure" +$repoPrefix = "terraform-azurerm-avm-" +$baseDirectory = "/Users/segraef/Git/GitHub/$orgName" + +# Function to check if user is authenticated to GitHub +function Test-GithubAuth { + try { + $authStatus = gh auth status 2>&1 + if ($authStatus -match "Logged in to github.com") { + Write-Host "✅ Already authenticated to GitHub" -ForegroundColor Green + return $true + } else { + Write-Host "⚠️ Not authenticated to GitHub" -ForegroundColor Yellow + return $false + } + } + catch { + Write-Host "⚠️ Not authenticated to GitHub" -ForegroundColor Yellow + return $false + } +} + +# Function to authenticate to GitHub +function Connect-Github { + Write-Host "🔑 Please authenticate to GitHub..." + gh auth login + if (-not (Test-GithubAuth)) { + Write-Host "❌ Failed to authenticate to GitHub. Exiting script." -ForegroundColor Red + exit 1 + } +} + +# Function to update existing repository +function Update-Repository { + param ( + [string]$repoPath, + [string]$repoName + ) + + Write-Host "🔄 Updating repository: $repoName" -ForegroundColor Cyan + Set-Location $repoPath + + # Check if we're on main branch, if not switch to it + $currentBranch = git branch --show-current + if ($currentBranch -ne "main") { + Write-Host " Switching to main branch..." + git switch main + } + + # Fetch and pull latest changes + Write-Host " Fetching latest changes..." + git fetch --all + Write-Host " Pulling latest changes..." + git pull + Write-Host " ✅ Repository updated: $repoName" -ForegroundColor Green +} + +# Function to clone new repository +function New-Repository { + param ( + [string]$repoPath, + [string]$repoName, + [string]$repoUrl + ) + + Write-Host "📥 Cloning repository: $repoName" -ForegroundColor Magenta + git clone $repoUrl $repoPath + if ($LASTEXITCODE -eq 0) { + Write-Host " ✅ Repository cloned: $repoName" -ForegroundColor Green + } else { + Write-Host " ❌ Failed to clone repository: $repoName" -ForegroundColor Red + } +} + +# Check if authenticated to GitHub +if (-not (Test-GithubAuth)) { + Connect-Github +} + +# Create base directory if it doesn't exist +if (-not (Test-Path $baseDirectory)) { + Write-Host "📁 Creating base directory: $baseDirectory" -ForegroundColor Yellow + New-Item -ItemType Directory -Path $baseDirectory -Force | Out-Null +} + +# Change to base directory +Set-Location $baseDirectory + +# Get all repositories starting with the prefix +Write-Host "🔍 Searching for repositories with prefix: $repoPrefix in $orgName organization..." -ForegroundColor Blue +$repos = gh repo list $orgName --json name,url --limit 1000 | ConvertFrom-Json | Where-Object { $_.name -like "$repoPrefix*" } + +if (-not $repos) { + Write-Host "❌ No repositories found matching the prefix: $repoPrefix" -ForegroundColor Red + exit 1 +} + +Write-Host "🎉 Found $($repos.Count) repositories matching the prefix" -ForegroundColor Green + +# Process each repository +foreach ($repo in $repos) { + $repoName = $repo.name + $repoUrl = $repo.url + $repoPath = Join-Path $baseDirectory $repoName + + # Check if repository exists locally + if (Test-Path $repoPath) { + # Check if it's a Git repository + if (Test-Path (Join-Path $repoPath ".git")) { + # Update repository + Update-Repository -repoPath $repoPath -repoName $repoName + } else { + Write-Host "⚠️ Directory exists but is not a Git repository: $repoName" -ForegroundColor Yellow + # Rename existing directory + $backupPath = "$repoPath-backup-$(Get-Date -Format 'yyyyMMddHHmmss')" + Write-Host " Moving existing directory to $backupPath" + Move-Item -Path $repoPath -Destination $backupPath + # Clone repository + New-Repository -repoPath $repoPath -repoName $repoName -repoUrl $repoUrl + } + } else { + # Clone repository + New-Repository -repoPath $repoPath -repoName $repoName -repoUrl $repoUrl + } +} + +Write-Host "✅ All repositories processed successfully!" -ForegroundColor Green diff --git a/_in progress/process-aiml-resource-groups.ps1 b/_in progress/process-aiml-resource-groups.ps1 new file mode 100755 index 0000000..4d5c4fd --- /dev/null +++ b/_in progress/process-aiml-resource-groups.ps1 @@ -0,0 +1,160 @@ +#!/usr/bin/env pwsh + +# Script to iterate through Azure resource groups starting with "rg-aiml-" +# Date: June 20, 2025 + +$ErrorActionPreference = "Stop" +$rgPrefix = "rg-aiml-" + +# Role assignment parameters +$roleDefinitionName = "Owner" +$objectId = "" +$objectType = "Group" + +# Note: Azure CLI doesn't support --start-time and --end-time parameters directly +# Time-bound assignments must be done through the Azure Portal or PowerShell Az module + +# Function to check if user is authenticated to Azure +function Test-AzureAuth { + try { + $account = az account show --query "name" -o tsv 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "✅ Already authenticated to Azure as: $account" -ForegroundColor Green + return $true + } else { + Write-Host "⚠️ Not authenticated to Azure" -ForegroundColor Yellow + return $false + } + } + catch { + Write-Host "⚠️ Not authenticated to Azure" -ForegroundColor Yellow + return $false + } +} + +# Function to authenticate to Azure +function Connect-Azure { + Write-Host "🔑 Please authenticate to Azure..." + az login + if (-not (Test-AzureAuth)) { + Write-Host "❌ Failed to authenticate to Azure. Exiting script." -ForegroundColor Red + exit 1 + } + + # List available subscriptions and let user select one if there are multiple + $subscriptions = az account list --query "[].{Name:name, Id:id, IsDefault:isDefault}" -o json | ConvertFrom-Json + if ($subscriptions.Count -gt 1) { + Write-Host "Multiple subscriptions found. Please select one:" -ForegroundColor Yellow + for ($i = 0; $i -lt $subscriptions.Count; $i++) { + $defaultMark = if ($subscriptions[$i].IsDefault) { "[DEFAULT]" } else { "" } + Write-Host "[$i] $($subscriptions[$i].Name) ($($subscriptions[$i].Id)) $defaultMark" + } + + $selection = Read-Host "Enter the number of the subscription to use (press Enter for default)" + if ($selection -ne "") { + $selectedSubscription = $subscriptions[$selection].Id + Write-Host "Setting subscription to: $($subscriptions[$selection].Name)" + az account set --subscription $selectedSubscription + } + } +} + +# Check if authenticated to Azure +if (-not (Test-AzureAuth)) { + Connect-Azure +} + +# Get current subscription details +$currentSubscription = az account show --query "{Name:name, Id:id}" -o json | ConvertFrom-Json +Write-Host "Using subscription: $($currentSubscription.Name) ($($currentSubscription.Id))" -ForegroundColor Blue + +# Get all resource groups starting with the prefix +Write-Host "🔍 Searching for resource groups with prefix: $rgPrefix..." -ForegroundColor Blue +$resourceGroups = az group list --query "[?starts_with(name, '$rgPrefix')].{Name:name, Location:location, Tags:tags}" -o json | ConvertFrom-Json + +if (-not $resourceGroups -or $resourceGroups.Count -eq 0) { + Write-Host "❌ No resource groups found matching the prefix: $rgPrefix" -ForegroundColor Red + exit 1 +} + +Write-Host "🎉 Found $($resourceGroups.Count) resource groups matching the prefix" -ForegroundColor Green + +# Process each resource group +foreach ($rg in $resourceGroups) { + Write-Host "Processing resource group: $($rg.Name) in $($rg.Location)" -ForegroundColor Cyan + + # Get resources in the resource group + Write-Host " 📋 Listing resources in resource group $($rg.Name)..." + $resources = az resource list --resource-group $rg.Name --query "[].{Name:name, Type:type, Location:location}" -o json | ConvertFrom-Json + + Write-Host " 📊 Found $($resources.Count) resources in resource group $($rg.Name)" -ForegroundColor Yellow + + # Example: Display resources by type + $resourceTypes = $resources | Group-Object -Property Type + foreach ($type in $resourceTypes) { + Write-Host " 🔹 $($type.Count) resources of type: $($type.Name)" -ForegroundColor Magenta + foreach ($resource in $type.Group) { + Write-Host " - $($resource.Name) ($($resource.Location))" + } + } + + # Example: Get detailed information for specific resource types + # Uncomment and modify as needed + <# + $storageAccounts = $resources | Where-Object { $_.Type -eq "Microsoft.Storage/storageAccounts" } + if ($storageAccounts) { + Write-Host " 💾 Storage Account details:" -ForegroundColor Green + foreach ($sa in $storageAccounts) { + $saDetails = az storage account show --name $sa.Name --resource-group $rg.Name --query "{Name:name, Sku:sku.name, Kind:kind}" -o json | ConvertFrom-Json + Write-Host " - $($saDetails.Name) (SKU: $($saDetails.Sku), Kind: $($saDetails.Kind))" + } + } + #> + + # Example: Custom operations for each resource group + # This is where you can add your specific operations + Write-Host " 🔧 Performing custom operations for resource group $($rg.Name)..." -ForegroundColor Cyan + + # Assign the external group as Owner to the resource group + Write-Host " 👥 Assigning external group as Owner to resource group $($rg.Name)..." -ForegroundColor Yellow + + # Get the scope for the resource group + $scope = "/subscriptions/$($currentSubscription.Id)/resourceGroups/$($rg.Name)" + + # Check if role assignment already exists + $existingAssignment = az role assignment list --assignee $objectId --role $roleDefinitionName --scope $scope --query "[0]" -o json 2>$null | ConvertFrom-Json + + if ($existingAssignment) { + Write-Host " ⚠️ Role assignment already exists for this group on resource group $($rg.Name)" -ForegroundColor Yellow + } + else { + # Create the role assignment (no time-bound options in Azure CLI) + try { + $assignmentResult = az role assignment create ` + --role $roleDefinitionName ` + --assignee-object-id $objectId ` + --assignee-principal-type $objectType ` + --scope $scope -o json 2>&1 + + if ($LASTEXITCODE -eq 0) { + Write-Host " ✅ Successfully assigned role 'Owner' to external group for resource group $($rg.Name)" -ForegroundColor Green + } else { + Write-Host " ❌ Failed to assign role: $assignmentResult" -ForegroundColor Red + } + } + catch { + Write-Host " ❌ Error assigning role: $_" -ForegroundColor Red + } + } + + Write-Host " ✅ Completed processing resource group: $($rg.Name)" -ForegroundColor Green + Write-Host "" +} + +Write-Host "✅ All resource groups processed successfully!" -ForegroundColor Green +Write-Host "" +Write-Host "📝 Note: For time-bound role assignments (with start/end dates):" -ForegroundColor Yellow +Write-Host " Azure CLI doesn't support setting expiration dates directly." -ForegroundColor Yellow +Write-Host " To create time-bound assignments like shown in the screenshot, use PowerShell Az module instead:" -ForegroundColor Yellow +Write-Host " New-AzRoleAssignment -ObjectId $objectId -RoleDefinitionName $roleDefinitionName -Scope \$scope -ExpiryOn '2025-12-17T15:29:15Z'" -ForegroundColor Cyan +Write-Host ""