Kyle Bracke

<#
.SYNOPSIS
    Nightly SQL Server health check. Covers backup age, index fragmentation,
    and long-running queries. Emails a formatted report with status flags.
#>

$ErrorActionPreference = 'Stop'
$Config = @{
    SQLServer     = 'SQL-SERVER-01'
    Databases     = @('CompanyDB', 'SAPB1DB', 'ReportingDB')
    Recipients    = @('[email protected]', '[email protected]')
    SmtpServer    = 'smtp.company.com'
    FragThreshPct = 30      # flag indexes above this fragmentation %
    LongQuerySec  = 60      # flag queries running longer than this
    BackupMaxSec  = 86400   # 24 hours
}

$Results = [System.Collections.Generic.List[PSCustomObject]]::new()
$Errors  = [System.Collections.Generic.List[string]]::new()

foreach ($DB in $Config.Databases) {
    try {
        $SizeBackup = Invoke-Sqlcmd -ServerInstance $Config.SQLServer -Database $DB `
            -ErrorAction Stop -Query @"
SELECT
    DB_NAME()                                    AS DatabaseName,
    CAST(SUM(size * 8.0 / 1024) AS INT)          AS SizeMB,
    (SELECT TOP 1 backup_finish_date
     FROM   msdb.dbo.backupset
     WHERE  database_name = DB_NAME() AND type = 'D'
     ORDER  BY backup_finish_date DESC)           AS LastFullBackup
FROM sys.database_files
WHERE type = 0
"@
        # Pull index data and filter in PowerShell to avoid SQL comparison operators
        $IndexData = Invoke-Sqlcmd -ServerInstance $Config.SQLServer -Database $DB -Query @"
SELECT index_id, avg_fragmentation_in_percent
FROM   sys.dm_db_index_physical_stats(DB_ID(), NULL, NULL, NULL, 'LIMITED')
WHERE  index_type_desc != 'HEAP'
"@
        $FragCount = ($IndexData |
            Where-Object { $_.avg_fragmentation_in_percent -gt $Config.FragThreshPct }).Count

        $AgeSec = if ($SizeBackup.LastFullBackup) {
            [int]((Get-Date) - $SizeBackup.LastFullBackup).TotalSeconds
        } else { [int]::MaxValue }

        $Results.Add([PSCustomObject]@{
            Database       = $DB
            SizeMB         = $SizeBackup.SizeMB
            LastFullBackup = if ($SizeBackup.LastFullBackup) {
                                 $SizeBackup.LastFullBackup.ToString('yyyy-MM-dd HH:mm')
                             } else { 'NEVER' }
            BackupAgeHours = [Math]::Round($AgeSec / 3600, 1)
            BackupStatus   = if ($AgeSec -le $Config.BackupMaxSec) { 'OK' } else { 'STALE' }
            FragIdxCount   = $FragCount
        })
    }
    catch { $Errors.Add("${DB}: $_") }
}

# Long-running query check (fetch all active requests, filter in PowerShell)
try {
    $AllActive = Invoke-Sqlcmd -ServerInstance $Config.SQLServer -Database master -Query @"
SELECT r.session_id, DB_NAME(r.database_id) AS DatabaseName,
       r.status, r.command,
       r.total_elapsed_time / 1000           AS ElapsedSec,
       SUBSTRING(t.text, 1, 100)             AS QuerySnippet
FROM sys.dm_exec_requests r
CROSS APPLY sys.dm_exec_sql_text(r.sql_handle) t
WHERE r.session_id != @@SPID
"@
    $LongRunning = $AllActive | Where-Object {
        $_.session_id -gt 50 -and $_.ElapsedSec -gt $Config.LongQuerySec
    } | Select-Object -First 5
} catch { $LongRunning = @() }

$HasIssues = ($Results | Where-Object { $_.BackupStatus -eq 'STALE' }) -or $Errors -or $LongRunning
$Subject   = "SQL Health $(if ($HasIssues) {'[ACTION REQUIRED]'} else {'[OK]'}) - $(Get-Date -f 'yyyy-MM-dd')"

$Body  = "SQL Server Health Report - $(Get-Date -f 'yyyy-MM-dd HH:mm')`n"
$Body += ("=" * 60) + "`n`n"
$Body += ($Results | Format-Table Database, SizeMB, LastFullBackup, BackupAgeHours, BackupStatus, FragIdxCount -AutoSize | Out-String)
if ($LongRunning) { $Body += "`nLONG-RUNNING QUERIES:`n$($LongRunning | Format-Table -AutoSize | Out-String)" }
if ($Errors)      { $Body += "`nERRORS:`n$($Errors -join "`n")" }

Send-MailMessage -To $Config.Recipients -From '[email protected]' `
    -Subject $Subject -Body $Body -SmtpServer $Config.SmtpServer

Write-Host "Report sent to $($Config.Recipients -join ', '). Issues detected: $([int][bool]$HasIssues)"

#Requires -Modules Az.RecoveryServices, Az.Accounts
<#
.SYNOPSIS
    Azure Backup job audit with Teams alerting and retry logic.
    Checks the past 48 hours across a Recovery Services Vault.
    Runs nightly via Task Scheduler using managed identity auth.
#>

$ErrorActionPreference = 'Stop'
$Config = @{
    VaultName      = 'CompanyBackupVault'
    ResourceGroup  = 'Company-RG-Backup'
    LookbackHours  = 48
    MaxAuthRetries = 3
    TeamsWebhook   = $env:TEAMS_BACKUP_WEBHOOK   # stored in system env, never hardcoded
    AlertEmail     = '[email protected]'
    SmtpServer     = 'smtp.company.com'
}

function Send-TeamsAlert {
    param([string]$Title, [string]$Details, [string]$Color = 'attention')
    if (-not $Config.TeamsWebhook) { return }
    try {
        $Card = @{
            type        = 'message'
            attachments = @(@{
                contentType = 'application/vnd.microsoft.card.adaptive'
                content     = @{
                    '$schema' = 'http://adaptivecards.io/schemas/adaptive-card.json'
                    type      = 'AdaptiveCard'
                    version   = '1.2'
                    body      = @(
                        @{ type = 'TextBlock'; text = $Title;   weight = 'Bolder'; color = $Color },
                        @{ type = 'TextBlock'; text = $Details; wrap   = $true }
                    )
                }
            })
        } | ConvertTo-Json -Depth 10
        Invoke-RestMethod -Uri $Config.TeamsWebhook -Method Post -Body $Card `
            -ContentType 'application/json' -ErrorAction SilentlyContinue | Out-Null
    }
    catch { Write-Warning "Teams alert failed: $_" }
}

# Connect via managed identity with retry
$AuthAttempt = 0
do {
    try   { Connect-AzAccount -Identity -ErrorAction Stop; break }
    catch {
        $AuthAttempt++
        if ($AuthAttempt -ge $Config.MaxAuthRetries) {
            throw "Azure authentication failed after $AuthAttempt attempts: $_"
        }
        Write-Warning "Auth attempt $AuthAttempt failed. Retrying in 30s..."
        Start-Sleep -Seconds 30
    }
} while ($AuthAttempt -lt $Config.MaxAuthRetries)

$Vault = Get-AzRecoveryServicesVault -Name $Config.VaultName -ResourceGroupName $Config.ResourceGroup
Set-AzRecoveryServicesVaultContext -Vault $Vault

$Jobs    = Get-AzRecoveryServicesBackupJob -From (Get-Date).AddHours(-$Config.LookbackHours) -VaultId $Vault.ID
$Summary = [PSCustomObject]@{
    Total    = $Jobs.Count
    Success  = ($Jobs | Where-Object Status -eq 'Completed').Count
    Warnings = ($Jobs | Where-Object Status -eq 'CompletedWithWarnings').Count
    Failed   = ($Jobs | Where-Object Status -eq 'Failed').Count
    Running  = ($Jobs | Where-Object Status -eq 'InProgress').Count
}
Write-Host ($Summary | Format-List | Out-String)

$FailedJobs  = $Jobs | Where-Object Status -eq 'Failed'
$WarningJobs = $Jobs | Where-Object Status -eq 'CompletedWithWarnings'

if ($FailedJobs) {
    $Details = $FailedJobs | ForEach-Object {
        $Duration = if ($_.StartTime -and $_.EndTime) {
            "$([Math]::Round(($_.EndTime - $_.StartTime).TotalMinutes, 1))m"
        } else { 'N/A' }
        "$($_.WorkloadName) | Duration: $Duration | $($_.ErrorDetails.ErrorMessage -replace '\s+', ' ')"
    }
    $AlertText = "$($Summary.Failed) backup job(s) failed in vault '$($Config.VaultName)':`n$($Details -join "`n")"

    Send-TeamsAlert -Title "Azure Backup Failure ($($Summary.Failed) jobs)" `
        -Details ($Details -join ' | ') -Color 'attention'
    Send-MailMessage -To $Config.AlertEmail -From '[email protected]' `
        -Subject "ALERT: Azure Backup Failure - $(Get-Date -f 'yyyy-MM-dd')" `
        -Body $AlertText -SmtpServer $Config.SmtpServer
    Write-Warning $AlertText
}

if ($WarningJobs) {
    $WarnDetails = $WarningJobs | ForEach-Object { "$($_.WorkloadName): $($_.ErrorDetails.ErrorMessage)" }
    Write-Warning "$($Summary.Warnings) job(s) completed with warnings:`n$($WarnDetails -join "`n")"
    Send-TeamsAlert -Title "Backup Warnings ($($Summary.Warnings) jobs)" `
        -Details ($WarnDetails -join '; ') -Color 'warning'
}

if (-not $FailedJobs -and -not $WarningJobs) {
    Write-Host "All $($Summary.Success) backup jobs completed successfully." -ForegroundColor Green
}