# Dev Drive benchmark, from the Smarter Business Tech post
# "Defender, Claude Code and the Dev Drive: what we measured"
# https://smarterbusiness.tech/blog/defender-claude-code-and-the-dev-drive
#
# Times the local work an AI coding session does in a Node project (clone, npm ci,
# build, git status, delete) with the code and npm cache on C: and on a Dev Drive,
# and how much CPU Microsoft Defender (MsMpEng) spends during each step. Optionally
# times starting a Python tool through uvx. It changes no Windows or Defender
# settings: the npm and uv caches are chosen per command, not globally.
#
# Read it before you run it. Run it in Windows PowerShell 5.1 or PowerShell 7,
# from a normal (not administrator) window:
#
#   powershell -NoProfile -ExecutionPolicy Bypass -File .\dev-drive-benchmark.ps1 -Source D:\code\my-node-repo -DevDrive D:
#
# -Source must be a git repository whose `npm ci` and `npm run build` work on your
# machine. Results go to <DevDrive>\bench\results as CSV files. The first run fills
# the npm and uv caches from the network; after that every timed step runs offline
# where it can. Delete C:\Users\<you>\bench and <DevDrive>\bench when you are done.

param(
    [Parameter(Mandatory = $true)][string]$Source,
    [string]$DevDrive = 'D:',
    [int]$Passes = 7,
    [string]$Label = 'run',
    [string]$CRoot = "$env:USERPROFILE\bench",
    [int]$Spawns = 100,
    # A Python tool to start through uvx, for example mcp-proxy-for-aws. Leave empty to skip.
    [string]$UvTool = ''
)

$ErrorActionPreference = 'Stop'
$env:ASTRO_TELEMETRY_DISABLED = '1'
$env:NO_UPDATE_NOTIFIER = '1'
$DRoot = "$DevDrive\bench"
$bash = 'C:\Program Files\Git\usr\bin\bash.exe'
$useUv = $UvTool -and (Get-Command uvx -ErrorAction SilentlyContinue)

$conditions = @(
    [pscustomobject]@{ Name = 'C: code, C: caches'; Work = "$CRoot\work"; Npm = "$CRoot\cache\npm"; Uv = "$CRoot\cache\uv" },
    [pscustomobject]@{ Name = 'D: code, C: caches'; Work = "$DRoot\work"; Npm = "$CRoot\cache\npm"; Uv = "$CRoot\cache\uv" },
    [pscustomobject]@{ Name = 'D: code, D: caches'; Work = "$DRoot\work"; Npm = "$DRoot\cache\npm"; Uv = "$DRoot\cache\uv" }
)

$stamp = Get-Date -Format 'yyyy-MM-dd_HHmm'
$outDir = "$DRoot\results\$Label-$stamp"
New-Item -ItemType Directory -Force -Path $outDir, "$outDir\logs" | Out-Null
$csv = "$outDir\steps.csv"

$defender = New-Object System.Diagnostics.PerformanceCounter('Process', '% Processor Time', 'MsMpEng')
$total = New-Object System.Diagnostics.PerformanceCounter('Processor', '% Processor Time', '_Total')

function Write-Note([string]$text) { Write-Host ("[{0:HH:mm:ss}] {1}" -f (Get-Date), $text) }

# Runs a command line through cmd so its output goes to a log and never trips PowerShell's error handling.
function Invoke-Logged([string]$commandLine, [string]$cwd, [string]$log) {
    Push-Location $cwd
    try { cmd /c "$commandLine > `"$log`" 2>&1"; return $LASTEXITCODE }
    finally { Pop-Location }
}

function Measure-Step([int]$pass, [string]$condition, [string]$step, [scriptblock]$action) {
    [void]$total.NextValue(); Start-Sleep -Seconds 1
    $background = [math]::Round($total.NextValue(), 1)
    $d0 = $defender.RawValue
    $sw = [Diagnostics.Stopwatch]::StartNew()
    $exit = & $action
    $sw.Stop()
    # On a Dev Drive Defender scans after the file opens, so some of its work lands after the
    # step ends. Wait until it goes quiet (or 15 seconds) and count that work against this step.
    $settle = [Diagnostics.Stopwatch]::StartNew()
    do {
        $before = $defender.RawValue; Start-Sleep -Milliseconds 500
        $recent = ($defender.RawValue - $before) / 1e7
    } while ($recent -gt 0.05 -and $settle.Elapsed.TotalSeconds -lt 15)
    $defenderSeconds = ($defender.RawValue - $d0) / 1e7
    $row = [pscustomobject]@{
        label = $Label; pass = $pass; condition = $condition; step = $step
        seconds = [math]::Round($sw.Elapsed.TotalSeconds, 3)
        defender_cpu_s = [math]::Round($defenderSeconds, 3)
        settle_s = [math]::Round($settle.Elapsed.TotalSeconds, 1)
        background_cpu_pct = $background
        exit = if ($null -eq $exit) { 0 } else { ($exit | Select-Object -Last 1) }
        time = (Get-Date -Format 's')
    }
    $row | Export-Csv -Path $csv -Append -NoTypeInformation
    Write-Note ("pass {0}  {1,-20} {2,-16} {3,8:N2}s  Defender {4,6:N2}s  exit {5}" -f $pass, $condition, $step, $row.seconds, $row.defender_cpu_s, $row.exit)
}

function Remove-Tree([string]$path) { if (Test-Path $path) { cmd /c "rmdir /s /q `"$path`"" } }

# Machine and Defender details, for comparing results.
$mp = Get-MpComputerStatus
[pscustomobject]@{
    label = $Label; started = (Get-Date -Format 's'); passes = $Passes; source = $Source
    cpu = (Get-CimInstance Win32_Processor | Select-Object -First 1).Name
    ram_gb = [math]::Round((Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory / 1GB)
    disk = (Get-CimInstance Win32_DiskDrive | Select-Object -First 1).Model
    windows_build = (Get-CimInstance Win32_OperatingSystem).BuildNumber
    volumes = (Get-Volume | Where-Object DriveLetter | ForEach-Object { "$($_.DriveLetter): $($_.FileSystemType) $($_.FileSystemLabel)" }) -join '; '
    defender_platform = $mp.AMProductVersion; defender_engine = $mp.AMEngineVersion
    defender_signatures = $mp.AntivirusSignatureVersion; realtime = $mp.RealTimeProtectionEnabled
    node = (node --version); npm = (cmd /c npm --version); git = (git --version)
} | ConvertTo-Json | Set-Content "$outDir\machine.json"

# Fill each cache once, untimed, so no timed step waits on the network.
foreach ($c in $conditions | Sort-Object Npm -Unique) {
    if (Test-Path "$($c.Npm)\.warm") { continue }
    Write-Note "Filling npm cache $($c.Npm)"
    New-Item -ItemType Directory -Force -Path $c.Work, $c.Npm | Out-Null
    $dir = "$($c.Work)\warm"; Remove-Tree $dir
    git clone --no-hardlinks -q $Source $dir
    $exit = Invoke-Logged "npm ci --no-audit --no-fund --cache `"$($c.Npm)`"" $dir "$outDir\logs\warm-npm.log"
    if ($exit -ne 0) { throw "npm ci failed while filling $($c.Npm); see $outDir\logs\warm-npm.log" }
    Remove-Tree $dir
    New-Item -ItemType File -Path "$($c.Npm)\.warm" | Out-Null
}
if ($useUv) {
    foreach ($c in $conditions | Sort-Object Uv -Unique) {
        if (Test-Path "$($c.Uv)\.warm") { continue }
        Write-Note "Filling uv cache $($c.Uv)"
        New-Item -ItemType Directory -Force -Path $c.Uv | Out-Null
        $env:UV_CACHE_DIR = $c.Uv
        $exit = Invoke-Logged "uvx $UvTool --help" $DRoot "$outDir\logs\warm-uv.log"
        if ($exit -ne 0) { throw "uvx failed while filling $($c.Uv); see $outDir\logs\warm-uv.log" }
        New-Item -ItemType File -Path "$($c.Uv)\.warm" | Out-Null
    }
    Remove-Item Env:UV_CACHE_DIR -ErrorAction SilentlyContinue
}

for ($pass = 1; $pass -le $Passes; $pass++) {
    # Rotate the order each pass so a busy minute does not always land on the same condition.
    $order = for ($i = 0; $i -lt $conditions.Count; $i++) { $conditions[($i + $pass - 1) % $conditions.Count] }
    foreach ($c in $order) {
        $dir = "$($c.Work)\project"
        Remove-Tree $dir
        $log = "$outDir\logs\p$pass-" + ($c.Name -replace '[^A-Za-z]', '')
        Measure-Step $pass $c.Name 'clone' { git clone --no-hardlinks -q $Source $dir; $LASTEXITCODE }
        Measure-Step $pass $c.Name 'npm ci' { Invoke-Logged "npm ci --prefer-offline --no-audit --no-fund --cache `"$($c.Npm)`"" $dir "$log-ci.log" }
        Measure-Step $pass $c.Name 'build' { Invoke-Logged 'npm run build' $dir "$log-build.log" }
        Measure-Step $pass $c.Name 'git status x10' { for ($i = 0; $i -lt 10; $i++) { $null = git -C $dir status --porcelain }; $LASTEXITCODE }
        if ($useUv) {
            $env:UV_CACHE_DIR = $c.Uv
            Measure-Step $pass $c.Name 'uvx tool, cold' { Invoke-Logged "uvx --offline --reinstall $UvTool --help" $dir "$log-uv-cold.log" }
            Measure-Step $pass $c.Name 'uvx tool, warm' { Invoke-Logged "uvx --offline $UvTool --help" $dir "$log-uv-warm.log" }
            Remove-Item Env:UV_CACHE_DIR
        }
        Measure-Step $pass $c.Name 'delete' { Remove-Tree $dir; $LASTEXITCODE }
    }
    # Starting a program costs the same wherever the code is, so this runs once a pass.
    Measure-Step $pass 'any' "start git x$Spawns" { for ($i = 0; $i -lt $Spawns; $i++) { $null = git --version }; $LASTEXITCODE }
    Measure-Step $pass 'any' "start node x$Spawns" { for ($i = 0; $i -lt $Spawns; $i++) { $null = node -e 0 }; $LASTEXITCODE }
    if (Test-Path $bash) {
        Measure-Step $pass 'any' "start bash x$Spawns" { for ($i = 0; $i -lt $Spawns; $i++) { $null = & $bash -c 'exit 0' }; $LASTEXITCODE }
    }
}

# Medians per condition and step.
function Get-Median([double[]]$values) {
    $s = @($values | Sort-Object); $n = $s.Count
    if ($n -eq 0) { return $null }
    if ($n % 2) { return $s[[int][math]::Floor($n / 2)] }
    return ($s[$n / 2 - 1] + $s[$n / 2]) / 2
}
$rows = Import-Csv $csv
$summary = $rows | Group-Object condition, step | ForEach-Object {
    $g = $_.Group
    [pscustomobject]@{
        condition = $g[0].condition; step = $g[0].step; runs = $g.Count
        median_s = [math]::Round((Get-Median ($g | ForEach-Object { [double]$_.seconds })), 2)
        min_s = [math]::Round(($g | ForEach-Object { [double]$_.seconds } | Measure-Object -Minimum).Minimum, 2)
        max_s = [math]::Round(($g | ForEach-Object { [double]$_.seconds } | Measure-Object -Maximum).Maximum, 2)
        median_defender_cpu_s = [math]::Round((Get-Median ($g | ForEach-Object { [double]$_.defender_cpu_s })), 2)
        median_background_pct = [math]::Round((Get-Median ($g | ForEach-Object { [double]$_.background_cpu_pct })), 1)
        failed_runs = @($g | Where-Object { $_.exit -ne '0' }).Count
    }
}
$summary | Export-Csv "$outDir\summary.csv" -NoTypeInformation
$summary | Format-Table -AutoSize | Out-String -Width 200 | Write-Host
Write-Note "Results in $outDir"