552 lines
21 KiB
PowerShell
552 lines
21 KiB
PowerShell
<#
|
|
DISKSPD - VM Fleet
|
|
|
|
Copyright(c) Microsoft Corporation
|
|
All rights reserved.
|
|
|
|
MIT License
|
|
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
of this software and associated documentation files (the "Software"), to deal
|
|
in the Software without restriction, including without limitation the rights
|
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
copies of the Software, and to permit persons to whom the Software is
|
|
furnished to do so, subject to the following conditions:
|
|
|
|
The above copyright notice and this permission notice shall be included in all
|
|
copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
|
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
SOFTWARE.
|
|
#>
|
|
|
|
Set-StrictMode -Version 3.0
|
|
|
|
# display name
|
|
# ctr name
|
|
# display order
|
|
# format string
|
|
# scalar multiplier
|
|
|
|
class CounterColumn {
|
|
|
|
[string] $displayname
|
|
[string] $setname
|
|
[string[]] $ctrname
|
|
[int] $width
|
|
[string] $fmt
|
|
[decimal] $multiplier
|
|
[ValidateSet("Average","AverageAggregate","Sum")][string] $aggregate
|
|
[boolean] $divider
|
|
|
|
CounterColumn(
|
|
[string] $displayname,
|
|
[string] $setname,
|
|
[string[]] $ctrname,
|
|
[int] $width,
|
|
[string] $fmt,
|
|
[decimal] $multiplier,
|
|
[string] $aggregate,
|
|
[boolean] $divider
|
|
)
|
|
{
|
|
$this.displayname = $displayname
|
|
$this.setname = $setname
|
|
$this.ctrname = $ctrname
|
|
$this.width = $width
|
|
$this.fmt = $fmt
|
|
$this.multiplier = $multiplier
|
|
$this.aggregate = $aggregate
|
|
$this.divider = $divider
|
|
|
|
if ($this.width -lt $this.displayname.length + 1) {
|
|
$this.width = $this.displayname.length + 1
|
|
}
|
|
}
|
|
}
|
|
|
|
class CounterColumnSet {
|
|
|
|
[CounterColumn[]] $columns
|
|
[string[]] $counters
|
|
[string] $topfmt
|
|
[string] $linfmt
|
|
[string] $name
|
|
|
|
[string] $totalline
|
|
[string[]] $nodelines
|
|
|
|
CounterColumnSet($name)
|
|
{
|
|
$this.columns = $null
|
|
$this.name = $name
|
|
}
|
|
|
|
[void] Add([CounterColumn] $c)
|
|
{
|
|
$this.columns += $c
|
|
}
|
|
|
|
[void] Seal()
|
|
{
|
|
# assemble the top-line fmt and per-line fmt strings
|
|
# top-line is just strings/width
|
|
# per-line adds the (likely numeric) format specifier
|
|
$n = 1
|
|
$this.topfmt = $this.linfmt = "{0,-16}"
|
|
foreach ($col in $this.columns) {
|
|
$str = ''
|
|
if ($col.divider) {
|
|
$str += '| '
|
|
}
|
|
$str += "{$n,-$($col.width)"
|
|
$this.topfmt += "$str}"
|
|
$this.linfmt += "$($str):$($col.fmt)}"
|
|
|
|
$n += 1
|
|
}
|
|
|
|
# assemble the list of counter instances that will be needed
|
|
# note that some may be internally aggregated (i.e., total = read + write)
|
|
# in cases where a counterset does not provide an explicit total
|
|
$this.counters = ($this.columns |% {
|
|
$s = $_.setname
|
|
$_.ctrname |% { "\$($s)(_Total)\$($_)" }
|
|
} | group -NoElement).Name
|
|
}
|
|
|
|
[void] DisplayPre(
|
|
[hashtable] $samples,
|
|
[hashtable] $psamples
|
|
)
|
|
{
|
|
# aggregate each column across all live sampled nodes
|
|
$this.totalline = $this.linfmt -f $(
|
|
"Total"
|
|
foreach ($col in $this.columns) {
|
|
|
|
# average aggreate doesn't work across nodes if the base is not consistent
|
|
# for instance: cannot average latency safely, but can average cpu utilization
|
|
if ($col.aggregate -ne 'Average' -or $col.aggregate -eq 'AverageAggregate') {
|
|
$(foreach ($node in $psamples.keys) {
|
|
get-samples $psamples $node $col
|
|
}) | get-aggregate $col
|
|
} else {
|
|
$null
|
|
}
|
|
}
|
|
)
|
|
|
|
# individual nodes
|
|
# flags downed/non-responsive nodes by noting which are not
|
|
# present in the processed samples
|
|
$this.nodelines = foreach ($node in $samples.keys | sort) {
|
|
if ($psamples.ContainsKey($node)) {
|
|
$this.linfmt -f $(
|
|
$node
|
|
foreach ($col in $this.columns) {
|
|
$s = get-samples $psamples $node $col
|
|
if ($null -ne $s) {
|
|
$a = $s | get-aggregate $col
|
|
$a
|
|
} else {
|
|
"-"
|
|
}
|
|
}
|
|
)
|
|
} else {
|
|
$this.topfmt -f $(
|
|
$node
|
|
0..($this.columns.count - 1) |% { "-" }
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
[void] Display()
|
|
{
|
|
write-host ($this.topfmt -f (,$this.name + $this.columns.displayname))
|
|
write-host -fore green $this.totalline
|
|
$this.nodelines |% { write-host $_ }
|
|
}
|
|
}
|
|
|
|
function Watch-FleetCluster
|
|
{
|
|
|
|
[CmdletBinding()]
|
|
param(
|
|
[Parameter()]
|
|
[string]
|
|
$Cluster = ".",
|
|
|
|
[Parameter()]
|
|
[int]
|
|
$SampleInterval = 2,
|
|
|
|
[Parameter()]
|
|
[ValidateSet("CSV FS","SSB Cache","SBL","SBL Local","SBL Remote","SBL*","S2D BW","Hyper-V LCPU","SMB SRV","SMB Transport","*")]
|
|
[string[]] $Sets = "CSV FS",
|
|
|
|
[Parameter()]
|
|
[string]
|
|
$LogFile
|
|
)
|
|
|
|
if ($PSBoundParameters.ContainsKey('LogFile'))
|
|
{
|
|
$script:log = $LogFile
|
|
Remove-Item -Force $log -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
function WriteLog(
|
|
[Parameter(ValueFromPipeline = $true)]
|
|
[string[]]
|
|
$String
|
|
)
|
|
{
|
|
PROCESS {
|
|
if ($null -ne $script:log) {
|
|
"$(get-date) $String" | Out-File -Append -FilePath $script:log -Width 9999 -Encoding ascii
|
|
}
|
|
}
|
|
}
|
|
|
|
function get-aggregate(
|
|
[CounterColumn] $col
|
|
)
|
|
{
|
|
BEGIN {
|
|
$n = 0
|
|
$v = 0
|
|
}
|
|
PROCESS {
|
|
$n += 1
|
|
$v += $_
|
|
}
|
|
END {
|
|
if ($n -gt 0) {
|
|
switch -wildcard ($col.aggregate) {
|
|
'Sum' {
|
|
#write-host $col.displayname $col.multipler $v
|
|
$col.multiplier * $v
|
|
}
|
|
'Average*' {
|
|
#write-host $col.displayname $col.multiplier $v $n
|
|
$col.multiplier * $v / $n
|
|
}
|
|
}
|
|
} else {
|
|
$null
|
|
}
|
|
}
|
|
}
|
|
|
|
# get samples out of per-node hash of ctr hashes
|
|
function get-samples(
|
|
[hashtable] $h,
|
|
[string] $node,
|
|
[CounterColumn] $col
|
|
)
|
|
{
|
|
foreach ($i in $col.ctrname) {
|
|
|
|
$k = "$($col.setname)+$($i)"
|
|
|
|
if ($h.ContainsKey($node)) {
|
|
if ($h[$node].ContainsKey($k)) {
|
|
$h[$node][$k]
|
|
} else {
|
|
WriteLog "missing $node[$k] : $($h[$node].Keys.Count) total keys"
|
|
}
|
|
} else {
|
|
WriteLog "missing $node"
|
|
}
|
|
}
|
|
}
|
|
|
|
$allctrs = @()
|
|
|
|
###
|
|
$c = [CounterColumnSet]::new("CSV FS")
|
|
$c.Add([CounterColumn]::new("IOPS", "Cluster CSVFS", @("Reads/sec","Writes/sec"), 12, '#,#', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Reads", "Cluster CSVFS", @("Reads/sec"), 12, '#,#', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Writes", "Cluster CSVFS", @("Writes/sec"), 12, '#,#', 1, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("BW (MB/s)", "Cluster CSVFS", @("Read Bytes/sec","Write Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
|
|
$c.Add([CounterColumn]::new("Read", "Cluster CSVFS", @("Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Read Lat (ms)", "Cluster CSVFS", @("Avg. sec/Read"), 15, '0.000', 1000, 'Average', $true))
|
|
$c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Avg. sec/Write"), 8, '0.000', 1000, 'Average', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Read QAvg", "Cluster CSVFS", @("Avg. Read Queue Length"), 11, '0.000', 1, 'Average', $true))
|
|
$c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Avg. Write Queue Length"), 8, '0.000', 1, 'Average', $false))
|
|
|
|
$c.Seal()
|
|
$allctrs += $c
|
|
|
|
###
|
|
$c = [CounterColumnSet]::new("SSB Cache")
|
|
|
|
$c.Add([CounterColumn]::new("Hit/Sec", "Cluster Storage Hybrid Disks", @("Cache Hit Reads/Sec"), 12, '#,#', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Miss/Sec", "Cluster Storage Hybrid Disks", @("Cache Miss Reads/Sec"), 12, '#,#', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Remap/Sec" ,"Cluster Storage Cache Stores", @("Page ReMap/sec"), 12, '#,#', 1, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Cache (MB/s)", "Cluster Storage Hybrid Disks", @("Cache Populate Bytes/sec","Cache Write Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
|
|
$c.Add([CounterColumn]::new("RdPop", "Cluster Storage Hybrid Disks", @("Cache Populate Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("WrPop", "Cluster Storage Hybrid Disks", @("Cache Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Destage (MB/s)", "Cluster Storage Cache Stores", @("Destage Bytes/sec"), 15, '#,#', 0.000001, 'Sum', $true))
|
|
$c.Add([CounterColumn]::new("Update", "Cluster Storage Cache Stores", @("Update Bytes/sec"), 7, '#,#', 0.000001, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Total (Pgs)", "Cluster Storage Cache Stores", @("Cache Pages"), 11, '0.00E+0', 1, 'Sum', $true))
|
|
$c.Add([CounterColumn]::new("Standby", "Cluster Storage Cache Stores", @("Cache Pages StandBy"), 9, '0.00E+0', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("L0", "Cluster Storage Cache Stores", @("Cache Pages StandBy L0"), 9, '0.00E+0', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("L1", "Cluster Storage Cache Stores", @("Cache Pages StandBy L1"), 9, '0.00E+0', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("L2", "Cluster Storage Cache Stores", @("Cache Pages StandBy L2"), 9, '0.00E+0', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Dirty", "Cluster Storage Cache Stores", @("Cache Pages Dirty"), 9, '0.00E+0', 1, 'Sum', $false))
|
|
|
|
$c.Seal()
|
|
$allctrs += $c
|
|
|
|
###
|
|
|
|
foreach ($subset in '','Local','Remote') {
|
|
$name = 'SBL'
|
|
$prefix = ''
|
|
if ($subset.Length) {
|
|
$name += " $subset"
|
|
$prefix = "$($subset): "
|
|
}
|
|
|
|
$c = [CounterColumnSet]::new($name)
|
|
|
|
$c.Add([CounterColumn]::new("IOPS", "Cluster Disk Counters", @(($prefix + "Read/sec"),($prefix + "Writes/sec")), 12, '#,#', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Reads", "Cluster Disk Counters", @($prefix + "Read/sec"), 12, '#,#', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Writes", "Cluster Disk Counters", @($prefix + "Writes/sec"), 12, '#,#', 1, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("BW (MB/s)", "Cluster Disk Counters", @(($prefix + "Read - Bytes/sec"),($prefix + "Write - Bytes/sec")), 13, '#,#', 0.000001, 'Sum', $true))
|
|
$c.Add([CounterColumn]::new("Read", "Cluster Disk Counters", @($prefix + "Read - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @($prefix + "Write - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Read Lat (ms)", "Cluster Disk Counters", @($prefix + "Read Latency"), 15, '0.000', 1000, 'Average', $true))
|
|
$c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @($prefix + "Write Latency"), 8, '0.000', 1000, 'Average', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Read QAvg", "Cluster Disk Counters", @($prefix + "Read Avg. Queue Length"), 11, '0.000', 1, 'Average', $true))
|
|
$c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @($prefix + "Write Avg. Queue Length"), 8, '0.000', 1, 'Average', $false))
|
|
|
|
$c.Seal()
|
|
$allctrs += $c
|
|
}
|
|
|
|
###
|
|
$c = [CounterColumnSet]::new("SMB SRV")
|
|
|
|
$c.Add([CounterColumn]::new("IOPS", "SMB Server Shares", @("Data Requests/sec"), 12, '#,#', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Reads", "SMB Server Shares", @("Read Requests/sec"), 12, '#,#', 1, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Writes", "SMB Server Shares", @("Write Requests/sec"), 12, '#,#', 1, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Data BW (MB/s)", "SMB Server Shares", @("Data Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
|
|
$c.Add([CounterColumn]::new("Read", "SMB Server Shares", @("Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Write", "SMB Server Shares", @("Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Total BW (MB/s)", "SMB Server Shares", @("Transferred Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
|
|
$c.Add([CounterColumn]::new("Rcv", "SMB Server Shares", @("Received Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Snd", "SMB Server Shares", @("Sent Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
|
|
$c.Seal()
|
|
$allctrs += $c
|
|
|
|
##
|
|
$c = [CounterColumnSet]::new("S2D BW")
|
|
|
|
$c.Add([CounterColumn]::new("CSV (MB/s)", "Cluster CSVFS", @("Read Bytes/sec","Write Bytes/sec"), 10, '#,#', 0.000001, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Read", "Cluster CSVFS", @("Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Write Bytes/sec"), 8 ,'#,#', 0.000001, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("SBL (MB/s)", "Cluster Disk Counters", @("Read - Bytes/sec","Write - Bytes/sec"), 10, '#,#', 0.000001, 'Sum', $true))
|
|
$c.Add([CounterColumn]::new("Read", "Cluster Disk Counters", @("Read - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @("Write - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Cache (MB/s)", "Cluster Storage Hybrid Disks", @("Cache Hit Read Bytes/sec","Cache Write Bytes/sec"), 12, '#,#', 0.000001, 'Sum', $true))
|
|
$c.Add([CounterColumn]::new("Read", "Cluster Storage Hybrid Disks", @("Cache Hit Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Write", "Cluster Storage Hybrid Disks", @("Cache Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Disk (MB/s)", "Cluster Storage Hybrid Disks", @("Disk Read Bytes/sec","Disk Write Bytes/sec"), 11, '#,#', 0.000001, 'Sum', $true))
|
|
$c.Add([CounterColumn]::new("Read", "Cluster Storage Hybrid Disks", @("Disk Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
$c.Add([CounterColumn]::new("Write", "Cluster Storage Hybrid Disks", @("Disk Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
|
|
|
|
$c.Seal()
|
|
$allctrs += $c
|
|
|
|
##
|
|
$c = [CounterColumnSet]::new("Hyper-V LCPU")
|
|
$c.Add([CounterColumn]::new("Logical Total%", "Hyper-V Hypervisor Logical Processor", @("% Total Run Time"), 8, "0.00", 1, 'AverageAggregate', $false))
|
|
$c.Add([CounterColumn]::new("Guest%", "Hyper-V Hypervisor Logical Processor", @("% Guest Run Time"), 8, "0.00", 1, 'AverageAggregate', $false))
|
|
$c.Add([CounterColumn]::new("Hypervisor%", "Hyper-V Hypervisor Logical Processor", @("% Hypervisor Run Time"), 13, "0.00", 1, 'AverageAggregate', $false))
|
|
|
|
$c.Add([CounterColumn]::new("Root Total%", "Hyper-V Hypervisor Root Virtual Processor", @("% Total Run Time"), 12, "0.00", 1, 'AverageAggregate', $true))
|
|
$c.Add([CounterColumn]::new("Guest%", "Hyper-V Hypervisor Root Virtual Processor", @("% Guest Run Time"), 8, "0.00", 1, 'AverageAggregate', $false))
|
|
$c.Add([CounterColumn]::new("Hypervisor%", "Hyper-V Hypervisor Root Virtual Processor", @("% Hypervisor Run Time"), 12, "0.00", 1, 'AverageAggregate', $false))
|
|
$c.Add([CounterColumn]::new("Remote%", "Hyper-V Hypervisor Root Virtual Processor", @("% Remote Run Time"), 7, "0.00", 1, 'AverageAggregate', $false))
|
|
|
|
$c.Seal()
|
|
$allctrs += $c
|
|
|
|
##
|
|
$c = [CounterColumnSet]::new("SMB Transport")
|
|
$c.add([CounterColumn]::new("Read IOPS", "SMB Client Shares", @("Read Requests/sec"), 11, "#,#", 1, 'Sum', $false))
|
|
$c.add([CounterColumn]::new("Write", "SMB Client Shares", @("Write Requests/sec"), 8, "#,#", 1, 'Sum', $false))
|
|
|
|
$c.add([CounterColumn]::new("RDMA Read", "SMB Client Shares", @("Read Requests transmitted via SMB Direct/sec"), 11, "#,#", 1, 'Sum', $true))
|
|
$c.add([CounterColumn]::new("Write", "SMB Client Shares", @("Write Requests transmitted via SMB Direct/sec"), 8, "#,#", 1, 'Sum', $false))
|
|
|
|
$c.Seal()
|
|
$allctrs += $c
|
|
|
|
##
|
|
if ($Sets.Count -eq 1 -and $Sets[0] -eq '*') {
|
|
$ctrs = $allctrs
|
|
} else {
|
|
$ctrs = $Sets |% {
|
|
$s = $_
|
|
$allctrs |? { $_.name -like $s } # allows the SBL* wildcard
|
|
}
|
|
}
|
|
|
|
function start-sample(
|
|
[CounterColumnSet[]] $ctrs,
|
|
[int] $SampleInterval
|
|
)
|
|
{
|
|
# clear any previous job instance
|
|
Get-Job -Name watch-cluster -ErrorAction SilentlyContinue | Stop-Job
|
|
Get-Job -Name watch-cluster -ErrorAction SilentlyContinue | Remove-Job
|
|
|
|
# flatten list of counters and uniquify for the total counter set
|
|
# some display counter sets may repeat specific values (which is fine)
|
|
$counters = ($ctrs.counters |% { $_ |% { $_ }} | group -NoElement).Name
|
|
|
|
icm -AsJob -JobName watch-cluster (Get-ClusterNode -Cluster $Cluster) {
|
|
|
|
# extract countersamples, the object does not survive transfer between powershell sessions
|
|
# extract as a list, not as the individual counters
|
|
get-counter -Continuous -SampleInterval $using:SampleInterval $using:ctrs.counters |% {
|
|
,$_.countersamples
|
|
}
|
|
}
|
|
}
|
|
|
|
# start the first sample job and allow frame draw the first time through
|
|
$j = start-sample $ctrs $SampleInterval
|
|
$downtime = $null
|
|
$skipone = $false
|
|
$loops = 0
|
|
$restart = $false
|
|
|
|
# hash of most recent samples/node
|
|
$samples = @{}
|
|
Get-ClusterNode -Cluster $Cluster |% { $samples[$_.Name] = $null }
|
|
|
|
while ($true) {
|
|
|
|
if (-not $restart) {
|
|
Start-Sleep -Seconds $SampleInterval
|
|
|
|
# sleep again if needed to prime the sample pipeline;
|
|
# there are no samples if we just restarted the sampling jobs
|
|
if ($skipone) {
|
|
$skipone = $false
|
|
continue
|
|
}
|
|
|
|
# receive updates into the per-node hash
|
|
foreach ($child in $j.ChildJobs) {
|
|
$samples[$child.Location] = $child | receive-job -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
# null out downed nodes and remember first time we saw one drop out
|
|
$down = 0
|
|
$j.ChildJobs |? State -ne Running |% {
|
|
$samples[$_.Location] = $null
|
|
$down += 1
|
|
}
|
|
if ($down -and $null -eq $downtime) {
|
|
$downtime = get-date
|
|
}
|
|
|
|
# if everything is down, we will attempt restart
|
|
if ($down -eq $j.ChildJobs.Count) {
|
|
$restart = $true
|
|
break
|
|
}
|
|
}
|
|
|
|
# if explicit restart is required, or it has been 30 seconds with a downed node, restart the jobs to retry
|
|
if ($restart -or ($null -ne $downtime -and ((get-date)-$downtime).totalseconds -gt 30)) {
|
|
$j | stop-job
|
|
$j | remove-job
|
|
$j = start-sample $ctrs $SampleInterval
|
|
|
|
# force gc to clear out accumulated job state quickly
|
|
[system.gc]::Collect()
|
|
|
|
$downtime = $null
|
|
$skipone = $true
|
|
$restart = $false
|
|
continue
|
|
}
|
|
|
|
# now process samples into per-node hashes of set/ctr containing lists of the
|
|
# cooked values acrosss the (possible) multiple instances
|
|
$psamples = @{}
|
|
foreach ($node in $samples.keys) {
|
|
|
|
if ($samples[$node]) {
|
|
|
|
$nsamples = @{}
|
|
|
|
# flatten samples - if we are lagging, we'll have a list
|
|
# of consecutive (increasing by timestamp) samples
|
|
# we could try to be more efficient by dumping all but the
|
|
# final sample, but later ...
|
|
$samples[$node] |% { $_ } |% {
|
|
|
|
($setinst,$ctr) = $($_.path -split '\\')[3..4]
|
|
$set = ($setinst -split '\(')[0]
|
|
|
|
$k = "$set+$ctr"
|
|
$nsamples[$k] = $_.cookedvalue
|
|
}
|
|
|
|
$psamples[$node] = $nsamples
|
|
}
|
|
}
|
|
|
|
# post-process the samples into the counterset, then clear and dump
|
|
$ctrs.DisplayPre($samples, $psamples)
|
|
Clear-Host
|
|
$drawsep = $false
|
|
$ctrs |% {
|
|
if ($drawsep) {
|
|
write-host -fore Green $('-'*20)
|
|
}
|
|
$drawsep = $true
|
|
$_.Display()
|
|
|
|
}
|
|
|
|
# restart the jobs every so many loops to prevent resource growth
|
|
$loops += 1
|
|
if ($loops -gt 100) {
|
|
$loops = 0
|
|
$restart = $true
|
|
}
|
|
}
|
|
} |