Добавлена папка source в CristalDiskMark

CristalDiskMark/source/diskspd22/.gitattributes

@@ -0,0 +1,26 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto
+
+# Use text conventions for commonly used text extensions.
+*.csv text
+*.ini text
+*.json text
+*.txt text
+*.xml text
+*.ps1 text
+*.ps1xml text
+*.psd1 text
+
+
+# Denote all files that are truly binary and should not be modified.
+*.dll binary
+*.exe binary
+*.gz binary
+*.ico binary
+*.jpg binary
+*.lib binary
+*.pdb binary
+*.pdf binary
+*.png binary
+*.wim binary
+*.zip binary

CristalDiskMark/source/diskspd22/.gitignore

@@ -0,0 +1,184 @@
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+
+# User-specific files
+*.suo
+*.user
+*.sln.docstates
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+build/
+bld/
+[Bb]in/
+[Oo]bj/
+
+# Roslyn cache directories
+*.ide/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+#NUNIT
+*.VisualState.xml
+TestResult.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+*_i.c
+*_p.c
+*_i.h
+*.ilk
+*.meta
+*.obj
+*.pch
+*.pdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+.vs/
+ipch/
+*.aps
+*.ncb
+*.opensdf
+*.sdf
+*.cachefile
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# JustCode is a .NET coding addin-in
+.JustCode
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+# TODO: Comment the next line if you want to checkin your web deploy settings
+# but database connection strings (with potential passwords) will be unencrypted
+*.pubxml
+*.publishproj
+
+# NuGet Packages
+*.nupkg
+# The packages folder can be ignored because of Package Restore
+**/packages/*
+# except build/, which is used as an MSBuild target.
+!**/packages/build/
+# If using the old MSBuild-Integrated Package Restore, uncomment this:
+#!**/packages/repositories.config
+
+# Windows Azure Build Output
+csx/
+*.build.csdef
+
+# Windows Store app package directory
+AppPackages/
+
+# Others
+sql/
+*.Cache
+ClientBin/
+[Ss]tyle[Cc]op.*
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.pfx
+*.publishsettings
+node_modules/
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+
+# SQL Server files
+*.mdf
+*.ldf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+
+# Microsoft Fakes
+FakesAssemblies/

CristalDiskMark/source/diskspd22/CmdRequestCreator/CmdRequestCreator.cpp

@@ -0,0 +1,230 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+// CmdRequestCreator.cpp : Defines the entry point for the console application.
+//
+
+#include "CmdRequestCreator.h"
+#include <windows.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "common.h"
+#include "errors.h"
+#include "CmdLineParser.h"
+#include "XmlProfileParser.h"
+#include "IORequestGenerator.h"
+#include "ResultParser.h"
+#include "XmlResultParser.h"
+
+/*****************************************************************************/
+// global variables
+static HANDLE g_hAbortEvent = NULL;     // handle to the 'abort' event
+                                        // it allows stopping I/O Request Generator in the middle of its work
+                                        // the results of its work will be passed to the Results Parser
+static HANDLE g_hEventStarted = NULL;   // event signalled to notify that the actual (measured) test is to be started
+static HANDLE g_hEventFinished = NULL;  // event signalled to notify that the actual test has finished
+
+/*****************************************************************************/
+BOOL WINAPI ctrlCRoutine(DWORD dwCtrlType)
+{
+    if( CTRL_C_EVENT == dwCtrlType )
+    {
+        printf("\n*** Interrupted by Ctrl-C. Stopping I/O Request Generator. ***\n");
+        if( !SetEvent(g_hAbortEvent) )
+        {
+            fprintf(stderr, "Warning: Setting abort event failed (error code: %u)\n", GetLastError());
+        }
+        SetConsoleCtrlHandler(ctrlCRoutine, FALSE);
+
+        //indicate that the signal has been handled
+        return TRUE;
+    }
+    else
+    {
+        return FALSE;
+    }
+}
+
+/*****************************************************************************/
+void TestStarted()
+{
+    if( (NULL != g_hEventStarted) && !SetEvent(g_hEventStarted) )
+    {
+        fprintf(stderr, "Warning: Setting test start notification event failed (error code: %u)\n", GetLastError());
+    }
+}
+
+/*****************************************************************************/
+void TestFinished()
+{
+    if( (NULL != g_hEventFinished) && !SetEvent(g_hEventFinished) )
+    {
+        fprintf(stderr, "Warning: Setting test finish notification event failed (error code: %u)\n", GetLastError());
+    }
+}
+
+/*****************************************************************************/
+/// for CrystalDiskMark
+DWORD pid = 0;
+
+int __cdecl main(int argc, const char* argv[])
+{
+    /// for CrystalDiskMark
+    int totalScore = 0;
+    double averageLatency = 0.0;
+
+    //
+    // parse cmd line parameters
+    //
+    struct Synchronization synch;        //sychronization structure
+    synch.ulStructSize = sizeof(synch);
+    synch.hStopEvent = NULL;
+    synch.hStartEvent = NULL;
+
+    CmdLineParser cmdLineParser;
+    Profile profile;
+    if (!cmdLineParser.ParseCmdLine(argc, argv, &profile, &synch, &g_SystemInformation))
+    {
+        return ERROR_PARSE_CMD_LINE;
+    }
+
+    // Instantiate parsers
+    ResultParser resultParser;
+    XmlResultParser xmlResultParser;
+    IResultParser *pResultParser = nullptr;
+    if (profile.GetResultsFormat() == ResultsFormat::Xml)
+    {
+        pResultParser = &xmlResultParser;
+    }
+    else
+    {
+        pResultParser = &resultParser;
+    }
+
+    // Profile only? If so, complete now.
+    if (profile.GetProfileOnly())
+    {
+        string s = pResultParser->ParseProfile(profile);
+        printf("%s", s.c_str());
+        return 0;
+    }
+
+    synch.pfnCallbackTestStarted = TestStarted;
+    synch.pfnCallbackTestFinished = TestFinished;
+
+    //
+    // create abort event if stop event is not explicitly provided by the user (otherwise use the stop event)
+    //
+
+    if (NULL == synch.hStopEvent)
+    {
+        synch.hStopEvent = CreateEvent(NULL, TRUE, FALSE, NULL);
+        if( NULL == synch.hStopEvent )
+        {
+            fprintf(stderr, "Unable to create an abort event for CTRL+C\n");
+            //FUTURE EXTENSION: change error code
+            return 1;
+        }
+    }
+    g_hAbortEvent = synch.hStopEvent;   // set abort event to either stop event provided by user or the just created event
+
+    //
+    // capture ctrl+c
+    //
+    if( !SetConsoleCtrlHandler(ctrlCRoutine, TRUE) )
+    {
+        fprintf(stderr, "Unable to set CTRL+C routine\n");
+        //FUTURE EXTENSION: change error code
+        return 1;
+    }
+
+    TraceLoggingRegister(g_hEtwProvider);
+
+    //
+    // call IO request generator
+    //
+
+    IORequestGenerator ioGenerator;
+    /// for CrystalDiskMark
+    if (!ioGenerator.GenerateRequests(profile, *pResultParser, &synch, &totalScore, &averageLatency))
+    // if (!ioGenerator.GenerateRequests(profile, *pResultParser, &synch))
+    {
+        if (profile.GetResultsFormat() == ResultsFormat::Xml)
+        {
+            fprintf(stderr, "\n");
+        }
+
+        fprintf(stderr, "Error generating I/O requests\n");
+        return 1;
+    }
+
+    TraceLoggingUnregister(g_hEtwProvider);
+
+    if( NULL != synch.hStartEvent )
+    {
+        CloseHandle(synch.hStartEvent);
+    }
+    if( NULL != synch.hStopEvent )
+    {
+        CloseHandle(synch.hStopEvent);
+    }
+    if( g_hEventStarted )
+    {
+        CloseHandle(g_hEventStarted);
+    }
+    if( NULL != g_hEventFinished )
+    {
+        CloseHandle(g_hEventFinished);
+    }
+
+    /// for CrystalDiskMark
+    CHAR name[32];
+    snprintf(name, 32, "CrystalDiskMark%08X", pid);
+    auto size = 8;
+
+    HANDLE hSharedMemory = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, NULL, size, name);
+    if (hSharedMemory != NULL)
+    {
+        auto pMemory = (double*)MapViewOfFile(hSharedMemory, FILE_MAP_ALL_ACCESS, NULL, NULL, size);
+        if (pMemory != NULL)
+        {
+            *pMemory = averageLatency;
+            UnmapViewOfFile(pMemory);
+            CloseHandle(hSharedMemory);
+        }
+    }
+    printf("Score: %d\n", totalScore);
+    printf("averageLatency: %f\n", averageLatency);
+    //fprintf(stderr, "%f", averageLatency);
+
+    return totalScore;
+
+
+    // return 0;
+}

CristalDiskMark/source/diskspd22/CmdRequestCreator/diskspd.rc

@@ -0,0 +1,19 @@
+#include <windows.h>
+#include "Version.h"
+
+DISKSPD.XSD HTML "..\\XmlProfileParser\\diskspd.xsd"
+
+#include <ntverp.h>
+
+#define VER_FILETYPE                VFT_APP
+#define VER_FILESUBTYPE             VFT2_UNKNOWN
+#define VER_FILEDESCRIPTION_STR     "DiskSpd Storage Performance Tool"
+#define VER_INTERNALNAME_STR        "diskspd.exe"
+
+#undef VER_PRODUCTVERSION
+#define VER_PRODUCTVERSION DISKSPD_MAJOR,DISKSPD_MINOR,DISKSPD_BUILD,DISKSPD_QFE
+
+#undef VER_PRODUCTVERSION_STR
+#define VER_PRODUCTVERSION_STR      DISKSPD_NUMERIC_VERSION_STRING
+
+#include "common.ver"

CristalDiskMark/source/diskspd22/Common/CmdLineParser.h

@@ -0,0 +1,73 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include "Common.h"
+
+namespace UnitTests { class CmdLineParserUnitTests; }
+
+class CmdLineParser
+{
+public:
+    CmdLineParser();
+    ~CmdLineParser();
+
+    bool ParseCmdLine(const int argc, const char *argv[], Profile *pProfile, struct Synchronization *synch, SystemInformation *pSystem = nullptr);
+
+private:
+    bool _ReadParametersFromCmdLine(const int argc, const char *argv[], Profile *pProfile, struct Synchronization *synch, bool& fXMLProfile);
+    bool _ReadParametersFromXmlFile(const char *pszPath, Profile *pProfile, vector<Target> *pvSubstTargets);
+
+    bool _ParseETWParameter(const char *arg, Profile *pProfile);
+    bool _ParseFlushParameter(const char *arg, MemoryMappedIoFlushMode *FlushMode );
+    bool _ParseAffinity(const char *arg, TimeSpan *pTimeSpan);
+    bool _ParseRandomDistribution(const char *arg, vector<Target>& vTargets);
+
+    void _DisplayUsageInfo(const char *pszFilename) const;
+    bool _GetSizeInBytes(const char *pszSize, UINT64& ullSize, const char **pszRest) const;
+    bool _GetRandomDataWriteBufferData(const string& sArg, UINT64& cb, string& sPath);
+
+    static bool _IsSwitchChar(const char c) { return (c == '/' || c == '-'); }
+    enum class ParseState {
+        Unknown,
+        True,
+        False,
+        Bad
+    };
+
+    DWORD _dwBlockSize;         // block size; other parameters may be stated in blocks
+                                // so the block size is needed to process them
+
+    UINT32 _ulWriteRatio;       // default percentage of write requests
+
+    HANDLE _hEventStarted;      // event signalled to notify that the actual (measured) test is to be started
+    HANDLE _hEventFinished;     // event signalled to notify that the actual test has finished
+
+    friend class UnitTests::CmdLineParserUnitTests;
+};

CristalDiskMark/source/diskspd22/Common/CmdRequestCreator.h

@@ -0,0 +1,38 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+// stdafx.h : include file for standard system include files,
+// or project specific include files that are used frequently, but
+// are changed infrequently
+//
+
+#pragma once
+
+#include <tchar.h>
+#include <stdio.h>

CristalDiskMark/source/diskspd22/Common/Histogram.h

@@ -0,0 +1,330 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+
+#include <map>
+#include <unordered_map>
+#include <string>
+#include <sstream>
+#include <limits>
+#include <cmath>
+
+// Plain min/max macros from common headers will interfere with std::numeric_limits
+
+#pragma push_macro("min")
+#pragma push_macro("max")
+#undef min
+#undef max
+
+template<typename T>
+class Histogram
+{
+private:
+
+    mutable std::unordered_map<T, unsigned> _data;
+    mutable unsigned _samples;
+
+    mutable std::map<T, unsigned> _sdata;
+    mutable unsigned _ssamples;
+
+    // Save most recent percentile/iterator/nth-distance query. If the next is strictly >= it allows
+    // an efficient forward iteration through an ascending set of queries.
+
+    mutable double _lastptile;
+    mutable unsigned _lastptilen;
+    mutable decltype(_sdata.cbegin()) _lastptilepos;
+
+    // A histogram starts writable/unsealed and automatically seals after the first read operation.
+    // Subsequent writes which add data restart from empty.
+
+    void _SealData() const
+    {
+        if (!_data.empty())
+        {
+            _sdata.clear();
+
+            _sdata = std::map<T, unsigned>(_data.cbegin(), _data.cend());
+            _ssamples = _samples;
+
+            // invalid ptile > 1; first ptile query will initialize
+            _lastptile = 1.1;
+
+            _data.clear();
+            _samples = 0;
+        }
+    }
+
+public:
+
+    Histogram()
+        : _samples(0),
+        _ssamples(0)
+    {}
+
+    void Clear()
+    {
+        _data.clear();
+        _samples = 0;
+
+        _sdata.clear();
+        _ssamples = 0;
+
+    }
+
+    void Add(T v)
+    {
+        _data[v]++;
+        _samples++;
+    }
+
+    void Merge(const Histogram<T> &other)
+    {
+        for (auto i : other._data)
+        {
+            _data[ i.first ] += i.second;
+        }
+
+        _samples += other._samples;
+    }
+
+    T GetMin() const
+    {
+        _SealData();
+
+        // Default low if empty
+        if (!_ssamples)
+        {
+            return std::numeric_limits<T>::min();
+        }
+
+        return _sdata.cbegin()->first;
+    }
+
+    T GetMax() const
+    {
+        _SealData();
+
+        // Default low if empty
+        if (!_ssamples)
+        {
+            return std::numeric_limits<T>::min();
+        }
+
+        return _sdata.crbegin()->first;
+    }
+
+    unsigned GetSampleBuckets() const
+    {
+        return (unsigned) (_samples ? _data.size() : _sdata.size());
+    }
+
+    unsigned GetSampleSize() const
+    {
+        return _samples ? _samples : _ssamples;
+    }
+
+    T GetPercentile(double p) const
+    {
+        if ((p < 0.0) || (p > 1.0))
+        {
+            throw std::invalid_argument("Percentile must be >= 0 and <= 1");
+        }
+
+        _SealData();
+
+        // Default low if empty
+        if (!_ssamples)
+        {
+            return std::numeric_limits<T>::min();
+        }
+
+        const double target = p * _ssamples;
+
+        // Default to beginning; n is the number of samples iterated over so far
+        unsigned n = 0;
+        auto pos = _sdata.cbegin();
+
+        // Resume from last?
+        if (p >= _lastptile)
+        {
+            n = _lastptilen;
+            pos = _lastptilepos;
+        }
+
+        while (pos != _sdata.cend())
+        {
+            if (n + pos->second >= target)
+            {
+                // Save position. Note the pre-incremented distance through the histogram
+                // must be saved in case next is still in the same bucket.
+                _lastptile = p;
+                _lastptilen = n;
+                _lastptilepos = pos;
+
+                return pos->first;
+            }
+
+            n += pos->second;
+            ++pos;
+        }
+
+        throw std::overflow_error("overran end trying to find percentile");
+    }
+
+    T GetPercentile(int p) const
+    {
+        return GetPercentile(static_cast<double>(p) / 100);
+    }
+
+    T GetMedian() const
+    {
+        return GetPercentile(0.5);
+    }
+
+    double GetStdDev()  const { return GetStandardDeviation(); }
+    double GetAvg()     const { return GetMean(); }
+
+    double GetMean() const
+    {
+        _SealData();
+
+        // Default low if empty
+        if (!_ssamples)
+        {
+            return std::numeric_limits<T>::min();
+        }
+
+        double sum(0);
+
+        for (const auto i : _sdata)
+        {
+            double bucket_val =
+                static_cast<double>(i.first) * i.second / _ssamples;
+
+            if (sum + bucket_val < 0)
+            {
+                throw std::overflow_error("while trying to accumulate sum");
+            }
+
+            sum += bucket_val;
+        }
+
+        return sum;
+    }
+
+    double GetStandardDeviation() const
+    {
+        double mean(GetMean());
+        double ssd(0);
+
+        for (const auto i : _sdata)
+        {
+            double dev = static_cast<double>(i.first) - mean;
+            double sqdev = dev * dev;
+            ssd += i.second * sqdev;
+        }
+
+        return sqrt(ssd / _ssamples);
+    }
+
+    std::string GetHistogramCsv(const unsigned bins) const
+    {
+        return GetHistogramCsv(bins, GetMin(), GetMax());
+    }
+
+    std::string GetHistogramCsv(const unsigned bins, const T LOW, const T HIGH) const
+    {
+        _SealData();
+
+        // ISSUE-REVIEW
+        // Currently bins are defined as strictly less-than
+        // their upper limit, with the exception of the last
+        // bin.  Otherwise where would I put the max value?
+        const double binSize = static_cast<double>((HIGH - LOW) / bins);
+        double limit = static_cast<double>(LOW);
+
+        std::ostringstream os;
+        os.precision(std::numeric_limits<T>::digits10);
+
+        auto pos = _sdata.cbegin();
+        unsigned cumulative = 0;
+
+        for (unsigned bin = 1; bin <= bins; ++bin)
+        {
+            unsigned count = 0;
+            limit += binSize;
+
+            while (pos != _sdata.end() &&
+                (pos->first < limit || bin == bins))
+            {
+                count += pos->second;
+                ++pos;
+            }
+
+            cumulative += count;
+
+            os << limit << "," << count << "," << cumulative << std::endl;
+        }
+
+        return os.str();
+    }
+
+    std::string GetRawCsv() const
+    {
+        _SealData();
+
+        std::ostringstream os;
+        os.precision(std::numeric_limits<T>::digits10);
+
+        for (const auto i : _sdata)
+        {
+            os << i.first << "," << i.second << std::endl;
+        }
+
+        return os.str();
+    }
+
+    std::string GetRaw() const
+    {
+        _SealData();
+
+        std::ostringstream os;
+
+        for (const auto i : _sdata)
+        {
+            os << i.second << " " << i.first << std::endl;
+        }
+
+        return os.str();
+    }
+};
+
+#pragma pop_macro("min")
+#pragma pop_macro("max")

CristalDiskMark/source/diskspd22/Common/IORequestGenerator.h

@@ -0,0 +1,84 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#define INITGUID        //Include this #define to use SystemTraceControlGuid in Evntrace.h.
+#include <Evntrace.h>   //ETW
+#include <Winternl.h>   //ntdll.dll
+
+
+void PrintError(const char *format, ...);
+namespace UnitTests
+{
+    class IORequestGeneratorUnitTests;
+}
+
+class IORequestGenerator
+{
+public:
+    IORequestGenerator() :
+        _hNTDLL(nullptr)
+    {
+
+    }
+
+    //bool GenerateRequests(Profile& profile, IResultParser& resultParser, struct Synchronization *pSynch);
+    /// for CrystalDiskMark
+    bool GenerateRequests(Profile& profile, IResultParser& resultParser, struct Synchronization* pSynch, int* totalScore, double* averageLatency);
+
+private:
+
+    struct CreateFileParameters
+    {
+        string sPath;
+        UINT64 ullFileSize;
+        bool fZeroWriteBuffers;
+    };
+
+    bool _GenerateRequestsForTimeSpan(const Profile& profile, const TimeSpan& timeSpan, Results& results, struct Synchronization *pSynch);
+    void _AbortWorkerThreads(HANDLE hStartEvent, vector<HANDLE>& vhThreads) const;
+    void _CloseOpenFiles(vector<HANDLE>& vhFiles) const;
+    DWORD _CreateDirectoryPath(const char *path) const;
+    bool _CreateFile(UINT64 ullFileSize, const char *pszFilename, bool fZeroBuffers, bool fVerbose) const;
+    bool _GetActiveGroupsAndProcs() const;
+    struct ETWSessionInfo _GetResultETWSession(const EVENT_TRACE_PROPERTIES *pTraceProperties) const;
+    bool _GetSystemPerfInfo(vector<SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION>& vSPPI, bool fVerbose) const;
+    void _InitializeGlobalParameters();
+    bool _LoadDLLs();
+    bool _StopETW(bool fUseETW, TRACEHANDLE hTraceSession) const;
+    void _TerminateWorkerThreads(vector<HANDLE>& vhThreads) const;
+    bool _ValidateProfile(const Profile& profile) const;
+    vector<struct CreateFileParameters> _GetFilesToPrecreate(const Profile& profile) const;
+    void _MarkFilesAsCreated(Profile& profile, const vector<struct CreateFileParameters>& vFiles) const;
+    bool _PrecreateFiles(Profile& profile) const;
+
+    HINSTANCE volatile _hNTDLL;     //handle to ntdll.dll
+
+    friend class UnitTests::IORequestGeneratorUnitTests;
+};

CristalDiskMark/source/diskspd22/Common/IoBucketizer.cpp

@@ -0,0 +1,228 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#include "IoBucketizer.h"
+#include <stdexcept>
+
+/*
+Calculating stddev using an online algorithm:
+avg = sum(1..n, a[n]) / n
+stddev = sqrt(sum(1..n, (a[n] - avg)^2) / n)
+       = sqrt(sum(1..n, a[n]^2 - 2 * a[n] * avg + avg^2) / n)
+       = sqrt((sum(1..n, a[n]^2) - 2 * avg * sum(1..n, a[n]) + n * avg^2) / n)
+       = sqrt((sum(1..n, a[n]^2) - 2 * (sum(1..n, a[n]) / n) * sum(1..n, a[n]) + n * (sum(1..n], a[n]) / n)^2) / n)
+       = sqrt((sum(1..n, a[n]^2) - (2 / n) * sum(1..n, a[n])^2 + (1 / n) * sum(1..n, a[n])^2) / n)
+       = sqrt((sum(1..n, a[n]^2) - (1 / n) * sum(1..n, a[n])^2) / n)
+
+So if we track n, sum(a[n]) and sum(a[n]^2) we can calculate the stddev.  This
+is used to calculate the stddev of the latencies below.
+*/
+
+const unsigned __int64 INVALID_BUCKET_DURATION = 0;
+
+IoBucketizer::IoBucketizer()
+    : _bucketDuration(INVALID_BUCKET_DURATION),
+      _validBuckets(0),
+      _totalBuckets(0)
+{}
+
+void IoBucketizer::Initialize(unsigned __int64 bucketDuration, size_t validBuckets)
+{
+    if (_bucketDuration != INVALID_BUCKET_DURATION)
+    {
+        throw std::runtime_error("IoBucketizer has already been initialized");
+    }
+    if (bucketDuration == INVALID_BUCKET_DURATION)
+    {
+        throw std::invalid_argument("Bucket duration must be a positive integer");
+    }
+
+    _bucketDuration = bucketDuration;
+    _validBuckets = validBuckets;
+    _vBuckets.resize(_validBuckets);
+}
+
+void IoBucketizer::Add(unsigned __int64 ioCompletionTime, double ioDuration)
+{
+    if (_bucketDuration == INVALID_BUCKET_DURATION)
+    {
+        throw std::runtime_error("IoBucketizer has not been initialized");
+    }
+
+    size_t bucketNumber = static_cast<size_t>(ioCompletionTime / _bucketDuration);
+    _totalBuckets = bucketNumber + 1;
+
+    if (bucketNumber >= _validBuckets)
+    {
+        return;
+    }
+    
+    _vBuckets[bucketNumber].lfSumDuration += ioDuration;
+    _vBuckets[bucketNumber].lfSumSqrDuration += ioDuration * ioDuration;
+
+    if (_vBuckets[bucketNumber].ulCount == 0 ||
+        ioDuration < _vBuckets[bucketNumber].lfMinDuration)
+    {
+        _vBuckets[bucketNumber].lfMinDuration = ioDuration;
+    }
+    if (_vBuckets[bucketNumber].ulCount == 0 ||
+        ioDuration > _vBuckets[bucketNumber].lfMaxDuration)
+    {
+        _vBuckets[bucketNumber].lfMaxDuration = ioDuration;
+    }
+
+    _vBuckets[bucketNumber].ulCount++;
+}
+
+size_t IoBucketizer::GetNumberOfValidBuckets() const 
+{
+    return (_totalBuckets > _validBuckets ? _validBuckets : _totalBuckets);
+}
+
+unsigned int IoBucketizer::GetIoBucketCount(size_t bucketNumber) const 
+{
+    if (bucketNumber < _validBuckets)
+    {
+        return _vBuckets[bucketNumber].ulCount;
+    }
+    
+    return 0;
+}
+
+double IoBucketizer::GetIoBucketMinDurationUsec(size_t bucketNumber) const
+{
+    if (bucketNumber < _validBuckets)
+    {
+        return _vBuckets[bucketNumber].lfMinDuration;
+    }
+    
+    return 0;
+}
+
+double IoBucketizer::GetIoBucketMaxDurationUsec(size_t bucketNumber) const
+{
+    if (bucketNumber < _validBuckets)
+    {
+        return _vBuckets[bucketNumber].lfMaxDuration;
+    }
+    
+    return 0;
+}
+
+double IoBucketizer::GetIoBucketAvgDurationUsec(size_t bucketNumber) const
+{
+    if (bucketNumber < _validBuckets && _vBuckets[bucketNumber].ulCount != 0)
+    {
+        return _vBuckets[bucketNumber].lfSumDuration / static_cast<double>(_vBuckets[bucketNumber].ulCount);
+    }
+
+    return 0;
+}
+
+double IoBucketizer::GetIoBucketDurationStdDevUsec(size_t bucketNumber) const
+{
+    if (bucketNumber < _validBuckets && _vBuckets[bucketNumber].ulCount != 0)
+    {
+        double sum_of_squares = _vBuckets[bucketNumber].lfSumSqrDuration;
+        double square_of_sum = _vBuckets[bucketNumber].lfSumDuration * _vBuckets[bucketNumber].lfSumDuration;
+        double count = static_cast<double>(_vBuckets[bucketNumber].ulCount);
+        double square_stddev = (sum_of_squares - (square_of_sum / count)) / count;
+        
+        return sqrt(square_stddev);
+    }
+
+    return 0;
+}
+
+double IoBucketizer::_GetMeanIOPS() const 
+{ 
+    size_t numBuckets = GetNumberOfValidBuckets();
+    double sum = 0;
+
+    for (size_t i = 0; i < numBuckets; i++)
+    {
+        sum += static_cast<double>(_vBuckets[i].ulCount) / numBuckets;
+    }
+
+    return sum;
+}
+
+double IoBucketizer::GetStandardDeviationIOPS() const
+{ 
+    size_t numBuckets = GetNumberOfValidBuckets();
+
+    if(numBuckets == 0) 
+    {
+        return 0.0;
+    }
+
+    double mean = _GetMeanIOPS();
+    double ssd = 0;
+
+    for (size_t i = 0; i < numBuckets; i++)
+    {
+        double dev = static_cast<double>(_vBuckets[i].ulCount) - mean;
+        double sqdev = dev*dev;
+        ssd += sqdev;
+    }
+
+    return sqrt(ssd / numBuckets);
+}
+
+void IoBucketizer::Merge(const IoBucketizer& other) 
+{
+    if(other._vBuckets.size() > _vBuckets.size())
+    {
+        _vBuckets.resize(other._vBuckets.size());
+    }
+    for(size_t i = 0; i < other._vBuckets.size(); i++) 
+    {
+        _vBuckets[i].ulCount += other._vBuckets[i].ulCount;
+        _vBuckets[i].lfSumDuration += other._vBuckets[i].lfSumDuration;
+        _vBuckets[i].lfSumSqrDuration += other._vBuckets[i].lfSumSqrDuration;
+        
+        if (i >= _validBuckets ||
+            other._vBuckets[i].lfMinDuration < _vBuckets[i].lfMinDuration)
+        {
+            _vBuckets[i].lfMinDuration = other._vBuckets[i].lfMinDuration;
+        }
+        if (other._vBuckets[i].lfMaxDuration > _vBuckets[i].lfMaxDuration)
+        {
+            _vBuckets[i].lfMaxDuration = other._vBuckets[i].lfMaxDuration;
+        }
+    }
+    if (other._validBuckets > _validBuckets)
+    {
+        _validBuckets = other._validBuckets;
+    }
+    if (other._totalBuckets > _totalBuckets)
+    {
+        _totalBuckets = other._totalBuckets;
+    }
+}

CristalDiskMark/source/diskspd22/Common/IoBucketizer.h

@@ -0,0 +1,73 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+
+#include <vector>
+
+class IoBucketizer 
+{
+public:
+    IoBucketizer();
+    void Initialize(unsigned __int64 bucketDuration, size_t validBuckets);
+
+    size_t GetNumberOfValidBuckets() const;
+    unsigned int GetIoBucketCount(size_t bucketNumber) const;
+    double GetIoBucketMinDurationUsec(size_t bucketNumber) const;
+    double GetIoBucketMaxDurationUsec(size_t bucketNumber) const;
+    double GetIoBucketAvgDurationUsec(size_t bucketNumber) const;
+    double GetIoBucketDurationStdDevUsec(size_t bucketNumber) const;
+    void Add(unsigned __int64 ioCompletionTime, double ioDuration);
+    double GetStandardDeviationIOPS() const;
+    void Merge(const IoBucketizer& other);
+private:
+    double _GetMeanIOPS() const;
+
+    struct IoBucket {
+        IoBucket() :
+            ulCount(0),
+            lfMinDuration(0),
+            lfMaxDuration(0),
+            lfSumDuration(0),
+            lfSumSqrDuration(0)
+        {
+        }
+        
+        unsigned int ulCount;
+        double lfMinDuration;
+        double lfMaxDuration;
+        double lfSumDuration;
+        double lfSumSqrDuration;
+    };
+
+    unsigned __int64 _bucketDuration;
+    size_t _validBuckets;
+    size_t _totalBuckets;
+    std::vector<IoBucket> _vBuckets;
+};

CristalDiskMark/source/diskspd22/Common/OverlappedQueue.h

@@ -0,0 +1,50 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include <Windows.h>
+
+//
+// OverlappedQueue is a simple class that implements a queue for OVERLAPPED elements
+//
+class OverlappedQueue
+{
+public:
+    OverlappedQueue(void);
+
+    void Add(OVERLAPPED *pOverlapped);
+    bool IsEmpty(void) const;
+    OVERLAPPED * Remove(void);
+    size_t GetCount() const;
+
+private:
+    OVERLAPPED *_pHead;
+    OVERLAPPED *_pTail;
+    size_t _cItems;
+};

CristalDiskMark/source/diskspd22/Common/ResultParser.h

@@ -0,0 +1,76 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include "Common.h"
+
+namespace UnitTests
+{
+    class ResultParserUnitTests;
+}
+
+class ResultParser : public IResultParser
+{
+public:
+    string ParseResults(const Profile& profile, const SystemInformation& system, vector<Results> vResults);
+    string ParseProfile(const Profile& profile);
+
+    /// for CrystalDiskMark
+    int GetTotalScore();
+    double GetAverageLatency();
+
+private:
+    void _DisplayFileSize(UINT64 fsize, UINT32 align = 0);
+    void _DisplayETWSessionInfo(struct ETWSessionInfo sessionInfo);
+    void _DisplayETW(struct ETWMask ETWMask, struct ETWEventCounters EtwEventCounters);
+    void _Print(const char *format, ...);
+    void _PrintProfile(const Profile& profile);
+    void _PrintSystemInfo(const SystemInformation& system);
+    void _PrintCpuUtilization(const Results& results, const SystemInformation& system);
+    enum class _SectionEnum {TOTAL, READ, WRITE};
+    void _PrintSectionFieldNames(const TimeSpan& timeSpan);
+    void _PrintSectionBorderLine(const TimeSpan& timeSpan);
+    void _PrintSection(_SectionEnum, const TimeSpan&, const Results&);
+    void _PrintLatencyPercentiles(const Results&);
+    void _PrintLatencyChart(const Histogram<float>& readLatencyHistogram,
+        const Histogram<float>& writeLatencyHistogram,
+        const Histogram<float>& totalLatencyHistogram);
+    void _PrintTimeSpan(const TimeSpan &timeSpan);
+    void _PrintTarget(const Target &target, bool fUseThreadsPerFile, bool fUseRequestsPerFile, bool fCompletionRoutines);
+    void _PrintDistribution(DistributionType dT, const vector<DistributionRange>& v, char* spc);
+    void _PrintEffectiveDistributions(const Results& results);
+    void _PrintWaitStats(const Results& result);
+
+    string _sResult;
+    /// for CrystalDiskMark
+    int _totalScore;
+    double _averageLatency;
+
+    friend class UnitTests::ResultParserUnitTests;
+};

CristalDiskMark/source/diskspd22/Common/ThroughputMeter.h

@@ -0,0 +1,61 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include <Windows.h>
+
+// ThroughputMeter class assists in metering out throughput over
+// time.  The meter is started by calling Start() with the throughput
+// to be simulated.  GetSleepTime() returns 0 when the next IO can be issued.
+// Adjust() is called to notify the ThroughputMeter about how many bytes were read/written.
+class ThroughputMeter
+{
+public:
+    ThroughputMeter(void);
+
+    bool IsRunning(void) const;
+    void Start(DWORD cBytesPerMillisecond, DWORD dwBlockSize, DWORD dwThinkTime, DWORD dwBurstSize);
+    DWORD GetSleepTime(void) const;
+    void Adjust(size_t cb);
+
+private:
+    DWORD _GetThrottleTime(void) const;
+
+    bool _fRunning;                 // true = throughput monitoring is on
+    bool _fThrottle;                // true = throttling is on
+    bool _fThink;                   // true = think time is enabled
+    ULONGLONG _cbCompleted;         // completed IO
+    DWORD _cbBlockSize;
+    DWORD _cBytesPerMillisecond;    // rate of throttling
+    ULONGLONG _ullStartTimestamp;
+    ULONGLONG _ullDelayUntil;       // timestamp at which the next IO can be executed
+    DWORD _thinkTime;               // time to sleep between burst of IOs
+    DWORD _burstSize;               // number of IOs in a burst. meaningless if think time is zero
+    DWORD _cIO;                     // count of IOs in the current burst
+};

CristalDiskMark/source/diskspd22/Common/Version.h

@@ -0,0 +1,52 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+// versioning material. for simplicity in consumption, please ensure that the date string
+// parses via the System.Datetime constructor as follows (in Powershell):
+//
+//      [datetime] "string"
+//
+// this should result in a valid System.Datetime object, rendered like:
+//
+//      Monday, June 16, 2014 12:00:00 AM
+
+#define DISKSPD_RELEASE_TAG ""
+#define DISKSPD_REVISION    ""
+
+#define DISKSPD_MAJOR       2
+#define DISKSPD_MINOR       2
+#define DISKSPD_BUILD       0
+#define DISKSPD_QFE         0
+
+#define DISKSPD_MAJORMINOR_VER_STR(x,y,z) #x "." #y "." #z
+#define DISKSPD_MAJORMINOR_VERSION_STRING(x,y,z) DISKSPD_MAJORMINOR_VER_STR(x,y,z)
+#define DISKSPD_MAJORMINOR_VERSION_STR DISKSPD_MAJORMINOR_VERSION_STRING(DISKSPD_MAJOR, DISKSPD_MINOR, DISKSPD_BUILD)
+
+#define DISKSPD_NUMERIC_VERSION_STRING DISKSPD_MAJORMINOR_VERSION_STR DISKSPD_REVISION DISKSPD_RELEASE_TAG
+#define DISKSPD_DATE_VERSION_STRING "2024/6/3"

CristalDiskMark/source/diskspd22/Common/XmlProfileParser.h

@@ -0,0 +1,65 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include <MsXml6.h>
+#include "Common.h"
+
+class XmlProfileParser
+{
+public:
+    bool ParseFile(const char *pszPath, Profile *pProfile, vector<Target> *pvSubstTargets, HMODULE hModule);
+
+private:
+    HRESULT _ParseEtw(IXMLDOMDocument2 *pXmlDoc, Profile *pProfile);
+    HRESULT _ParseTimeSpans(IXMLDOMDocument2 *pXmlDoc, Profile *pProfile, vector<pair<string, bool>>& vSubsts);
+    HRESULT _ParseTimeSpan(IXMLDOMNode *pXmlNode, TimeSpan *pTimeSpan, vector<pair<string, bool>>& vSubsts);
+    HRESULT _ParseTargets(IXMLDOMNode *pXmlNode, TimeSpan *pTimeSpan, vector<pair<string, bool>>& vSubsts);
+    HRESULT _ParseRandomDataSource(IXMLDOMNode *pXmlNode, Target *pTarget);
+    HRESULT _ParseWriteBufferContent(IXMLDOMNode *pXmlNode, Target *pTarget);
+    HRESULT _ParseTarget(IXMLDOMNode *pXmlNode, Target *pTarget);
+    HRESULT _ParseThreadTargets(IXMLDOMNode *pXmlNode, Target *pTarget);
+    HRESULT _ParseThroughput(IXMLDOMNode *pXmlNode, Target *pTarget);
+    HRESULT _ParseThreadTarget(IXMLDOMNode *pXmlNode, ThreadTarget *pThreadTarget);
+    HRESULT _ParseAffinityAssignment(IXMLDOMNode *pXmlNode, TimeSpan *pTimeSpan);
+    HRESULT _ParseAffinityGroupAssignment(IXMLDOMNode *pXmlNode, TimeSpan *pTimeSpan);
+    HRESULT _ParseDistribution(IXMLDOMNode *pXmlNode, Target *pTarget);
+    HRESULT _SubstTarget(Target *pTarget, vector<pair<string, bool>>& vSubsts);
+
+    HRESULT _GetString(IXMLDOMNode *pXmlNode, const char *pszQuery, string *psValue) const;
+    HRESULT _GetUINT32(IXMLDOMNode *pXmlNode, const char *pszQuery, UINT32 *pulValue) const;
+    HRESULT _GetUINT64(IXMLDOMNode *pXmlNode, const char *pszQuery, UINT64 *pullValue) const;
+    HRESULT _GetDWORD(IXMLDOMNode *pXmlNode, const char *pszQuery, DWORD *pdwValue) const;
+    HRESULT _GetBool(IXMLDOMNode *pXmlNode, const char *pszQuery, bool *pfValue) const;
+
+    HRESULT _GetUINT32Attr(IXMLDOMNode *pXmlNode, const char *pszAttr, UINT32 *pulValue) const;
+    
+    HRESULT _GetVerbose(IXMLDOMDocument2 *pXmlDoc, bool *pfVerbose);
+    HRESULT _GetProgress(IXMLDOMDocument2 *pXmlDoc, DWORD *pdwProgress);
+};

CristalDiskMark/source/diskspd22/Common/XmlResultParser.h

@@ -0,0 +1,61 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include "Common.h"
+
+class XmlResultParser: public IResultParser
+{
+public:
+    string ParseResults(const Profile& profile, const SystemInformation& system, vector<Results> vResults);
+    string ParseProfile(const Profile& profile);
+    /// for CrystalDiskMark
+    int GetTotalScore();
+	double GetAverageLatency();
+
+private:
+    void _PrintCpuUtilization(const Results& results, const SystemInformation& system);
+    void _PrintETW(struct ETWMask ETWMask, struct ETWEventCounters EtwEventCounters);
+    void _PrintETWSessionInfo(struct ETWSessionInfo sessionInfo);
+    void _PrintLatencyPercentiles(const Results& results);
+    void _PrintTargetResults(const TargetResults& results);
+    void _PrintTargetLatency(const TargetResults& results);
+    void _PrintTargetIops(const IoBucketizer& readBucketizer, const IoBucketizer& writeBucketizer, UINT32 bucketTimeInMs);
+    void _PrintOverallIops(const Results& results, UINT32 bucketTimeInMs);
+    void _PrintIops(const IoBucketizer& readBucketizer, const IoBucketizer& writeBucketizer, UINT32 bucketTimeInMs);
+    void _PrintWaitStats(const ThreadResults& threadResult);
+
+    void _PrintV(const char *format, va_list listArg);
+    void _Print(const char *format, ...);
+    void _PrintInc(const char *format, ...);
+    void _PrintDec(const char *format, ...);
+
+    string _sResult;
+    UINT32 _indent = 0;
+};

CristalDiskMark/source/diskspd22/Common/errors.h

@@ -0,0 +1,38 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#ifndef __DISKSPD_ERRORS_H
+#define __DISKSPD_ERRORS_H
+
+#define ERROR_LOAD_LIBRARY          1   //error during LoadLibrary call
+#define ERROR_GET_PROC_ADDRESS      2   //error during GetProcAddress
+#define ERROR_PARSE_CMD_LINE        3   //error parsing cmd line parameters
+#define ERROR_WAIT_FOR_START_SIGNAL 4   //error waiting for a signal to start work
+
+#endif

CristalDiskMark/source/diskspd22/Common/etw.h

@@ -0,0 +1,41 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+
+
+#include <windows.h>
+#include <Wmistr.h>		///WNODE_HEADER
+#define INITGUID		//Include this #define to use SystemTraceControlGuid in Evntrace.h.
+#include <Evntrace.h>	//ETW
+#include "Common.h"
+
+BOOL TraceEvents();
+TRACEHANDLE StartETWSession(const Profile& profile);
+PEVENT_TRACE_PROPERTIES StopETWSession(TRACEHANDLE hTraceSession);

CristalDiskMark/source/diskspd22/Frameworks/VMFleet/ArcVM-Setup.md

@@ -0,0 +1,329 @@
+# From Setup to Start-FleetSweep for Arc Enabled Virtual Machines in HCI clusters
+
+This is the traditional path of setting up VMFleet to deploy Arc enabled VMs on HCI clusters and running it using your desired DiskSpd parameters/flags.
+
+ 
+
+## Prerequisites
+
+Before we begin setting up VMFleet, there are a few prerequisites that you should have ready.
+
+Ensure an HCI cluster is setup.
+
+Ensure that you have 1 CSV per node
+
+Within Storage Spaces Direct, CPU usage is based on the host. Therefore, it is recommended that you split the storage load by creating as many CSVs as there are host nodes. We can go ahead and create a CSV per node in the cluster.
+
+You may run the following:
+ 
+
+```
+Get-ClusterNode |% {
+    New-Volume -StoragePoolFriendlyName SU1_Pool* -FriendlyName $_ -FileSystem CSVFS_ReFS -Size <DESIRED SIZE>
+}
+
+``` 
+
+Ensure that you create a “collect” volume.
+
+You may run the following:
+ 
+```
+
+New-Volume -StoragePoolFriendlyName SU1_Pool* -FriendlyName collect -FileSystem CSVFS_ReFS -Size 200GB
+
+``` 
+
+If you have ran VMFleet in the past, please ensure that prior VMFleet directories are completely removed from existing volumes.
+
+Retrieve or install a Server Core VHDX file. If you do not have one handy, we can create a new one by following the below instructions.
+
+Download WS2019 Server Core ISO from the public website.
+
+Open Hyper-V Manager.
+
+Click “New”, then “Virtual Machine”.
+
+Navigate through the prompts and pick a location to store your VM.
+
+Once your VM is created, boot up the VM and follow the instructions. This is where you will decide your VM password, or what we will later call, “adminpass”.
+
+This is important as we will use this later, so make sure you write this down.
+
+Log out of the VM, and navigate to where you stored your VM. You should find a “Virtual Hard Disks” folder. Inside, you should find your new Server Core VHDX file.
+
+Rename it to “Base1.vhdx”.
+
+Copy or move the file to the cluster environment that you want to run VMFleet in.
+
+Done!
+
+### Deployment
+
+1. First, we need to install the new PowerShell Module from the PowerShell Gallery and then load it into our terminal. We also need to disable cache as it is not supported currently for ArcVMs. Run the following:
+ 
+```
+Install-Module -Name “VMFleet”
+Import-Module VMFleet
+Set-ClusterStorageSpacesDirect -CacheState Disabled; 
+ 
+```
+
+
+2. Sanity Check:
+    Run "Get-Module VMFleet" to confirm the module exists.
+
+    Run "Get-Command -Module VMFleet" to obtain a list of functions included in the module.
+
+    We will now set up the directory structure within the “Collect” CSV created earlier. Run "Install-Fleet"
+
+    This creates the necessary VMFleet directories which include:
+
+       * Collect/control
+
+            Contains arc.json which stores Arc configuration and the scripts that the Virtual Machines continuously monitor.
+
+       * Control.ps1: the control script the VMs use to implement the control loop (what used to be called “master.ps1”).
+
+       * Run.ps1: The VMs continuously look for the most recent version of run.ps1 and runs the newly updated script (parameters).
+
+       * Collect/flag
+
+            Location where the control script drops the “go”, “pause”, and “done” flag files. Users should not need to look at these files.
+
+       * Collect/result
+
+            Location of the output files from the VMFleet test run.
+
+       * Collect/tools
+
+            DiskSpd will be preinstalled in this folder.
+
+3. You need to create a new or use existing Resource group under the same subscription as the Resource bridge VM is under. Set-ArcConfig will take care of creating one if not already present.
+
+4. Setup configuration required for creating Arc enabled Virtual Machines.
+
+```
+Set-ArcConfig -ResourceGroup [ENTER_RESOURCE_GROUP] -AzureRegistrationUser [ENTER_AZURE_REGUSER] -AzureRegistrationPassword [ENTER_AZURE_REGPASS]-StoragePathCsv [ENTER_CSV_PATH] -Enabled $true -StoragePathName [ENTER_StoragePath_Name] -ImageName [ENTER_Image_Name]  -ResetSalt
+```
+
+    -"ResourceGroup" is the resource group where ARC VMs will be deployed.
+
+    -"AzureRegistrationUser" and "AzureRegistrationPassword" are the azure account credentials.
+
+    -"StoragePathCsv" (optional) is the csv path where storage path resource will be created. If not provided, one of the existing csvs which were created earlier will be used by default.
+
+    -"StoragePathName" (optional) is the storage path name which will be used to create gallery image. If not provided, default name will be used.
+
+    -"ImageName" (optional) is the gallery image name which will be used to create Arc-enabled virtual machines. If not provided, default name will be used. Currently, only windows gallery image is the supported image type for Arc-Enabled VMs.
+
+    -"Enabled" (optional) is set to $false by default. Set to $true if Arc-enabled virtual machines are to be created.
+
+    -"ResetSalt" (optional) is a flag used to reset salt. Salt is a random 4 character (alphanumeric) used in Arc resource names for arc enabled virtual machines (for eg: vm-group-node-salt-001). In case, user wants to regenerate the salt, they can run "Set-ArcConfig -ResetSalt" which will override existing salt with a new one in the arc.json and this will be used to create new VMs. This is useful in case of multiple clusters under same subscription using same resource group on a virtual setup. A 4 character (alphanumeric) Salt will be generated by default when user runs Set-ArcConfig the first time and stored in arc.json along with other arc configs. Within Set-ArcConfig, a quick test is done to check if atleast one vm exists with same salt in the resource group. If it does, it is regenerated.
+
+5. By default, VMs with 2GB static memory and 1 processor count will be created.
+
+    Note: Please move the VHDX file into the collect folder. CSV Cache is also turned off by default.
+
+    We will now create our “fleet” of VMs by running:
+    
+```
+New-Fleet -basevhd <PATH TO VHDX> -vms [ENTER_NUM_VMS] -adminpass [ENTER_ADMINPASS] -connectuser [ENTER_NODE_USER] -connectpass [ENTER_NODE_PASS]
+``` 
+
+    -"adminpass" is the administrator password for the Server Core Image. This is the password you set on your Virtual Machine earlier.
+
+    -"connectuser" is a domain account with access to the cluster.
+
+    -"connectpass " is the password for the above domain account.
+
+    -"vms" is the number of VM's to create per node.
+
+6. If "vms" parameter is not provided, the default is a 1:1 subscription ratio where the Number of VMs = Number of physical cores.
+
+[Optional] You can consider modifying the VM hardware configuration. Run
+ 
+```
+Set-Fleet -ProcessorCount 1 -MemoryStartupBytes 2048mb -MemoryMaximumBytes 2048mb -MemoryMinimumBytes 2048mb
+ ```
+
+Note:
+
+If you specify “MemoryMaximumBytes”, you must specify “MemoryMinimumBytes”, which implies that your VMs will have dynamic memory.
+
+If you omit “MemoryMaximumBytes” or “MemoryMinimumBytes”, it implies that your VMs will have static memory.
+
+If MemoryStartupBytes = MemoryMinimumBytes = MemoryMaximumBytes, that also denotes static memory.
+
+“MemoryStartupBytes” is a mandatory parameter.
+
+
+### Start Running VMFleet!
+
+7. Open 2 PowerShell terminals. In the first one, run Watch-Cluster and in the second one, run Start-Fleet. This second function will turn on all the VMs in a “paused” state.
+
+8. At this point you can run Start-FleetSweep [ENTER_PARAMETERS] or take this time to explore and run any of the other functions!
+
+Here is a sample sweep command to help you get started: "Start-FleetSweep -b 4 -t 8 -o 8 -w 0 -d 300 -p r"
+
+9. Done!
+
+### Aftermath
+
+Once you are done running VMFleet you can run Stop-Fleet to shut down all the virtual machines or run Remove-Fleet to completely delete all the virtual machines on your environment.
+
+# From Setup to Measure-FleetCoreWorkload
+
+This is a new workflow for setting up VMFleet and the predefined profile workloads (General, Peak, VDI, SQL).
+
+ 
+
+## Prerequisites
+
+Before we begin setting up VMFleet, there are a few prerequisites that you should have ready.
+
+Ensure an HCI cluster is setup.
+
+Ensure that you have 1 CSV per node
+
+Within Storage Spaces Direct, CPU usage is based on the host. Therefore, it is recommended that you split the storage load by creating as many CSVs as there are host nodes. We can go ahead and create a CSV per node in the cluster.
+
+In order to be precise about the CSV size, please use our new VMFleet command: "Get-FleetVolumeEstimate"
+
+This will output a prescribed CSV size based on different resiliency types. We recommend you select a 2-way mirrored value or 3-way mirrored value depending on your node count.
+
+1. You may run the following: (use the CSV size from Get-FleetVolumeEstimate)
+ 
+```
+Get-ClusterNode |% {
+    New-Volume -StoragePoolFriendlyName SU1_Pool* -FriendlyName $_ -FileSystem CSVFS_ReFS -Size <DESIRED SIZE>
+}
+```
+
+    Ensure that you create a “collect” volume.
+
+2. You may run the following:
+ 
+```
+New-Volume -StoragePoolFriendlyName SU1_Pool* -FriendlyName collect -FileSystem CSVFS_ReFS -Size 200GB
+``` 
+
+3. If you have ran VMFleet in the past, please ensure that prior VMFleet directories are completely removed from existing volumes.
+
+    Retrieve or install a Server Core VHDX file. If you do not have one handy, we can create a new one by following the below instructions.
+
+    Download WS2019 Server Core ISO from the public website.
+
+    Open Hyper-V Manager.
+
+    Click “New”, then “Virtual Machine”.
+
+    Navigate through the prompts and pick a location to store your VM.
+
+    Once your VM is created, boot up the VM and follow the instructions. This is where you will decide your VM password, or what we will later call, “adminpass”.
+
+    This is important as we will use this later, so make sure you write this down.
+
+    Log out of the VM, and navigate to where you stored your VM. You should find a “Virtual Hard Disks” folder. Inside, you should find your new Server Core VHDX file.
+    Rename it to “Base1.vhdx”.
+
+    Copy or move the file to the cluster environment that you want to run VMFleet in.
+
+4. Done!
+
+## Deployment
+5. Let’s begin deploying VMFleet. First, we need to install the new PowerShell Module from the PowerShell Gallery and then load it into the terminal. Run the following:
+ 
+```
+Install-Module -Name “VMFleet”
+Import-Module VMFleet
+Set-ClusterStorageSpacesDirect -CacheState Disabled; 
+```
+
+ 
+
+## Sanity Check:
+
+6. Run "Get-Module VMFleet" to confirm the module exists.
+
+7. Run "Get-Command -Module VMFleet" to obtain a list of commands included in the module.
+
+8. We will now set up the directory structure within the “Collect” CSV that we created earlier. Run "Install-Fleet"
+
+9. You need to create a new or use existing Resource group under the same subscription as the Resource bridge VM is under. Set-ArcConfig will take care of creating one if not already present.
+
+10. Setup configuration required for creating Arc enabled Virtual Machines.
+
+```
+Set-ArcConfig -ResourceGroup [ENTER_RESOURCE_GROUP] -AzureRegistrationUser [ENTER_AZURE_REGUSER] -AzureRegistrationPassword [ENTER_AZURE_REGPASS]-StoragePathCsv [ENTER_CSV_PATH] -Enabled $true -StoragePathName [ENTER_StoragePath_Name] -ImageName [ENTER_Image_Name]  -ResetSalt
+```
+
+    -"ResourceGroup" is the resource group where ARC VMs will be deployed.
+
+    -"AzureRegistrationUser" and "AzureRegistrationPassword" are the azure account credentials.
+
+    -"StoragePathCsv" (optional) is the csv path where storage path resource will be created. If not provided, one of the existing csvs which were created earlier will be used by default.
+
+    -"StoragePathName" (optional) is the storage path name which will be used to create gallery image. If not provided, default name will be used.
+
+    -"ImageName" (optional) is the gallery image name which will be used to create Arc-enabled virtual machines. If not provided, default name will be used.
+
+    -"Enabled" (optional) is set to $false by default. Set to $true if Arc-enabled virtual machines are to be created.
+
+    -"ResetSalt" (optional) is a flag used to reset salt. Salt is a random 4 character (alphanumeric) used in Arc resource names for arc enabled virtual machines (for eg: vm-group-node-salt-001). In case, user wants to regenerate the salt, they can run "Set-ArcConfig -ResetSalt" which will override existing salt with a new one in the arc.json and this will be used to create new VMs. This is useful in case of multiple clusters under same subscription using same resource group on a virtual setup. A 4 character (alphanumeric) Salt will be generated by default when user runs Set-ArcConfig the first time and stored in arc.json along with other arc configs. Within Set-ArcConfig, a quick test is done to check if atleast one vm exists with same salt in the resource group. If it does, it is regenerated.
+
+11. By default, VMs with 2GB static memory and 1 processor count will be created.
+
+    Note: Please move the VHDX file into the collect folder. CSV Cache is also turned off by default.
+
+    We will now create our “fleet” of VMs by running:
+    
+```
+New-Fleet -basevhd <PATH TO VHDX> -vms [ENTER_NUM_VMS] -adminpass [ENTER_ADMINPASS] -connectuser [ENTER_NODE_USER] -connectpass [ENTER_NODE_PASS]
+``` 
+
+    -"adminpass" is the administrator password for the Server Core Image. This is the password you set on your Virtual Machine earlier.
+
+    -"connectuser" is a domain account with access to the cluster.
+
+    -"connectpass " is the password for the above domain account.
+
+    -"vms" is the number of VM's to create per node.
+
+
+12. Measure-FleetCoreWorkload also collects diagnostic data (Get-SDDCDiagnosticInfo). Therefore, before running the command, we must also install the NuGet Package if you have not previously done so. In doing so, we also need to temporairly set the PSGallery as a trusted repository source (Note: This will temporarily relax the security boundary). 
+
+ 
+```
+$repo = Get-PSRepository -Name PSGallery
+if ($null -eq $repo) { Write-Host "The PSGallery is not configured on this system, please address this before continuing" }
+else {
+    if ($repo.InstallationPolicy -ne 'Trusted') {
+        Write-Host "Setting the PSGallery repository to Trusted, original InstallationPolicy: $($repo.InstallationPolicy)"
+        Set-PSRepository -Name PSGallery -InstallationPolicy Trusted
+    } 
+
+    ### Installing the pre-requisite modules
+
+    Install-PackageProvider NuGet -Force
+    Install-Module -Name PrivateCloud.DiagnosticInfo -Force
+    Install-Module -Name MSFT.Network.Diag -Force
+
+    if ($repo.InstallationPolicy -ne 'Trusted') {
+        Write-Host "Resetting the PSGallery repository to $($repo.InstallationPolicy)"
+        Set-PSRepository -Name PSGallery -InstallationPolicy $repo.InstallationPolicy
+    } 
+}  
+``` 
+
+## Run Measure-FleetCoreWorkload!
+13. We can now run Measure-FleetCoreWorkload. Running the command below will automatically run all 4 workloads (General, Peak, VDI, SQL) and place the individual outputs in the result directory. IMPORTANT: If you plan on running another test, please clear the result directory.
+
+ 
+```
+Measure-FleetCoreWorkload
+``` 
+
+14. Congratulations! You’re done! All you need to do is wait for the test to complete.
+
+Note: If you ever run into an error and need to rerun Measure-FleetCoreWorkload, don’t be afraid to do so! It is smart enough to pick up from where it last stopped and continue the test without starting from scratch.

CristalDiskMark/source/diskspd22/Frameworks/VMFleet/VMFleet.Tests.ps1

@@ -0,0 +1,728 @@
+Describe "TestModulePresent" {
+
+    BeforeAll {
+
+        $script:e = $null
+    }
+
+    if (Test-Path .\VMFleet.psd1)
+    {
+        It "ShouldLoad-Local" {
+            { Import-Module .\VMFleet.psd1 -Force -ErrorVariable script:e } | Should Not Throw
+            $script:e | Should BeNullOrEmpty Because "this should be successful"
+        }
+    }
+    else
+    {
+        It "ModuleExists-Installed" {
+            Get-Module VMFleet -ListAvailable -ErrorVariable script:e | Should Not BeNullOrEmpty Because "module should be installed on system"
+            $script:e | Should BeNullOrEmpty Because "this should be successful"
+        }
+
+        It "ShouldLoad-Installed" {
+            { Import-Module VMFleet -Force -ErrorVariable script:e } | Should Not Throw
+            $script:e | Should BeNullOrEmpty Because "this should be successful"
+        }
+    }
+
+    It "ShouldBeLoaded" {
+        Get-Module VMFleet -ErrorVariable script:e | Should Not BeNullOrEmpty Because "module should now be loaded"
+        $script:e | Should BeNullOrEmpty Because "this should be successful"
+    }
+}
+
+InModuleScope VMFleet {
+
+    Describe "GetDistributedShift" {
+
+        BeforeAll {
+
+            $a = @('a', 'b', 'c', 'd', 'e')
+            $b = @('f', 'g', 'h', 'i', 'j')
+            $c = @('k', 'l', 'm', 'n', 'o')
+            $d = @('p', 'q', 'r', 's', 't')
+
+            $bad = @('z')
+
+            $a20 = 1..20 |% { "a$_" }
+            $b20 = 1..20 |% { "b$_" }
+            $c20 = 1..20 |% { "c$_" }
+            $d20 = 1..20 |% { "d$_" }
+        }
+
+        It "ShouldRequireListOfLists" {
+            GetDistributedShift -Group $a -N 1 | Should Throw
+        }
+
+        It "ShouldRequireMoreThanOne" {
+            GetDistributedShift -Group @($a) -N 1 | Should Throw
+        }
+
+        It "ShouldRequireAllSameSize" {
+            GetDistributedShift -Group $a,$b,$bad -N 1 | Should Throw
+        }
+
+        It "ShouldRequirePositiveRotation" {
+            GetDistributedShift -Group $a,$b -N 0 | Should Throw
+        }
+
+        It "ShouldNotShiftTooMany" {
+            GetDistributedShift -Group $a,$b -N 6 | Should Throw
+        }
+
+        #
+        # Two group rotations
+        #
+
+        It "Rotate2-1" {
+            { GetDistributedShift -Group $a,$b -N 1 } | Should Not Throw
+            $a1, $b1 = GetDistributedShift -Group $a,$b -N 1
+            $a1.Count | Should Be 5
+            $b1.Count | Should Be 5
+
+            ($a1 -join '') | Should Be "abcdj"
+            ($b1 -join '') | Should Be "fghie"
+        }
+
+        It "Rotate2-2" {
+            { GetDistributedShift -Group $a,$b -N 2 } | Should Not Throw
+            $a1, $b1 = GetDistributedShift -Group $a,$b -N 2
+            $a1.Count | Should Be 5
+            $b1.Count | Should Be 5
+
+            ($a1 -join '') | Should Be "abcij"
+            ($b1 -join '') | Should Be "fghde"
+        }
+
+        It "Rotate2-3" {
+            { GetDistributedShift -Group $a,$b -N 3 } | Should Not Throw
+            $a1, $b1 = GetDistributedShift -Group $a,$b -N 3
+            $a1.Count | Should Be 5
+            $b1.Count | Should Be 5
+
+            ($a1 -join '') | Should Be "abhij"
+            ($b1 -join '') | Should Be "fgcde"
+        }
+
+        It "Rotate2-4" {
+            { GetDistributedShift -Group $a,$b -N 4 } | Should Not Throw
+            $a1, $b1 = GetDistributedShift -Group $a,$b -N 4
+            $a1.Count | Should Be 5
+            $b1.Count | Should Be 5
+
+            ($a1 -join '') | Should Be "aghij"
+            ($b1 -join '') | Should Be "fbcde"
+        }
+
+        It "Rotate2-5" {
+            { GetDistributedShift -Group $a,$b -N 5 } | Should Not Throw
+            $a1, $b1 = GetDistributedShift -Group $a,$b -N 5
+            $a1.Count | Should Be 5
+            $b1.Count | Should Be 5
+
+            ($a1 -join '') | Should Be "fghij"
+            ($b1 -join '') | Should Be "abcde"
+        }
+
+        #
+        # Three group rotations
+        #
+
+        It "Rotate3-1" {
+            { GetDistributedShift -Group $a,$b,$c -N 1 } | Should Not Throw
+            $a1, $b1, $c1 = GetDistributedShift -Group $a,$b,$c -N 1
+            $a1.Count | Should Be 5
+            $b1.Count | Should Be 5
+            $c1.Count | Should Be 5
+
+            ($a1 -join '') | Should Be "abcdo"
+            ($b1 -join '') | Should Be "fghie"
+            ($c1 -join '') | Should Be "klmnj"
+        }
+
+        It "Rotate3-2" {
+            { GetDistributedShift -Group $a,$b,$c -N 2 } | Should Not Throw
+            $a1, $b1, $c1 = GetDistributedShift -Group $a,$b,$c -N 2
+            $a1.Count | Should Be 5
+            $b1.Count | Should Be 5
+            $c1.Count | Should Be 5
+
+            ($a1 -join '') | Should Be "abcio"
+            ($b1 -join '') | Should Be "fghne"
+            ($c1 -join '') | Should Be "klmdj"
+        }
+
+        It "Rotate3-3" {
+            { GetDistributedShift -Group $a,$b,$c -N 3 } | Should Not Throw
+            $a1, $b1, $c1 = GetDistributedShift -Group $a,$b,$c -N 3
+            $a1.Count | Should Be 5
+            $b1.Count | Should Be 5
+            $c1.Count | Should Be 5
+
+            ($a1 -join '') | Should Be "abmio"
+            ($b1 -join '') | Should Be "fgcne"
+            ($c1 -join '') | Should Be "klhdj"
+        }
+
+        It "Rotate3-4" {
+            { GetDistributedShift -Group $a,$b,$c -N 4 } | Should Not Throw
+            $a1, $b1, $c1 = GetDistributedShift -Group $a,$b,$c -N 4
+            $a1.Count | Should Be 5
+            $b1.Count | Should Be 5
+            $c1.Count | Should Be 5
+
+            ($a1 -join '') | Should Be "agmio"
+            ($b1 -join '') | Should Be "flcne"
+            ($c1 -join '') | Should Be "kbhdj"
+        }
+
+        It "Rotate3-5" {
+            { GetDistributedShift -Group $a,$b,$c -N 5 } | Should Not Throw
+            $a1, $b1, $c1 = GetDistributedShift -Group $a,$b,$c -N 5
+            $a1.Count | Should Be 5
+            $b1.Count | Should Be 5
+            $c1.Count | Should Be 5
+
+            ($a1 -join '') | Should Be "kgmio"
+            ($b1 -join '') | Should Be "alcne"
+            ($c1 -join '') | Should Be "fbhdj"
+        }
+
+        It "RotateLarge4-1" {
+            { GetDistributedShift -Group $a20,$b20,$c20,$d20 -N 1 } | Should Not Throw
+            $g = GetDistributedShift -Group $a20,$b20,$c20,$d20 -N 1
+
+            # note last group rotates 1
+            $g[0] -join '' | Should Be "a1a2a3a4a5a6a7a8a9a10a11a12a13a14a15a16a17a18a19d20"
+            $g[1] -join '' | Should Be "b1b2b3b4b5b6b7b8b9b10b11b12b13b14b15b16b17b18b19a20"
+            $g[2] -join '' | Should Be "c1c2c3c4c5c6c7c8c9c10c11c12c13c14c15c16c17c18c19b20"
+            $g[3] -join '' | Should Be "d1d2d3d4d5d6d7d8d9d10d11d12d13d14d15d16d17d18d19c20"
+        }
+
+        It "RotateLarge4-2" {
+            { GetDistributedShift -Group $a20,$b20,$c20,$d20 -N 2 } | Should Not Throw
+            $g = GetDistributedShift -Group $a20,$b20,$c20,$d20 -N 2
+
+            # note last group rotates 1, next 2
+            $g[0] -join '' | Should Be "a1a2a3a4a5a6a7a8a9a10a11a12a13a14a15a16a17a18c19d20"
+            $g[1] -join '' | Should Be "b1b2b3b4b5b6b7b8b9b10b11b12b13b14b15b16b17b18d19a20"
+            $g[2] -join '' | Should Be "c1c2c3c4c5c6c7c8c9c10c11c12c13c14c15c16c17c18a19b20"
+            $g[3] -join '' | Should Be "d1d2d3d4d5d6d7d8d9d10d11d12d13d14d15d16d17d18b19c20"
+        }
+
+        It "RotateLarge4-3" {
+            { GetDistributedShift -Group $a20,$b20,$c20,$d20 -N 3 } | Should Not Throw
+            $g = GetDistributedShift -Group $a20,$b20,$c20,$d20 -N 3
+
+            # note last group rotates 1, then 2, 3
+            $g[0] -join '' | Should Be "a1a2a3a4a5a6a7a8a9a10a11a12a13a14a15a16a17b18c19d20"
+            $g[1] -join '' | Should Be "b1b2b3b4b5b6b7b8b9b10b11b12b13b14b15b16b17c18d19a20"
+            $g[2] -join '' | Should Be "c1c2c3c4c5c6c7c8c9c10c11c12c13c14c15c16c17d18a19b20"
+            $g[3] -join '' | Should Be "d1d2d3d4d5d6d7d8d9d10d11d12d13d14d15d16d17a18b19c20"
+        }
+
+        It "RotateLarge4-4" {
+            { GetDistributedShift -Group $a20,$b20,$c20,$d20 -N 4 } | Should Not Throw
+            $g = GetDistributedShift -Group $a20,$b20,$c20,$d20 -N 4
+
+            # note last group rotates 1, then 2, 3, and back to 1
+            $g[0] -join '' | Should Be "a1a2a3a4a5a6a7a8a9a10a11a12a13a14a15a16d17b18c19d20"
+            $g[1] -join '' | Should Be "b1b2b3b4b5b6b7b8b9b10b11b12b13b14b15b16a17c18d19a20"
+            $g[2] -join '' | Should Be "c1c2c3c4c5c6c7c8c9c10c11c12c13c14c15c16b17d18a19b20"
+            $g[3] -join '' | Should Be "d1d2d3d4d5d6d7d8d9d10d11d12d13d14d15d16c17a18b19c20"
+        }
+    }
+
+    Describe "FilterObject" {
+
+        BeforeAll {
+            $o12 = [PSCustomObject]@{
+                K1 = 1
+                K2 = 2
+            }
+            $o13 = [PSCustomObject]@{
+                K1 = 1
+                K2 = 3
+            }
+        }
+
+        It "PassWithEmpty" {
+            $o12 | FilterObject -Filter @{} | Should Not BeNullOrEmpty
+        }
+        It "PassWithOneK" {
+            $o12 | FilterObject -Filter @{ K1 = 1 } | Should Not BeNullOrEmpty
+        }
+        It "PassWithTwoK" {
+            $o12 | FilterObject -Filter @{ K1 = 1; K2 = 2 } | Should Not BeNullOrEmpty
+        }
+        It "PassWithOneKDiffType" {
+            $o12 | FilterObject -Filter @{ K1 = '1' } | Should Not BeNullOrEmpty
+        }
+        It "PassWithTwoKDiffType" {
+            $o12 | FilterObject -Filter @{ K1 = '1'; K2 = '2' } | Should Not BeNullOrEmpty
+        }
+        It "NotPassMismatchSameType" {
+            $o12 | FilterObject -Filter @{ K1 = 2 } | Should BeNullOrEmpty
+        }
+        It "NotPassMismatchDiffType" {
+            $o12 | FilterObject -Filter @{ K1 = '2' } | Should BeNullOrEmpty
+        }
+
+        It "ShouldPassMultipleMatch" {
+            $r = @($o12,$o12 | FilterObject -Filter @{ K1 = 1 })
+            $r.Count | Should Be 2
+        }
+
+        It "ShouldPassTheMatch" {
+            $r = @($o12,$o13 | FilterObject -Filter @{ K2 = 2 })
+            $r.Count | Should Be 1
+            $r.K1 | Should Be 1
+            $r.K2 | Should Be 2
+        }
+    }
+}
+
+Describe "SetFleetProfile" {
+
+    BeforeAll {
+
+        It "ShouldBeExpectedSqlProfile" {
+            { Get-FleetprofileXml -Name SQL } | Should Not Throw
+            $x = Get-FleetprofileXml -Name SQL
+
+            $x.Profile.TimeSpans.TimeSpan.Targets.Target[0].Throughput.InnerText | Should Be 1500
+            $x.Profile.TimeSpans.TimeSpan.Targets.Target[1].Throughput.InnerText | Should Be 300
+        }
+
+        $x = Get-FleetprofileXml -Name SQL
+
+        $oWarmup = $x.Profile.TimeSpans.TimeSpan.Warmup
+        $oDuration = $x.Profile.TimeSpans.TimeSpan.Duration
+        $oCooldown = $x.Profile.TimeSpans.TimeSpan.Cooldown
+    }
+
+    It "ShouldSetByParam" {
+        { Set-FleetProfile -ProfileXml $x -Throughput 1000 } | Should Not Throw
+        $xn = Set-FleetProfile -ProfileXml $x -Throughput 1000
+
+        # original composition is 1500*4 threads + 300*1 thread = 6300
+        # 1000 * 6000/6300 / 4 threads = 238
+        # 1000 * 300/6300 / 1 thread = 48
+        $xn.Profile.TimeSpans.TimeSpan.Targets.Target[0].Throughput.InnerText | Should Be 238
+        $xn.Profile.TimeSpans.TimeSpan.Targets.Target[1].Throughput.InnerText | Should Be 48
+    }
+
+    It "ShouldSetByPipeline" {
+        { $x | Set-FleetProfile -Throughput 1000 } | Should Not Throw
+        $xn = $x | Set-FleetProfile -Throughput 1000
+
+        $xn.Profile.TimeSpans.TimeSpan.Targets.Target[0].Throughput.InnerText | Should Be 238
+        $xn.Profile.TimeSpans.TimeSpan.Targets.Target[1].Throughput.InnerText | Should Be 48
+    }
+
+    It "ShouldSetUnbounded" {
+        { $x | Set-FleetProfile -Throughput 0 } | Should Not Throw
+        $xn = $x | Set-FleetProfile -Throughput 0
+
+        $xn.Profile.TimeSpans.TimeSpan.ThreadCount | Should be 5
+        $xn.Profile.TimeSpans.TimeSpan.RequestCount | Should be 32
+
+        $xn.Profile.TimeSpans.SelectNodes("TimeSpan/Targets/Target/ThreadsPerFile") | Should BeNullOrEmpty
+        $xn.Profile.TimeSpans.SelectNodes("TimeSpan/Targets/Target/RequestCount") | Should BeNullOrEmpty
+        $xn.Profile.Timespans.TimeSpan.Targets.Target[1].InterlockedSequential | Should Be 'true'
+    }
+
+    It "ShouldSetWarmup" {
+        $oWarmup | Should Not Be 33
+        { $x | Set-FleetProfile -Warmup 33 } | Should Not Throw
+        $xn = $x | Set-FleetProfile -Warmup 33
+
+        $xn.Profile.TimeSpans.TimeSpan.Warmup | Should Be 33
+        $xn.Profile.TimeSpans.TimeSpan.Duration | Should Be $oDuration
+        $xn.Profile.TimeSpans.TimeSpan.Cooldown | Should Be $oCooldown
+    }
+
+    It "ShouldSetDuration" {
+        $oDuration | Should Not Be 33
+        { $x | Set-FleetProfile -Duration 33 } | Should Not Throw
+        $xn = $x | Set-FleetProfile -Duration 33
+
+        $xn.Profile.TimeSpans.TimeSpan.Warmup | Should Be $oWarmup
+        $xn.Profile.TimeSpans.TimeSpan.Duration | Should Be 33
+        $xn.Profile.TimeSpans.TimeSpan.Cooldown | Should Be $oCooldown
+    }
+
+    It "ShouldSetCooldown" {
+        $oCooldown | Should Not Be 33
+        { $x | Set-FleetProfile -Cooldown 33 } | Should Not Throw
+        $xn = $x | Set-FleetProfile -Cooldown 33
+
+        $xn.Profile.TimeSpans.TimeSpan.Warmup | Should Be $oWarmup
+        $xn.Profile.TimeSpans.TimeSpan.Duration | Should Be $oDuration
+        $xn.Profile.TimeSpans.TimeSpan.Cooldown | Should Be 33
+    }
+}
+
+Describe "GetFleetProfileXml" {
+    It "HasPeak" {
+        { $peak = Get-FleetProfileXml -Name Peak -BlockSize 8KB -WriteRatio 0 } | Should Not Throw
+        $peak = Get-FleetProfileXml -Name Peak -BlockSize 8KB -WriteRatio 0
+        $peak | Should Not BeNullOrEmpty
+    }
+
+    It "PeakDoesNotDefineBaseMax" {
+        $peak = Get-FleetProfileXml -Name Peak -BlockSize 8KB -WriteRatio 0
+        $peak.SelectNodes('Profile/TimeSpans/TimeSpan/Targets/Target/BaseFileOffset') | Should BeNullOrEmpty
+        $peak.SelectNodes('Profile/TimeSpans/TimeSpan/Targets/Target/MaxFileSize') | Should BeNullOrEmpty
+    }
+
+    It "PeakShouldAcceptBase" {
+        $peak = Get-FleetProfileXml -Name Peak -BlockSize 8KB -WriteRatio 0 -BaseOffset 1GB
+        $peak.SelectNodes('Profile/TimeSpans/TimeSpan/Targets/Target/MaxFileSize') | Should BeNullOrEmpty
+        $n = $peak.SelectNodes('Profile/TimeSpans/TimeSpan/Targets/Target/BaseFileOffset')
+        $n | Should Not BeNullOrEmpty
+        $n.Count | Should Be 1
+
+        $n.Item(0).InnerText | Should Be ([string] 1GB)
+    }
+
+    It "PeakShouldAcceptMax" {
+        $peak = Get-FleetProfileXml -Name Peak -BlockSize 8KB -WriteRatio 0 -MaxOffset 2GB
+        $peak.SelectNodes('Profile/TimeSpans/TimeSpan/Targets/Target/BaseFileOffset') | Should BeNullOrEmpty
+        $n = $peak.SelectNodes('Profile/TimeSpans/TimeSpan/Targets/Target/MaxFileSize')
+        $n | Should Not BeNullOrEmpty
+
+        $n.Count | Should Be 1
+        $n.Item(0).InnerText | Should Be ([string] 2GB)
+    }
+
+    It "PeakShouldAcceptBaseMax" {
+        $peak = Get-FleetProfileXml -Name Peak -BlockSize 8KB -WriteRatio 0 -BaseOffset 1GB -MaxOffset 2GB
+
+        $n = $peak.SelectNodes('Profile/TimeSpans/TimeSpan/Targets/Target/BaseFileOffset')
+        $n | Should Not BeNullOrEmpty
+        $n.Count | Should Be 1
+        $n.Item(0).InnerText | Should Be ([string] 1GB)
+
+        $n = $peak.SelectNodes('Profile/TimeSpans/TimeSpan/Targets/Target/MaxFileSize')
+        $n | Should Not BeNullOrEmpty
+        $n.Count | Should Be 1
+        $n.Item(0).InnerText | Should Be ([string] 2GB)
+    }
+
+    It "PeakShouldAcceptThreads" {
+        $peak = Get-FleetProfileXml -Name Peak -BlockSize 8KB -WriteRatio 0 -ThreadsPerTarget 2
+
+        $n = $peak.SelectNodes('Profile/TimeSpans/TimeSpan/Targets/Target/ThreadsPerFile')
+        $n | Should Not BeNullOrEmpty
+        $n.Count | Should Be 1
+        $n.Item(0).InnerText | Should Be ([string] 2)
+    }
+}
+
+
+InModuleScope VMFleet {
+
+    Describe "GetNextSplit" {
+
+        It "ShouldBeLargest@End" {
+            $o = [PSCustomObject]@{
+                    Value = 0
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 5
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 15
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 40
+                    CutoffType =[CutoffType]::No
+                }
+
+            $p1, $p2 = GetNextSplit $o 'Value' -OrderBy 'Value'
+            $p1.Value | Should Be 40
+            $p2.Value | Should Be 15
+        }
+
+        It "ShouldBeLargest@Begin" {
+            $o = [PSCustomObject]@{
+                    Value = 40
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 5
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 15
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 0
+                    CutoffType =[CutoffType]::No
+                }
+
+            $p1, $p2 = GetNextSplit $o 'Value' -OrderBy 'Value'
+            $p1.Value | Should Be 40
+            $p2.Value | Should Be 15
+        }
+
+        It "ShouldBeFirstOfEqual" {
+            $o = [PSCustomObject]@{
+                    Value = 0
+                    Order = 0
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 10
+                    Order = 1
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 20
+                    Order = 2
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 30
+                    Order = 3
+                    CutoffType =[CutoffType]::No
+                }
+
+            $p1, $p2 = GetNextSplit $o 'Value' -OrderBy 'Value'
+            $p1.Value | Should Be 10
+            $p2.Value | Should Be 0
+        }
+
+        It "ShouldRespectOrder" {
+            $o = [PSCustomObject]@{
+                    Value = 0
+                    Order = 0
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 10
+                    Order = 2
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 20
+                    Order = 3
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 30
+                    Order = 1
+                    CutoffType =[CutoffType]::No
+                }
+
+            $p1, $p2 = GetNextSplit $o 'Value' -OrderBy 'Order'
+            $p1.Value | Should Be 30
+            $p2.Value | Should Be 0
+        }
+
+        It "ShouldRespectCutoff" {
+            $o = [PSCustomObject]@{
+                    Value = 0
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 10
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 30
+                    CutoffType =[CutoffType]::Scale
+                },
+                [PSCustomObject]@{
+                    Value = 60
+                    CutoffType =[CutoffType]::Scale
+                }
+
+            $p1, $p2 = GetNextSplit $o 'Value' -OrderBy 'Value'
+            $p1.Value | Should Be 30
+            $p2.Value | Should Be 10
+        }
+
+        It "ShouldRespectCutoffOrdered" {
+            $o = [PSCustomObject]@{
+                    Value = 0
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 30
+                    CutoffType =[CutoffType]::Scale
+                },
+                [PSCustomObject]@{
+                    Value = 10
+                    CutoffType =[CutoffType]::No
+                },
+                [PSCustomObject]@{
+                    Value = 60
+                    CutoffType =[CutoffType]::Scale
+                }
+
+            $p1, $p2 = GetNextSplit $o 'Value' -OrderBy 'Value'
+            $p1.Value | Should Be 30
+            $p2.Value | Should Be 10
+        }
+    }
+
+    Describe "GetUpperAnchor" {
+
+        It "ShouldReturnFinalIfNotCutoff" {
+            $o = [PSCustomObject]@{
+                Value = 0
+                Order = 0
+                CutoffType =[CutoffType]::No
+            },
+            [PSCustomObject]@{
+                Value = 10
+                Order = 2
+                CutoffType =[CutoffType]::No
+            },
+            [PSCustomObject]@{
+                Value = 20
+                Order = 3
+                CutoffType =[CutoffType]::No
+            },
+            [PSCustomObject]@{
+                Value = 30
+                Order = 1
+                CutoffType =[CutoffType]::No
+            }
+
+            $p1,$p2 = GetUpperAnchor $o -OrderBy 'Order'
+            $p1.Value | Should Be 20
+            $p2.Value | Should Be 10
+        }
+
+        It "ShouldReturnFirstCutoff" {
+            $o = [PSCustomObject]@{
+                Value = 0
+                Order = 0
+                CutoffType =[CutoffType]::No
+            },
+            [PSCustomObject]@{
+                Value = 10
+                Order = 2
+                CutoffType =[CutoffType]::Scale
+            },
+            [PSCustomObject]@{
+                Value = 20
+                Order = 3
+                CutoffType =[CutoffType]::No
+            },
+            [PSCustomObject]@{
+                Value = 30
+                Order = 1
+                CutoffType =[CutoffType]::No
+            }
+
+            $p1,$p2 = GetUpperAnchor $o -OrderBy 'Order'
+            $p1.Value | Should Be 10
+            $p2.Value | Should Be 30
+        }
+    }
+
+    Describe "IsProfileThroughputLimited" {
+
+        It "PeakIsNot" {
+            $x = Get-FleetProfileXml -Name Peak -BlockSize 4KB -WriteRatio 0
+            IsProfileThroughputLimited -ProfileXml $x | Should Be $false
+        }
+
+        It "SqlIs" {
+            $x = Get-FleetProfileXml -Name Sql
+            IsProfileThroughputLimited -ProfileXml $x | Should Be $true
+        }
+    }
+
+    Describe "IsProfileSingleTimespan" {
+
+        It "PeakIs" {
+            $x = Get-FleetProfileXml -Name Peak -BlockSize 4KB -WriteRatio 0
+            IsProfileSingleTimespan -ProfileXml $x | Should Be $true
+        }
+    }
+
+    Describe "IsProfileSingleTarget" {
+
+        It "PeakIs" {
+            $x = Get-FleetProfileXml -Name Peak -BlockSize 4KB -WriteRatio 0
+            IsProfileSingleTarget -ProfileXml $x | Should Be $true
+        }
+
+        It "SqlIsNot" {
+            $x = Get-FleetProfileXml -Name Sql
+            IsProfileSingleTarget -ProfileXml $x | Should Be $false
+        }
+    }
+
+    Describe "GetFleetProfileFootprint" {
+
+        BeforeAll {
+            $sql = Get-FleetProfileXml -Name SQL
+            $vdi = Get-FleetProfileXml -Name VDI
+        }
+
+        It "CheckVDI" {
+            $vdi | Should Not BeNullOrEmpty
+            $f = GetFleetProfileFootprint -ProfileXml $vdi
+            $f.Count | Should Be 1
+            $f['*1'].BaseOffset | Should Be 0
+            $f['*1'].MaxOffset | Should Be (10GB)
+        }
+
+        It "CheckVDIRead" {
+            $vdi | Should Not BeNullOrEmpty
+            $f = GetFleetProfileFootprint -ProfileXml $vdi -Read
+            $f.Count | Should Be 1
+            $f['*1'].BaseOffset | Should Be 0
+            $f['*1'].MaxOffset | Should Be (8GB)
+        }
+
+        It "CheckSQL" {
+            $sql | Should Not BeNullOrEmpty
+            $f = GetFleetProfileFootprint -ProfileXml $sql
+            $f.Count | Should Be 1
+            $f['*1'].BaseOffset | Should Be 0
+            $f['*1'].MaxOffset | Should Be 0
+        }
+
+        It "CheckSQLRead" {
+            $sql | Should Not BeNullOrEmpty
+            $f = GetFleetProfileFootprint -ProfileXml $sql -Read
+            $f.Count | Should Be 1
+            $f['*1'].BaseOffset | Should Be (5GB)
+            $f['*1'].MaxOffset | Should Be 0
+        }
+    }
+
+    Describe "TimespanToString" {
+
+        BeforeAll {
+            $t0 = [datetime] '7/31/1996 12:00PM'
+        }
+
+        It "Seconds" {
+            TimespanToString ($t0.AddMilliseconds(1500) - $t0) | Should Be "01.5s"
+        }
+
+        It "Minutes" {
+            TimespanToString ($t0.AddMinutes(15) - $t0) | Should Be "15m:00.0s"
+        }
+
+        It "Hours" {
+            TimespanToString ($t0.AddHours(15) - $t0) | Should Be "15h:00m:00.0s"
+        }
+
+        It "Days" {
+            TimespanToString ($t0.AddDays(15) - $t0) | Should Be "15d.00h:00m:00.0s"
+        }
+    }
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet/VMFleet.format.ps1xml

@@ -0,0 +1,82 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Configuration>
+  <ViewDefinitions>
+    <!--
+
+    //
+    // VMFLeet.VolumeEstimate
+    //
+
+    -->
+    <View>
+      <Name>StorageBusBindingTableView</Name>
+      <ViewSelectedBy>
+        <TypeName>VolumeEstimate</TypeName>
+      </ViewSelectedBy>
+      <TableControl>
+        <TableHeaders>
+          <TableColumnHeader>
+            <Label>VolumeType</Label>
+          </TableColumnHeader>
+          <TableColumnHeader>
+            <Label>MirrorSize</Label>
+          </TableColumnHeader>
+          <TableColumnHeader>
+            <Label>MirrorTierName</Label>
+          </TableColumnHeader>
+          <TableColumnHeader>
+            <Label>ParitySize</Label>
+          </TableColumnHeader>
+          <TableColumnHeader>
+            <Label>ParityTierName</Label>
+          </TableColumnHeader>
+          <TableColumnHeader>
+            <Label>Size</Label>
+          </TableColumnHeader>
+        </TableHeaders>
+        <TableRowEntries>
+          <TableRowEntry>
+            <TableColumnItems>
+              <TableColumnItem>
+                <PropertyName>VolumeType</PropertyName>
+              </TableColumnItem>
+              <TableColumnItem>
+                <Alignment>Right</Alignment>
+                <ScriptBlock>
+                  $v = $_.MirrorSize;
+                  $postfixes = @( "B", "KB", "MB", "GB", "TB", "PB" )
+                  for ($i=0; $v -ge 1024 -and $i -lt $postfixes.Length; $i++) { $v /= 1024; }
+                  return "" + [System.Math]::Round($v,2) + " " + $postfixes[$i];
+                </ScriptBlock>
+              </TableColumnItem>
+              <TableColumnItem>
+                <PropertyName>MirrorTierName</PropertyName>
+              </TableColumnItem>
+              <TableColumnItem>
+                <Alignment>Right</Alignment>
+                <ScriptBlock>
+                  $v = $_.ParitySize;
+                  $postfixes = @( "B", "KB", "MB", "GB", "TB", "PB" )
+                  for ($i=0; $v -ge 1024 -and $i -lt $postfixes.Length; $i++) { $v /= 1024; }
+                  return "" + [System.Math]::Round($v,2) + " " + $postfixes[$i];
+                </ScriptBlock>
+              </TableColumnItem>
+              <TableColumnItem>
+                <PropertyName>ParityTierName</PropertyName>
+              </TableColumnItem>
+              <TableColumnItem>
+                <Alignment>Right</Alignment>
+                <ScriptBlock>
+                  $v = $_.Size;
+                  $postfixes = @( "B", "KB", "MB", "GB", "TB", "PB" )
+                  for ($i=0; $v -ge 1024 -and $i -lt $postfixes.Length; $i++) { $v /= 1024; }
+                  return "" + [System.Math]::Round($v,2) + " " + $postfixes[$i];
+                </ScriptBlock>
+              </TableColumnItem>
+            </TableColumnItems>
+          </TableRowEntry>
+        </TableRowEntries>
+      </TableControl>
+    </View>
+  </ViewDefinitions>
+</Configuration>

CristalDiskMark/source/diskspd22/Frameworks/VMFleet/VMFleet.psd1

@@ -0,0 +1,197 @@
+<#
+VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+#
+# Module manifest for module 'VMFleet'
+#
+# Generated on: 1/30/2021
+#
+
+@{
+
+# Script module or binary module file associated with this manifest.
+RootModule = 'VMFleet.psm1'
+
+# Version number of this module. Even build# is release, odd pre-release (Mj.Mn.Bd.Rv)
+ModuleVersion = '2.1.0.0'
+
+# Supported PSEditions
+# CompatiblePSEditions = @()
+
+# ID used to uniquely identify this module
+GUID = '753ad5da-01b3-4cc8-a475-16a09a021384'
+
+# Author of this module
+Author = 'Microsoft Corporation'
+
+# Company or vendor of this module
+CompanyName = 'Microsoft Corporation'
+
+# Copyright statement for this module
+Copyright = '(c) 2021 Microsoft Corporation. All rights reserved.'
+
+# Description of the functionality provided by this module
+Description = 'VM Fleet is a performance characterization and analyst framework for exploring the storage capabilities of Windows Server Hyper-Converged environments with Storage Spaces Direct'
+
+# Minimum version of the Windows PowerShell engine required by this module
+PowerShellVersion = '5.1'
+
+# Name of the Windows PowerShell host required by this module
+# PowerShellHostName = ''
+
+# Minimum version of the Windows PowerShell host required by this module
+# PowerShellHostVersion = ''
+
+# Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only.
+# DotNetFrameworkVersion = ''
+
+# Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only.
+# CLRVersion = ''
+
+# Processor architecture (None, X86, Amd64) required by this module
+# ProcessorArchitecture = ''
+
+# Modules that must be imported into the global environment prior to importing this module
+# RequiredModules = @()
+
+# Assemblies that must be loaded prior to importing this module
+# RequiredAssemblies = @()
+
+# Script files (.ps1) that are run in the caller's environment prior to importing this module.
+# ScriptsToProcess = @()
+
+# Type files (.ps1xml) to be loaded when importing this module
+# TypesToProcess = @()
+
+# Format files (.ps1xml) to be loaded when importing this module
+FormatsToProcess = @( 'VMFleet.format.ps1xml' )
+
+# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess
+NestedModules = @(
+    'Profile.psm1',
+    'WatchCluster.psm1',
+    'WatchCPU.psm1'
+)
+
+# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export.
+FunctionsToExport = @(
+    'Clear-FleetPause',
+    'Clear-FleetRunState',
+    'Convert-FleetXmlToString',
+    'Get-FleetComputeTemplate',
+    'Get-FleetDisk',
+    'Get-FleetDataDiskEstimate',
+    'Get-FleetPath',
+    'Get-FleetPause',
+    'Get-FleetPolynomialFit',
+    'Get-FleetPowerScheme',
+    'Get-FleetProfileXml',
+    'Get-FleetResultLog',
+    'Get-FleetVersion',
+    'Get-FleetVM',
+    'Get-FleetVolumeEstimate',
+    'Install-Fleet',
+    'Measure-FleetCoreWorkload',
+    'Move-Fleet',
+    'New-Fleet',
+    'Remove-Fleet',
+    'Repair-Fleet',
+    'Set-ArcConfig',
+    'Set-Fleet',
+    'Set-FleetPause',
+    'Set-FleetPowerScheme',
+    'Set-FleetProfile',
+    'Set-FleetQoS',
+    'Set-FleetRunProfileScript',
+    'Show-Fleet',
+    'Show-FleetCpuSweep',
+    'Show-FleetPause',
+    'Start-Fleet',
+    'Start-FleetReadCacheWarmup',
+    'Start-FleetResultRun',
+    'Start-FleetRun',
+    'Start-FleetSweep',
+    'Start-FleetWriteWarmup',
+    'Stop-Fleet',
+    'Test-FleetResultRun',
+    'Use-FleetPolynomialFit',
+    'Watch-FleetCluster',
+    'Watch-FleetCPU',
+    'Initialize-ArcVMs'
+)
+
+# Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export.
+CmdletsToExport = @(
+
+)
+
+# Variables to export from this module
+VariablesToExport = $null
+
+# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export.
+AliasesToExport = @()
+
+# DSC resources to export from this module
+# DscResourcesToExport = @()
+
+# List of all modules packaged with this module
+# ModuleList = @()
+
+# List of all files packaged with this module
+# FileList = @()
+
+# Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell.
+PrivateData = @{
+
+    PSData = @{
+
+        # Tags applied to this module. These help with module discovery in online galleries.
+        # Tags = @()
+
+        # A URL to the license for this module.
+        # LicenseUri = ''
+
+        # A URL to the main website for this project.
+        ProjectUri = 'https://www.github.com/microsoft/diskspd'
+
+        # A URL to an icon representing this module.
+        # IconUri = ''
+
+        # ReleaseNotes of this module
+        # ReleaseNotes = ''
+
+    } # End of PSData hashtable
+
+} # End of PrivateData hashtable
+
+# HelpInfo URI of this module
+# HelpInfoURI = ''
+
+# Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix.
+# DefaultCommandPrefix = ''
+
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet/WatchCPU.psm1

@@ -0,0 +1,282 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+Set-StrictMode -Version 3.0
+
+function Watch-FleetCPU
+{
+    [CmdletBinding()]
+    param(
+        [Parameter()]
+        [string]
+        $ComputerName = $env:COMPUTERNAME,
+
+        [Parameter()]
+        [switch]
+        $Guest,
+
+        [Parameter()]
+        [int]
+        $SampleInterval = 2
+        )
+
+    function div-to-width(
+        [int] $div
+        )
+    {
+        # 0 - 100 scale
+        # ex: 4 -> 100/4 = 25 buckets + 1 more for == 100
+        1+100/$div
+    }
+
+    function center-pad(
+        [string] $s,
+        [int] $width
+        )
+    {
+        if ($width -le $s.length) {
+            $s
+        } else {
+            (' ' * (($width - $s.length)/2) + $s)
+        }
+    }
+
+    function get-legend(
+        [string] $label,
+        [int] $width,
+        [int] $div
+        )
+    {
+        # now produce the scale, a digit at a time in vertical orientation
+        # at each multiple of 10% which aligns with a measurement bucket.
+        # the width is the width of the measured values
+        #
+        # 0 5 1
+        #   0 0
+        #     0
+
+        $lines = @()
+        $lines += '-' * $width
+
+        foreach ($dig in 0..2) {
+            $o = foreach ($pos in 0..($width - 1)) {
+
+                $val = $pos * $div
+                if ($val % 10 -eq 0) {
+                    switch ($dig) {
+                        0 { if ($val -eq 100) { 1 } else { $val / 10 }}
+                        1 { if ($val -ne 0) { 0 } else { ' ' }}
+                        2 { if ($val -eq 100) { 0 } else { ' ' }}
+                    }
+                } else { ' ' }
+            }
+
+            $lines += $o -join ''
+        }
+
+        # trailing comments (horizontal scale name)
+        $lines += center-pad $label $width
+
+        $lines
+    }
+
+    # minimum clip, the vertical height available for the cpu core bars
+    $minClip = 10
+
+    # these are the valid divisions, in units of percentage width.
+    # they must evenly divide 100% and either 10% or 20% for scale markings.
+    # determine which is the best fit based on window width.
+
+    $div = 0
+    $divs = 1,2,4,5
+    foreach ($i in $divs) {
+        if ((div-to-width $i) -le [console]::WindowWidth) {
+            $div = $i
+            break
+        }
+    }
+
+    # if nothing fit ... ask for minimum
+    # in practice this is not possible, but we check anyway
+    if ($div -eq 0) {
+        Write-Error "Window width must be at least $(div-to-width $divs[-1]) columns"
+        return
+    }
+
+    $width = div-to-width $div
+
+    # which processor counterset should we use?
+    # pi is only the root partition if hv is active; when hv is active:
+    #     hvlp is the host physical processors
+    #     hvvp is the guest virtual processors
+    # via ctrs, hv is active iff hvlp is present and has multiple instances
+    $cs = Get-Counter -ComputerName $ComputerName -ListSet 'Hyper-V Hypervisor Logical Processor' -ErrorAction SilentlyContinue
+    $hvactive = $null -ne $cs -and $cs.CounterSetType -eq [Diagnostics.PerformanceCounterCategoryType]::MultiInstance
+
+    if ($Guest -and -not $hvactive) {
+        Write-Error "Hyper-V is not active on $ComputerName"
+        return
+    }
+
+    if ($hvactive) {
+        if ($Guest) {
+            $cpuset = "\Hyper-V Hypervisor Virtual Processor(*)\% Guest Run Time"
+            $legend = get-legend "Percent Guest VCPU Utilization" $width $div
+        } else {
+            $cpuset = '\Hyper-V Hypervisor Logical Processor(*)\% Total Run Time'
+            $legend = get-legend "Percent Host LP Utilization" $width $div
+        }
+    } else {
+        $cpuset = '\Processor Information(*)\% Processor Time'
+        $legend = get-legend "Percent Processor Utilization" $width $div
+    }
+
+    # processor performance counter (turbo/speedstep)
+    # this is used to normalize the total cpu utilization (can be > 100%)
+    $ppset = '\Processor Information(_Total)\% Processor Performance'
+
+    # account for the constant legend in the window height
+    # use the remaining height for the vertical cpu core bars.
+    $clip = [console]::WindowHeight - ($legend.Count + 1)
+
+    # insist on a minimum amount of space
+    if ($clip -lt $minClip) {
+        $minWindowHeight = $minClip + $legend.Count + 1
+        Write-Error "Window height must be at least $minWindowHeight lines"
+        return
+    }
+
+    # set window and buffer size simultaneously so we don't have extra scrollbars
+    Clear-Host
+    [console]::SetWindowSize($width, [console]::WindowHeight)
+    [console]::BufferWidth = [console]::WindowWidth
+    [console]::BufferHeight = [console]::WindowHeight
+
+    # common params for Get-Counter
+    $gcParam = @{
+        SampleInterval = $SampleInterval
+        Counter = $cpuset,$ppset
+        ErrorAction = 'SilentlyContinue'
+    }
+
+    while ($true) {
+
+        # reset measurements
+        # these are the vertical height of a cpu bar in each column, e.g. 2 = 2 cpus
+        $m = @([int] 0) * $width
+
+        # avoid remoting for the local case
+        if ($ComputerName -eq $env:COMPUTERNAME) {
+            $ctrs = Get-Counter @gcParam
+        } else {
+            $ctrs = Get-Counter @gcParam -ComputerName $ComputerName
+        }
+
+        # if more than one countersample was returned (ppset + something more), we have data
+        if ($null -ne $ctrs -and $ctrs.Countersamples.Count -gt 1)
+        {
+            # get all specific instances and count them into appropriate measurement bucket
+            $ctrs.Countersamples |% {
+
+                # get scaling factor for total utility
+                if ($_.Path -like "*$ppset") {
+                    $pperf = $_.CookedValue/100
+                }
+
+                # a cpu: count into appropriate measurement bucket
+                # (ignore total and/or and per-numa total)
+                elseif ($_.InstanceName -notlike '*_Total') {
+                    $m[[math]::Floor($_.CookedValue/$div)] += 1
+                }
+
+                # get total
+                #
+                elseif ($_.InstanceName -eq '_Total') {
+                    $total = $_.CookedValue
+                }
+            }
+
+            # now produce the bar area as a series of lines
+            # work down the veritical altitude of each strip, starting at the clip/top
+            $altitude = $clip
+            $lines = do {
+                $($m |% {
+
+                    # top line - if we are potentially clipped, handle
+                    if ($altitude -eq $clip) {
+
+                        # clipped?
+                        if ($_ -gt $altitude) { 'x' }
+                        # unclipped but at clip?
+                        elseif ($_ -eq $altitude) { '*' }
+                        # nothing, bar less than altitude
+                        else { ' ' }
+
+                    } else {
+
+                        # below top line
+                        # >=, output bar
+                        if ($_ -ge $altitude) { '*' }
+                        # <, nothing
+                        else { ' ' }
+                    }
+                }) -join ''
+            } while (--$altitude)
+
+            $totalStr = "{0:0.0}%" -f $total
+            $normalStr = "{0:0.0}%" -f ($total*$pperf)
+
+            # move the cursor to indicate average utilization
+            # column number is zero based, width is 1-based
+            $cpos = [math]::Floor(($width - 1)*$total/100)
+        }
+        else
+        {
+            # Center no data message vertically and horizontally in the frame
+
+            $vpre = [math]::Floor($clip/2) - 1
+            $vpost = [math]::Floor($clip/2)
+
+            $lines  = @('') * $vpre
+            $lines += center-pad "No Data Available" $width
+            $lines += @('') * $vpost
+
+            $totalStr = $normalStr = "---"
+
+            # zero cursor
+            $cpos = 0
+        }
+
+        Clear-Host
+        Write-Host -NoNewline ($lines + $legend -join "`n")
+        Write-Host -NoNewLine ("`n" + (center-pad "$ComputerName Total: $totalStr Normalized: $normalStr" $width))
+
+        # move the cursor to indicate average utilization
+        # column number is zero based, width is 1-based
+        [console]::SetCursorPosition($cpos,[console]::CursorTop-$legend.Count)
+    }
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet/WatchCluster.psm1

@@ -0,0 +1,552 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+Set-StrictMode -Version 3.0
+
+# display name
+# ctr name
+# display order
+# format string
+# scalar multiplier
+
+class CounterColumn {
+
+    [string] $displayname
+    [string] $setname
+    [string[]] $ctrname
+    [int] $width
+    [string] $fmt
+    [decimal] $multiplier
+    [ValidateSet("Average","AverageAggregate","Sum")][string] $aggregate
+    [boolean] $divider
+
+    CounterColumn(
+        [string] $displayname,
+        [string] $setname,
+        [string[]] $ctrname,
+        [int] $width,
+        [string] $fmt,
+        [decimal] $multiplier,
+        [string] $aggregate,
+        [boolean] $divider
+    )
+    {
+        $this.displayname = $displayname
+        $this.setname = $setname
+        $this.ctrname = $ctrname
+        $this.width = $width
+        $this.fmt = $fmt
+        $this.multiplier = $multiplier
+        $this.aggregate = $aggregate
+        $this.divider = $divider
+
+        if ($this.width -lt $this.displayname.length + 1) {
+            $this.width = $this.displayname.length + 1
+        }
+    }
+}
+
+class CounterColumnSet {
+
+    [CounterColumn[]] $columns
+    [string[]] $counters
+    [string] $topfmt
+    [string] $linfmt
+    [string] $name
+
+    [string] $totalline
+    [string[]] $nodelines
+
+    CounterColumnSet($name)
+    {
+        $this.columns = $null
+        $this.name = $name
+    }
+
+    [void] Add([CounterColumn] $c)
+    {
+        $this.columns += $c
+    }
+
+    [void] Seal()
+    {
+        # assemble the top-line fmt and per-line fmt strings
+        # top-line is just strings/width
+        # per-line adds the (likely numeric) format specifier
+        $n = 1
+        $this.topfmt = $this.linfmt = "{0,-16}"
+        foreach ($col in $this.columns) {
+            $str = ''
+            if ($col.divider) {
+                $str += '| '
+            }
+            $str += "{$n,-$($col.width)"
+            $this.topfmt += "$str}"
+            $this.linfmt += "$($str):$($col.fmt)}"
+
+            $n += 1
+        }
+
+        # assemble the list of counter instances that will be needed
+        # note that some may be internally aggregated (i.e., total = read + write)
+        # in cases where a counterset does not provide an explicit total
+        $this.counters = ($this.columns |% {
+            $s = $_.setname
+            $_.ctrname |% { "\$($s)(_Total)\$($_)" }
+        } | group -NoElement).Name
+    }
+
+    [void] DisplayPre(
+        [hashtable] $samples,
+        [hashtable] $psamples
+        )
+    {
+        # aggregate each column across all live sampled nodes
+        $this.totalline =  $this.linfmt -f $(
+            "Total"
+            foreach ($col in $this.columns) {
+
+                # average aggreate doesn't work across nodes if the base is not consistent
+                # for instance: cannot average latency safely, but can average cpu utilization
+                if ($col.aggregate -ne 'Average' -or $col.aggregate -eq 'AverageAggregate') {
+                    $(foreach ($node in $psamples.keys) {
+                        get-samples $psamples $node $col
+                    }) | get-aggregate $col
+                } else {
+                    $null
+                }
+            }
+        )
+
+        # individual nodes
+        # flags downed/non-responsive nodes by noting which are not
+        # present in the processed samples
+        $this.nodelines = foreach ($node in $samples.keys | sort) {
+            if ($psamples.ContainsKey($node)) {
+                $this.linfmt -f $(
+                    $node
+                    foreach ($col in $this.columns) {
+                        $s = get-samples $psamples $node $col
+                        if ($null -ne $s) {
+                            $a = $s | get-aggregate $col
+                            $a
+                        } else {
+                            "-"
+                        }
+                    }
+                )
+            } else {
+                $this.topfmt -f $(
+                    $node
+                    0..($this.columns.count - 1) |% { "-" }
+                )
+            }
+        }
+    }
+
+    [void] Display()
+    {
+        write-host ($this.topfmt -f (,$this.name + $this.columns.displayname))
+        write-host -fore green $this.totalline
+        $this.nodelines |% { write-host $_ }
+    }
+}
+
+function Watch-FleetCluster
+{
+
+    [CmdletBinding()]
+    param(
+        [Parameter()]
+        [string]
+        $Cluster = ".",
+
+        [Parameter()]
+        [int]
+        $SampleInterval = 2,
+
+        [Parameter()]
+        [ValidateSet("CSV FS","SSB Cache","SBL","SBL Local","SBL Remote","SBL*","S2D BW","Hyper-V LCPU","SMB SRV","SMB Transport","*")]
+        [string[]] $Sets = "CSV FS",
+
+        [Parameter()]
+        [string]
+        $LogFile
+    )
+
+    if ($PSBoundParameters.ContainsKey('LogFile'))
+    {
+        $script:log = $LogFile
+        Remove-Item -Force $log -ErrorAction SilentlyContinue
+    }
+
+    function WriteLog(
+        [Parameter(ValueFromPipeline = $true)]
+        [string[]]
+        $String
+        )
+    {
+        PROCESS {
+            if ($null -ne $script:log) {
+                "$(get-date) $String" | Out-File -Append -FilePath $script:log -Width 9999 -Encoding ascii
+            }
+        }
+    }
+
+    function get-aggregate(
+        [CounterColumn] $col
+        )
+    {
+        BEGIN {
+            $n = 0
+            $v = 0
+        }
+        PROCESS {
+            $n += 1
+            $v += $_
+        }
+        END {
+            if ($n -gt 0) {
+                switch -wildcard ($col.aggregate) {
+                    'Sum' {
+                        #write-host $col.displayname $col.multipler $v
+                        $col.multiplier * $v
+                    }
+                    'Average*' {
+                        #write-host $col.displayname $col.multiplier $v $n
+                        $col.multiplier * $v / $n
+                    }
+                }
+            } else {
+                $null
+            }
+        }
+    }
+
+    # get samples out of per-node hash of ctr hashes
+    function get-samples(
+        [hashtable] $h,
+        [string] $node,
+        [CounterColumn] $col
+        )
+    {
+        foreach ($i in $col.ctrname) {
+
+            $k = "$($col.setname)+$($i)"
+
+            if ($h.ContainsKey($node)) {
+                if ($h[$node].ContainsKey($k)) {
+                    $h[$node][$k]
+                } else {
+                    WriteLog "missing $node[$k] : $($h[$node].Keys.Count) total keys"
+                }
+            } else {
+                WriteLog "missing $node"
+            }
+        }
+    }
+
+    $allctrs = @()
+
+    ###
+    $c = [CounterColumnSet]::new("CSV FS")
+    $c.Add([CounterColumn]::new("IOPS", "Cluster CSVFS", @("Reads/sec","Writes/sec"), 12, '#,#', 1, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Reads", "Cluster CSVFS", @("Reads/sec"), 12, '#,#', 1, 'Sum',  $false))
+    $c.Add([CounterColumn]::new("Writes", "Cluster CSVFS", @("Writes/sec"), 12, '#,#', 1, 'Sum',  $false))
+
+    $c.Add([CounterColumn]::new("BW (MB/s)", "Cluster CSVFS", @("Read Bytes/sec","Write Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
+    $c.Add([CounterColumn]::new("Read", "Cluster CSVFS", @("Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum',  $false))
+    $c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+    $c.Add([CounterColumn]::new("Read Lat (ms)", "Cluster CSVFS", @("Avg. sec/Read"), 15, '0.000', 1000, 'Average', $true))
+    $c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Avg. sec/Write"), 8, '0.000', 1000, 'Average', $false))
+
+    $c.Add([CounterColumn]::new("Read QAvg", "Cluster CSVFS", @("Avg. Read Queue Length"), 11, '0.000', 1, 'Average', $true))
+    $c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Avg. Write Queue Length"), 8, '0.000', 1, 'Average', $false))
+
+    $c.Seal()
+    $allctrs += $c
+
+    ###
+    $c = [CounterColumnSet]::new("SSB Cache")
+
+    $c.Add([CounterColumn]::new("Hit/Sec", "Cluster Storage Hybrid Disks", @("Cache Hit Reads/Sec"), 12, '#,#', 1, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Miss/Sec", "Cluster Storage Hybrid Disks", @("Cache Miss Reads/Sec"), 12, '#,#', 1, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Remap/Sec" ,"Cluster Storage Cache Stores", @("Page ReMap/sec"), 12, '#,#', 1, 'Sum', $false))
+
+    $c.Add([CounterColumn]::new("Cache (MB/s)", "Cluster Storage Hybrid Disks", @("Cache Populate Bytes/sec","Cache Write Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
+    $c.Add([CounterColumn]::new("RdPop", "Cluster Storage Hybrid Disks", @("Cache Populate Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+    $c.Add([CounterColumn]::new("WrPop", "Cluster Storage Hybrid Disks", @("Cache Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+    $c.Add([CounterColumn]::new("Destage (MB/s)", "Cluster Storage Cache Stores", @("Destage Bytes/sec"), 15, '#,#', 0.000001, 'Sum', $true))
+    $c.Add([CounterColumn]::new("Update", "Cluster Storage Cache Stores", @("Update Bytes/sec"), 7, '#,#', 0.000001, 'Sum', $false))
+
+    $c.Add([CounterColumn]::new("Total (Pgs)", "Cluster Storage Cache Stores", @("Cache Pages"), 11, '0.00E+0', 1, 'Sum', $true))
+    $c.Add([CounterColumn]::new("Standby", "Cluster Storage Cache Stores", @("Cache Pages StandBy"), 9, '0.00E+0', 1, 'Sum', $false))
+    $c.Add([CounterColumn]::new("L0", "Cluster Storage Cache Stores", @("Cache Pages StandBy L0"), 9, '0.00E+0', 1, 'Sum', $false))
+    $c.Add([CounterColumn]::new("L1", "Cluster Storage Cache Stores", @("Cache Pages StandBy L1"), 9, '0.00E+0', 1, 'Sum', $false))
+    $c.Add([CounterColumn]::new("L2", "Cluster Storage Cache Stores", @("Cache Pages StandBy L2"), 9, '0.00E+0', 1, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Dirty", "Cluster Storage Cache Stores", @("Cache Pages Dirty"), 9, '0.00E+0', 1, 'Sum', $false))
+
+    $c.Seal()
+    $allctrs += $c
+
+    ###
+
+    foreach ($subset in '','Local','Remote') {
+        $name = 'SBL'
+        $prefix = ''
+        if ($subset.Length) {
+            $name += " $subset"
+            $prefix = "$($subset): "
+        }
+
+        $c = [CounterColumnSet]::new($name)
+
+        $c.Add([CounterColumn]::new("IOPS", "Cluster Disk Counters", @(($prefix + "Read/sec"),($prefix + "Writes/sec")), 12, '#,#', 1, 'Sum', $false))
+        $c.Add([CounterColumn]::new("Reads", "Cluster Disk Counters", @($prefix + "Read/sec"), 12, '#,#', 1, 'Sum', $false))
+        $c.Add([CounterColumn]::new("Writes", "Cluster Disk Counters", @($prefix + "Writes/sec"), 12, '#,#', 1, 'Sum', $false))
+
+        $c.Add([CounterColumn]::new("BW (MB/s)", "Cluster Disk Counters", @(($prefix + "Read - Bytes/sec"),($prefix + "Write - Bytes/sec")), 13, '#,#', 0.000001, 'Sum', $true))
+        $c.Add([CounterColumn]::new("Read", "Cluster Disk Counters", @($prefix + "Read - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+        $c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @($prefix + "Write - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+        $c.Add([CounterColumn]::new("Read Lat (ms)", "Cluster Disk Counters", @($prefix + "Read Latency"), 15, '0.000', 1000, 'Average', $true))
+        $c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @($prefix + "Write Latency"), 8, '0.000', 1000, 'Average', $false))
+
+        $c.Add([CounterColumn]::new("Read QAvg", "Cluster Disk Counters", @($prefix + "Read Avg. Queue Length"), 11, '0.000', 1, 'Average', $true))
+        $c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @($prefix + "Write Avg. Queue Length"), 8, '0.000', 1, 'Average', $false))
+
+        $c.Seal()
+        $allctrs += $c
+    }
+
+    ###
+    $c = [CounterColumnSet]::new("SMB SRV")
+
+    $c.Add([CounterColumn]::new("IOPS", "SMB Server Shares", @("Data Requests/sec"), 12, '#,#', 1, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Reads", "SMB Server Shares", @("Read Requests/sec"), 12, '#,#', 1, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Writes", "SMB Server Shares", @("Write Requests/sec"), 12, '#,#', 1, 'Sum', $false))
+
+    $c.Add([CounterColumn]::new("Data BW (MB/s)", "SMB Server Shares", @("Data Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
+    $c.Add([CounterColumn]::new("Read", "SMB Server Shares", @("Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Write", "SMB Server Shares", @("Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+    $c.Add([CounterColumn]::new("Total BW (MB/s)", "SMB Server Shares", @("Transferred Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
+    $c.Add([CounterColumn]::new("Rcv", "SMB Server Shares", @("Received Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Snd", "SMB Server Shares", @("Sent Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+    $c.Seal()
+    $allctrs += $c
+
+    ##
+    $c = [CounterColumnSet]::new("S2D BW")
+
+    $c.Add([CounterColumn]::new("CSV (MB/s)", "Cluster CSVFS", @("Read Bytes/sec","Write Bytes/sec"), 10, '#,#', 0.000001, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Read", "Cluster CSVFS", @("Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Write Bytes/sec"), 8 ,'#,#', 0.000001, 'Sum', $false))
+
+    $c.Add([CounterColumn]::new("SBL (MB/s)", "Cluster Disk Counters", @("Read - Bytes/sec","Write - Bytes/sec"), 10, '#,#', 0.000001, 'Sum', $true))
+    $c.Add([CounterColumn]::new("Read", "Cluster Disk Counters", @("Read - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @("Write - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+    $c.Add([CounterColumn]::new("Cache (MB/s)", "Cluster Storage Hybrid Disks", @("Cache Hit Read Bytes/sec","Cache Write Bytes/sec"), 12, '#,#', 0.000001, 'Sum', $true))
+    $c.Add([CounterColumn]::new("Read", "Cluster Storage Hybrid Disks", @("Cache Hit Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Write", "Cluster Storage Hybrid Disks", @("Cache Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+    $c.Add([CounterColumn]::new("Disk (MB/s)", "Cluster Storage Hybrid Disks", @("Disk Read Bytes/sec","Disk Write Bytes/sec"), 11, '#,#', 0.000001, 'Sum', $true))
+    $c.Add([CounterColumn]::new("Read", "Cluster Storage Hybrid Disks", @("Disk Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Write", "Cluster Storage Hybrid Disks", @("Disk Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+    $c.Seal()
+    $allctrs += $c
+
+    ##
+    $c = [CounterColumnSet]::new("Hyper-V LCPU")
+    $c.Add([CounterColumn]::new("Logical Total%", "Hyper-V Hypervisor Logical Processor", @("% Total Run Time"), 8, "0.00", 1, 'AverageAggregate', $false))
+    $c.Add([CounterColumn]::new("Guest%", "Hyper-V Hypervisor Logical Processor", @("% Guest Run Time"), 8, "0.00", 1, 'AverageAggregate', $false))
+    $c.Add([CounterColumn]::new("Hypervisor%", "Hyper-V Hypervisor Logical Processor", @("% Hypervisor Run Time"), 13, "0.00", 1, 'AverageAggregate', $false))
+
+    $c.Add([CounterColumn]::new("Root Total%", "Hyper-V Hypervisor Root Virtual Processor", @("% Total Run Time"), 12, "0.00", 1, 'AverageAggregate', $true))
+    $c.Add([CounterColumn]::new("Guest%", "Hyper-V Hypervisor Root Virtual Processor", @("% Guest Run Time"), 8, "0.00", 1, 'AverageAggregate', $false))
+    $c.Add([CounterColumn]::new("Hypervisor%", "Hyper-V Hypervisor Root Virtual Processor", @("% Hypervisor Run Time"), 12, "0.00", 1, 'AverageAggregate', $false))
+    $c.Add([CounterColumn]::new("Remote%", "Hyper-V Hypervisor Root Virtual Processor", @("% Remote Run Time"), 7, "0.00", 1, 'AverageAggregate', $false))
+
+    $c.Seal()
+    $allctrs += $c
+
+    ##
+    $c = [CounterColumnSet]::new("SMB Transport")
+    $c.add([CounterColumn]::new("Read IOPS", "SMB Client Shares", @("Read Requests/sec"), 11, "#,#", 1, 'Sum', $false))
+    $c.add([CounterColumn]::new("Write", "SMB Client Shares", @("Write Requests/sec"), 8, "#,#", 1, 'Sum', $false))
+
+    $c.add([CounterColumn]::new("RDMA Read", "SMB Client Shares", @("Read Requests transmitted via SMB Direct/sec"), 11, "#,#", 1, 'Sum', $true))
+    $c.add([CounterColumn]::new("Write", "SMB Client Shares", @("Write Requests transmitted via SMB Direct/sec"), 8, "#,#", 1, 'Sum', $false))
+
+    $c.Seal()
+    $allctrs += $c
+
+    ##
+    if ($Sets.Count -eq 1 -and $Sets[0] -eq '*') {
+        $ctrs = $allctrs
+    } else {
+        $ctrs = $Sets |% {
+            $s = $_
+            $allctrs |? { $_.name -like $s } # allows the SBL* wildcard
+        }
+    }
+
+    function start-sample(
+        [CounterColumnSet[]] $ctrs,
+        [int] $SampleInterval
+        )
+    {
+        # clear any previous job instance
+        Get-Job -Name watch-cluster -ErrorAction SilentlyContinue | Stop-Job
+        Get-Job -Name watch-cluster -ErrorAction SilentlyContinue | Remove-Job
+
+        # flatten list of counters and uniquify for the total counter set
+        # some display counter sets may repeat specific values (which is fine)
+        $counters = ($ctrs.counters |% { $_ |% { $_ }} | group -NoElement).Name
+
+        icm -AsJob -JobName watch-cluster (Get-ClusterNode -Cluster $Cluster) {
+
+            # extract countersamples, the object does not survive transfer between powershell sessions
+            # extract as a list, not as the individual counters
+            get-counter -Continuous -SampleInterval $using:SampleInterval $using:ctrs.counters |% {
+                ,$_.countersamples
+            }
+        }
+    }
+
+    # start the first sample job and allow frame draw the first time through
+    $j = start-sample $ctrs $SampleInterval
+    $downtime = $null
+    $skipone = $false
+    $loops = 0
+    $restart = $false
+
+    # hash of most recent samples/node
+    $samples = @{}
+    Get-ClusterNode -Cluster $Cluster |% { $samples[$_.Name] = $null }
+
+    while ($true) {
+
+        if (-not $restart) {
+            Start-Sleep -Seconds $SampleInterval
+
+            # sleep again if needed to prime the sample pipeline;
+            # there are no samples if we just restarted the sampling jobs
+            if ($skipone) {
+                $skipone = $false
+                continue
+            }
+
+            # receive updates into the per-node hash
+            foreach ($child in $j.ChildJobs) {
+                $samples[$child.Location] = $child | receive-job -ErrorAction SilentlyContinue
+            }
+
+            # null out downed nodes and remember first time we saw one drop out
+            $down = 0
+            $j.ChildJobs |? State -ne Running |% {
+                $samples[$_.Location] = $null
+                $down += 1
+            }
+            if ($down -and $null -eq $downtime) {
+                $downtime = get-date
+            }
+
+            # if everything is down, we will attempt restart
+            if ($down -eq $j.ChildJobs.Count) {
+                $restart = $true
+                break
+            }
+        }
+
+        # if explicit restart is required, or it has been 30 seconds with a downed node, restart the jobs to retry
+        if ($restart -or ($null -ne $downtime -and ((get-date)-$downtime).totalseconds -gt 30)) {
+            $j | stop-job
+            $j | remove-job
+            $j = start-sample $ctrs $SampleInterval
+
+            # force gc to clear out accumulated job state quickly
+            [system.gc]::Collect()
+
+            $downtime = $null
+            $skipone = $true
+            $restart = $false
+            continue
+        }
+
+        # now process samples into per-node hashes of set/ctr containing lists of the
+        # cooked values acrosss the (possible) multiple instances
+        $psamples = @{}
+        foreach ($node in $samples.keys) {
+
+            if ($samples[$node]) {
+
+                $nsamples = @{}
+
+                # flatten samples - if we are lagging, we'll have a list
+                # of consecutive (increasing by timestamp) samples
+                # we could try to be more efficient by dumping all but the
+                # final sample, but later ...
+                $samples[$node] |% { $_ } |% {
+
+                    ($setinst,$ctr) = $($_.path -split '\\')[3..4]
+                    $set = ($setinst -split '\(')[0]
+
+                    $k = "$set+$ctr"
+                    $nsamples[$k] = $_.cookedvalue
+                }
+
+                $psamples[$node] = $nsamples
+            }
+        }
+
+        # post-process the samples into the counterset, then clear and dump
+        $ctrs.DisplayPre($samples, $psamples)
+        Clear-Host
+        $drawsep = $false
+        $ctrs |% {
+            if ($drawsep) {
+                write-host -fore Green $('-'*20)
+            }
+            $drawsep = $true
+            $_.Display()
+
+        }
+
+        # restart the jobs every so many loops to prevent resource growth
+        $loops += 1
+        if ($loops -gt 100) {
+            $loops = 0
+            $restart = $true
+        }
+    }
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/README.md

@@ -0,0 +1,47 @@
+## VM Fleet ##
+
+These are the historical release comments for VM Fleet 1.0.
+
+VM Fleet 0.9 10/2017 (minor)
+
+* watch-cpu: now provides total normalized cpu utility (accounting for turbo/speedstep)
+* sweep-cputarget: now provides average CSV FS read/write latency in the csv
+
+VM Fleet 0.8 6/2017
+
+* get-cluspc: add SMB Client/Server and SMB Direct (not defaulted in Storage group yet)
+* test-clusterhealth: flush output pipeline for Debug-StorageSubsystem output
+* watch-cluster: restart immediately if all child jobs are no longer running
+* watch-cpu: new, visualizer for CPU core utilization distributions
+
+VM Fleet 0.7 3/2017
+
+* create/destroy-vmfleet & update-csv: don't rely on the csv name containing the friendlyname of the vd
+* create-vmfleet: err if basevhd inaccessible
+* create-vmfleet: simplify call-throughs using $using: syntax
+* create-vmfleet: change vhdx layout to match scvmm behavior of seperate directory per VM (important for ReFS MRV)
+* create-vmfleet: use A1 VM size by default (1VCPU 1.75GiB RAM)
+* start-vmfleet: try starting "failed" vms, usually works
+* set-vmfleet: add support for -SizeSpec <Azure Size Specs> for A/D/D2v1 & v2 size specification, for ease of reconfig
+* stop-vmfleet: pass in full namelist to allow best-case internal parallelization of shutdown
+* sweep-cputarget: use %Processor Performance to rescale utilization and account for Turbo effects
+* test-clusterhealth: support cleaning out dumps/triage material to simplify ongoing monitoring (assume they're already gathered/etc.)
+* test-clusterhealth: additional triage output for storport unresponsive device events
+* test-clusterhealth: additional triage comments on SMB client connectivity events
+* test-clusterhealth: new test for Mellanox CX3/CX4 error counters that diagnose fabric issues (bad cable/transceiver/roce specifics/etc.)
+* get-log: new triage log gatherer for all hv/clustering/smb event channels
+* get-cluspc: new cross-cluster performance counter gatherer
+* remove run-<>.ps1 scripts that were replaced with run-demo-<>.ps1
+* check-outlier: EXPERIMENTAL way to ferret out outlier devices in the cluster, using average sampled latency
+
+VM Fleet 0.6 7/18/2016
+
+* CPU Target Sweep: a sweep script using StorageQoS and a linear CPU/IOPS model to build an empirical sweep of IOPS as a function of CPU, initially for the three classic small IOPS mixes (100r, 90:10 and 70:30 4K). Includes an analysis script which provides the linear model for each off of the results.
+* Update sweep mechanics which allow generalized specification of DISKSPD sweep parameters and host performance counter capture.
+* install-vmfleet to automate placement after CSV/VD structure is in place (add path, create dirs, copyin, pause)
+* add non-linearity detection to analyze-cputarget
+* get-linfit is now a utility script (produces objects describing fits)
+* all flag files (pause/go/done) pushed down to control\flag directory
+* demo scripting works again and autofills vm/node counts
+* watch-cluster handles downed/recovered nodes gracefully
+* update-csv now handles node names which are logical prefixes of another (node1, node10)

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/analyze-cputarget.ps1

@@ -0,0 +1,79 @@
+param(
+    [string] $csvfile = $(throw "please provide the path to a cputarget sweep result file"),
+    [switch] $zerointercept = $false,
+    [int] $sigfigs = 5
+    )
+
+function get-sigfigs(
+    [decimal]$value,
+    [int]$sigfigs
+    )
+{
+    $log = [math]::Ceiling([math]::log10([math]::abs($value)))
+    $decimalpt = $sigfigs - $log
+
+    # if all sigfigs are above the decimal point, round off to
+    # appropriate power of 10
+    if ($decimalpt -lt 0) {
+        $pow = [math]::Abs($decimalpt)
+        $decimalpt = 0
+        $value = [math]::Round($value/[math]::Pow(10,$pow))*[math]::pow(10,$pow)
+    }
+
+    "{0:F$($decimalpt)}" -f $value
+}
+
+write-host -ForegroundColor Green CPU Target Sweep Report`n
+write-host -ForegroundColor Green The following equations and coefficients are the linear
+write-host -ForegroundColor Green fit to the measured results at the given write ratios.
+write-host -ForegroundColor Cyan ("-"*20)
+
+write-host -ForegroundColor Yellow NOTE: take care that these formula are only used to reason about
+write-host -ForegroundColor Yellow "      " the region where these values are in a linear relationship.
+write-host -ForegroundColor Yellow "       In particular, at high AVCPU the system may be saturated."
+write-host -ForegroundColor Yellow "Use R^2 (coefficient of determination) as a quality check for the fit."
+write-host -ForegroundColor Yellow "Values close to 100% mean that the data is indeed linear. If R2 is"
+write-host -ForegroundColor Yellow "significantly less than 100%, a closer look at system behavior may"
+write-host -ForegroundColor Yellow "be required."
+
+if ($zerointercept) {
+    write-host -ForegroundColor Red NOTE: forcing to a "(AVCPU=0,IOPS=0)" intercept may introduce error
+} else {
+    write-host -ForegroundColor Red "NOTE: with a non-zero constant coefficient, care should be used at`nlow AVCPU that the result is meaningful"
+}
+
+# do the check fit of QoS to IOPS
+# this will let us check for non-CPU limited saturation (poor r2 is a giveaway)
+# we can have an excellent CPU->IOPS fit but not actually have been able to stress in much CPU
+# and have lots of repeated measurements as we tried to step up QoS
+$h = @{}
+get-linfit -csvfile $csvfile -xcol QOS -ycol IOPS -idxcol WriteRatio -zerointercept:$zerointercept |% {
+    $h[$_.Key] = $_
+}
+
+# now fit IOPS to Average CPU
+get-linfit -csvfile $csvfile -xcol AVCPU -ycol IOPS -idxcol WriteRatio -zerointercept:$zerointercept | sort -Property Key |% {
+
+    write-host -ForegroundColor Cyan ("-"*20)
+    write-host $_.Key
+
+    if ($zerointercept) {
+        write-host ("{0} = {1}({2})" -f $_.Y,$(get-sigfigs $_.B $sigfigs),$_.X)
+    } else {
+
+        if ($_.A -ge 0) {
+            $sign = "+"
+        } else {
+            $sign = ""
+        }
+
+        write-host ("{0} = {1}({2}){4}{3}" -f $_.Y,$(get-sigfigs $_.B $sigfigs),$_.X,$(get-sigfigs $_.A $sigfigs),$sign)
+    }
+
+    write-host ("N = {1}`nR^2 goodness of fit {0:P2}" -f $_.R2,$_.N)
+    if ($h[$_.Key].R2 -le 0.5) {
+        write-host -ForegroundColor Yellow "WARNING: for $($_.Key) it does not appear that IOPS moved in"
+        write-host -ForegroundColor Yellow "`trelation to attempts to raise the QoS limit. Check if the"
+        write-host -ForegroundColor Yellow "`tsystem is storage limited for this mix."
+    }
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/check-outlier.ps1

@@ -0,0 +1,201 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param (
+    [int] $interval = 10
+    )
+
+class RunningStat {
+
+    # This is an implementation of an online (add one value at a time) method to calculate
+    # up to the fourth central moment (mean, variance, skewness and kurtosis) of a series
+    # of decimal values.
+    #
+    # Implementation is due to https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
+    # which is in turn due http://people.xiph.org/~tterribe/notes/homs.html by Timothy Terriberry,
+    # also cited by Pebay: http://prod.sandia.gov/techlib/access-control.cgi/2008/086212.pdf
+    #
+    # Note: skewness appears to match Excel calculations. Kurtosis does NOT at this time, and
+    # should be used with caution.
+
+    [decimal] $n;
+    [decimal] $M1;
+    [decimal] $M2;
+    [decimal] $M3;
+    [decimal] $M4;
+    [decimal] $Min;
+    [decimal] $Max;
+    
+    RunningStat()
+    {
+        $this.M1 = 0
+        $this.M2 = 0
+        $this.M3 = 0
+        $this.M4 = 0
+        $this.Min = 0
+        $this.Max = 0
+    }
+
+    [void] Clear()
+    {
+        $this.M1 = 0
+        $this.M2 = 0
+        $this.M3 = 0
+        $this.M4 = 0
+        $this.Min = 0
+        $this.Max = 0
+    }
+
+    [void] Add([decimal] $v)
+    {
+        $n1 = $this.n
+        $this.n += 1
+        $delta = $v - $this.M1
+        $deltan = $delta / $this.n
+        $deltan2 = $deltan * $deltan
+        $term1 = $delta * $deltan * $n1
+        $this.M1 += $deltan
+        $this.M4 += $term1 * $deltan2 * (($this.n * $this.n) - (3 * $this.n) + 3) + (6 * $deltan2 * $this.M2) - (4 * $deltan * $this.M3)
+        $this.M3 += $term1 * $deltan * ($this.n - 2) - (3 * $deltan * $this.M2)
+        $this.M2 += $term1
+
+        if ($n1 -eq 0 -or $v -gt $this.Max) { $this.Max = $v }
+        if ($n1 -eq 0 -or $v -lt $this.Min) { $this.Min = $v }
+    }
+
+    [decimal] Min()
+    {
+        return $this.Min
+    }
+
+    [decimal] Max()
+    {
+        return $this.Max
+    }
+
+    [decimal] Mean()
+    {
+        return $this.M1
+    }
+
+    [decimal] Variance()
+    {
+        return $this.M2/($this.n - 1)
+    }
+
+    [decimal] StdDev()
+    {
+        return [math]::Sqrt($this.Variance())
+    }
+
+    [decimal] Skew()
+    {
+        return [math]::Sqrt($this.n) * $this.M3 / [math]::Pow($this.M2, 1.5)
+    }
+
+    [decimal] Kurtosis()
+    {
+        return (($this.n * $this.M4) / ($this.M2 * $this.M2)) - 3
+    }
+}
+
+# populate a hash by sbl device number
+$dev = gwmi -Namespace root\wmi ClusPortDeviceInformation
+$devhash = @{}
+$dev |% { $devhash[[int]$_.devicenumber] = $_ }
+
+function get-outliers(
+    [string[]] $paths
+    )
+{
+    # filters and labels for the sigma buckets
+    $sigflt = @(@(">5", '$sig -gt 5'),
+      @("4-5", '$sig -gt 4 -and $sig -le 5'),
+      @("3-4", '$sig -gt 3 -and $sig -le 4')) |% {
+
+        new-object -TypeName psobject -Property @{
+            Label = $_[0];
+            Test= $_[1];
+        }           
+    }
+
+    $ctrs = (get-counter -ComputerName (get-clusternode |? State -eq Up) -SampleInterval $interval -Counter $($paths |% { "\Cluster Disk Counters(*)\$_" })).countersamples |? { $_.instancename -ne "_total" }
+    $stat = [RunningStat]::new()
+
+    foreach ($path in $paths) {
+
+        # select out the specific subset of the counters (read, write, etc.)
+        $ctr = $ctrs |? { $_.Path -like "*$path" }
+
+        $stat.Clear()
+        $ctr.CookedValue |% { $stat.Add($_) }
+        $stddev = $stat.StdDev()
+        $mean = $stat.Mean()
+
+        # hash of flagged devices
+        $flagged = @{}
+
+        write-host -ForegroundColor Green ("-"*20)
+        write-host -ForegroundColor green Sample: $path
+        write-host Number of measured devices: $ctr.count
+        write-host Average of $("$path : {0:F3}ms" -f ($mean*1000))
+        write-host Standard Deviation of $("{0:F3}ms" -f ($stddev*1000))
+
+        # enumerate from highest to lowest deviation of merit
+        foreach ($sigma in $sigflt) {
+
+            # get new outliers at this deviation
+            $outlier = $ctr | sort -property InstanceName |? {
+                if ($stddev -eq 0) { $sig = 0 } else { $sig = (($_.CookedValue -$mean)/$stddev) }
+                -not $flagged[$_.InstanceName] -and (iex $sigma.Test)
+            }
+
+            if ($outlier) {
+
+                write-host -fore Yellow $sigma.Label sigma : $outlier.count total
+                $outlier |% {
+
+                    # remember we flagged this device already
+                    $flagged[$_.InstanceName] = 1
+
+                    $thisdev = $devhash[[int]$_.InstanceName]
+
+                    # parse source node
+                    # \\foo\... -> 0 1 2=foo 3 ...
+                    $sourcenode = ($_.Path -split '\\')[2]
+
+                    write-host -ForegroundColor Red $sourcenode SSB Device: $_.InstanceName`n`tAverage of $path $("{0:F3}ms ({1:F1} sigma)" -f ($_.CookedValue * 1000),(($_.CookedValue -$mean)/$stddev))
+                    write-host -ForegroundColor Red "`tConnected Node:" $thisdev.ConnectedNode
+                    write-host -ForegroundColor Red "`tConnected Node Local PhysicalDisk Number:" $thisdev.ConnectedNodeDeviceNumber
+                    write-host -ForegroundColor Red "`tModel | SerialNumber: $($thisdev.ProductId)` | $($thisdev.SerialNumber)"
+                }
+            }
+        }
+    }
+}
+
+get-outliers -paths "Remote: Read Latency","Local: Read Latency","Remote: Write Latency","Local: Write Latency"

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/check-pause.ps1

@@ -0,0 +1,90 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+# script to check the state/health of pause in the fleet
+#
+# isactive - whether there is an active pause (independent of health)
+# !isactive - whether there is an active pause and all VMs have responded to it
+#
+# true/false return
+
+param(
+    [switch] $isactive = $false
+    )
+
+$pause = "C:\ClusterStorage\collect\control\flag\pause"
+$pauseepoch = gc $pause -ErrorAction SilentlyContinue
+
+if ($pauseepoch -eq $null) {
+    write-host -fore red Pause not in force
+    return $false
+}
+
+# if only performing the pause active check, done
+if ($isactive) {
+    return $true
+}
+
+# accumulate hash of pause flags mapped to current/stale state
+$h = @{}
+
+dir $pause-* |% {
+
+    $thispause = gc $_ -ErrorAction SilentlyContinue
+    if ($thispause -eq $pauseepoch) {
+        $pausetype = 'Current'
+    } else {
+        $pausetype = 'Stale'
+    }
+
+    if ($_.name -match 'pause-(.+)\+(vm.+)') {
+        # 1 is CSV, 2 is VM name
+        $h[$matches[2]] = $pausetype
+    } else {
+        write-host -fore red ERROR: malformed pause $_.name present
+    }
+}
+
+# now correlate to online vms and see if we agree all online are paused.
+# note that if we shutdown some vms and then check pause the current flags
+# will be higher than online, so we need to verify individually to not
+# spoof ourselves.
+
+$vms = get-clustergroup |? GroupType -eq VirtualMachine |? Name -like 'vm-*' |? State -eq Online
+
+$pausedvms = $vms |? { $h[$_.Name] -eq 'Current' }
+
+if ($pausedvms.Count -eq $vms.Count) {
+    write-host -fore green OK: All $vms.count VMs paused
+} else {
+    write-host -fore red WARNING: of "$($vms.Count)," still waiting on ($vms.Count - $pausedvms.Count) to acknowledge pause
+
+    compare-object $vms $pausedvms -Property Name -PassThru | sort -Property Name | ft -AutoSize Name,OwnerNode | Out-Host
+    return $false
+}
+
+return $true

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/check-vmfleet.ps1

@@ -0,0 +1,126 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param ($group = '*')
+
+$g = Get-ClusterGroup |? GroupType -eq VirtualMachine |? Name -like "vm-$group-*"
+
+
+###########################################################
+write-host -fore green State Pivot
+$g | group -Property State -NoElement | sort -Property Name | ft -autosize
+
+###########################################################
+write-host -fore green Host Pivot
+$g | group -Property OwnerNode,State -NoElement | sort -Property Name | ft -autosize
+
+###########################################################
+write-host -fore green Group Pivot
+$g |% {
+    if ($_.Name -match "^vm-([^-]+)-") {
+        $_ | add-member -NotePropertyName Group -NotePropertyValue $matches[1] -PassThru
+    }
+} | group -Property Group,State -NoElement | sort -Property Name | ft -AutoSize
+
+###########################################################
+write-host -fore green IOPS Pivot
+# build 5 orders of log steps
+$logstep = 10,20,50
+$log = 1
+$logs = 1..5 |% {
+
+    $logstep |% { $_ * $log }
+    $log *= 10
+}
+
+# build log step names; 0 is the > range catchall
+$lognames = @{}
+foreach ($step in $logs) {
+
+    if ($step -eq $logs[0]) {
+        $lognames[$step] = "< $step"
+    } else {
+        $lognames[$step] = "$pstep - $($step - 1)"
+    }
+    $pstep = $step
+}
+$lognames[0] = "> $($logs[-1])"
+
+# now bucket up VMs by flow rates
+$qosbuckets = @{}
+$qosbuckets[0] = 0
+$logs |% {
+    $qosbuckets[$_] = 0
+}
+
+Get-StorageQoSFlow |% {
+    
+    $found = $false
+    foreach ($step in $logs) {
+
+        if ($_.InitiatorIops -lt $step) {
+            $qosbuckets[$step] += 1;
+            $found = $true
+            break
+        }
+    }
+    # if not bucketed, it is greater than range
+    if (-not $found) {
+        $qosbuckets[0] += 1
+    }
+}
+
+# find min/max buckets with nonzero counts, by $logs index
+# this lets us present a continuous range, with interleaved zeroes
+$bmax = -1
+$bmin = -1
+foreach ($i in 0..($logs.Count - 1)) {
+    if ($qosbuckets[$logs[$i]]) {
+    
+        if ($bmin -lt 0) {
+            $bmin = $i
+        }
+        $bmax = $i
+    }
+}
+# raise max if we have > range
+if ($qosbuckets[0]) {
+    $bmax = $logs.Count - 1
+}
+$range = @($logs[$bmin..$bmax])
+# add > range if needed, at end
+if ($qosbuckets[0]) {
+    $range += 0
+}
+
+$(foreach ($i in $range) {
+
+    New-Object -TypeName psobject -Property @{
+        Count = $qosbuckets[$i];
+        IOPS = $lognames[$i]
+    }    
+}) | ft Count,IOPS

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/clear-pause.ps1

@@ -0,0 +1,38 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+$pause = "C:\ClusterStorage\collect\control\flag\pause"
+
+if (gi $pause -ErrorAction SilentlyContinue) {
+    write-host -fore green Clearing pause from $([string](gi $pause).LastWriteTime)
+    do {
+        del $pause -Force -ErrorAction SilentlyContinue
+    } while (-not $?)
+    # del $pause-* -ErrorAction SilentlyContinue
+} else {
+    write-host -fore yellow Pause not set
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/create-vmfleet.ps1

@@ -0,0 +1,366 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    [string]$BaseVHD = $(throw "please specify a base vhd"),
+    [int]$VMs = $(throw "please specify a number of vms per node csv"),
+    [string[]]$Groups = @(),
+    [string]$AdminPass = $(throw 'need admin password for autologin'),
+    [string]$Admin = 'administrator',
+    [string]$ConnectPass = $(throw 'need password for loopback host connection'),
+    [string]$ConnectUser = $(throw 'need username for loopback host connection'),
+    [validateset('CreateVMSwitch','CopyVHD','CreateVM','CreateVMGroup','AssertComplete')][string]$StopAfter,
+    [validateset('Force','Auto','None')][string]$Specialize = 'Auto',
+    [switch]$FixedVHD = $true,
+    [string[]]$Nodes = @()
+    )
+
+function Stop-After($step)
+{
+    if ($stopafter -eq $step) {
+        write-host -ForegroundColor Green Stop after $step
+        return $true
+    }
+    return $false
+}
+
+##################
+
+# validate existence of basevhd
+if (!(test-path -path $BaseVHD)) {
+    throw "Base VHD $BaseVHD not found"
+}
+
+if (Get-ClusterNode |? State -ne Up) {
+    throw "not all cluster nodes are up; please address before creating vmfleet"
+}
+
+# if no nodes specified, use the entire cluster
+if ($nodes.count -eq 0) {
+    $nodes = Get-ClusterNode
+}
+
+# convert to fixed vhd(x) if needed
+if ((get-vhd $basevhd).VhdType -ne 'Fixed' -and $fixedvhd) {
+                                    
+    # push dynamic vhd to tmppath and place converted at original
+    # note that converting a dynamic will leave a sparse hole on refs
+    # this is OK, since the copy will not copy the hole
+    $f = gi $basevhd
+    $tmpname = "tmp-$($f.Name)"
+    $tmppath = join-path $f.DirectoryName $tmpname
+    del -Force $tmppath -ErrorAction SilentlyContinue
+    ren $f.FullName $tmpname
+
+    write-host -ForegroundColor Yellow "convert $($f.FullName) to fixed via $tmppath"
+    convert-vhd -Path $tmppath -DestinationPath $f.FullName -VHDType Fixed
+    if (-not $?) {
+        ren $tmppath $f.Name
+        throw "ERROR: could not convert $($f.fullname) to fixed vhdx"
+    }
+    
+    del $tmppath
+}
+
+# Create the fleet vmswitches with a fixed IP at the base of the APIPA range
+icm $nodes {
+
+    if (-not (Get-VMSwitch -Name Internal -ErrorAction SilentlyContinue)) {
+    
+        New-VMSwitch -name Internal -SwitchType Internal
+        Get-NetAdapter |? DriverDescription -eq 'Hyper-V Virtual Ethernet Adapter' |? Name -eq 'vEthernet (Internal)' | New-NetIPAddress -PrefixLength 16 -IPAddress '169.254.1.1'
+    }
+} | ft -AutoSize
+
+#### STOPAFTER
+if (Stop-After "CreateVMSwitch") {
+    return
+}
+
+# create $vms vms per each csv named as <nodename><group prefix>
+# vm name is vm-<group prefix><$group>-<hostname>-<number>
+
+icm $nodes -ArgumentList $stopafter,(Get-Command Stop-After) {
+
+    param( [string]$stopafter,
+           $fn )
+
+    set-item -Path function:\$($fn.name) -Value $fn.definition
+    # workaround evaluation bug and make $stopafter evaluate in the session
+    $null = $stopafter
+
+    function apply-specialization( $path )
+    {
+        # all steps here can fail immediately without cleanup
+
+        # error accumulator
+        $ok = $true
+
+        # create run directory
+
+        del -Recurse -Force z:\run -ErrorAction SilentlyContinue
+        mkdir z:\run
+        $ok = $ok -band $?
+        if (-not $ok) {
+            Write-Error "failed run directory creation for $vhdpath"
+            return $ok
+        }
+
+        # autologon
+        $null = reg load 'HKLM\tmp' z:\windows\system32\config\software
+        $ok = $ok -band $?
+        $null = reg add 'HKLM\tmp\Microsoft\Windows NT\CurrentVersion\WinLogon' /f /v DefaultUserName /t REG_SZ /d $using:admin
+        $ok = $ok -band $?
+        $null = reg add 'HKLM\tmp\Microsoft\Windows NT\CurrentVersion\WinLogon' /f /v DefaultPassword /t REG_SZ /d $using:adminpass
+        $ok = $ok -band $?
+        $null = reg add 'HKLM\tmp\Microsoft\Windows NT\CurrentVersion\WinLogon' /f /v AutoAdminLogon /t REG_DWORD /d 1
+        $ok = $ok -band $?
+        $null = reg add 'HKLM\tmp\Microsoft\Windows NT\CurrentVersion\WinLogon' /f /v Shell /t REG_SZ /d 'powershell.exe -noexit -command c:\users\administrator\launch.ps1'
+        $ok = $ok -band $?
+        $null = [gc]::Collect()
+        $ok = $ok -band $?
+        $null = reg unload 'HKLM\tmp'
+        $ok = $ok -band $?
+        if (-not $ok) {
+            Write-Error "failed autologon injection for $vhdpath"
+            return $ok
+        }
+
+        # scripts
+
+        copy -Force C:\ClusterStorage\collect\control\master.ps1 z:\run\master.ps1
+        $ok = $ok -band $?
+        if (-not $ok) {
+            Write-Error "failed injection of specd master.ps1 for $vhdpath"
+            return $ok
+        }
+
+        del -Force z:\users\administrator\launch.ps1 -ErrorAction SilentlyContinue
+        gc C:\ClusterStorage\collect\control\launch-template.ps1 |% { $_ -replace '__CONNECTUSER__',$using:connectuser -replace '__CONNECTPASS__',$using:connectpass } > z:\users\administrator\launch.ps1
+        $ok = $ok -band $?
+        if (-not $ok) {
+            Write-Error "failed injection of launch.ps1 for $vhdpath"
+            return $err
+        }
+
+        echo $vmspec > z:\vmspec.txt
+        $ok = $ok -band $?
+        if (-not $ok) {
+            Write-Error "failed injection of vmspec for $vhdpath"
+            return $ok
+        }
+
+        # load files
+        $f = 'z:\run\testfile1.dat'
+        if (-not (gi $f -ErrorAction SilentlyContinue)) {
+            fsutil file createnew $f (10GB)
+            $ok = $ok -band $?
+            fsutil file setvaliddata $f (10GB)
+            $ok = $ok -band $?
+        }
+        if (-not $ok) {
+            Write-Error "failed creation of initial load file for $vhdpath"
+            return $ok
+        }
+
+        return $ok
+    }
+
+    function specialize-vhd( $vhdpath )
+    {
+        $vhd = (gi $vhdpath)
+        $vmspec = $vhd.Directory.Name,$vhd.BaseName -join '+'
+
+        # mount vhd and its largest partition
+        $o = Mount-VHD $vhd -NoDriveLetter -Passthru
+        if ($o -eq $null) {
+            Write-Error "failed mount for $vhdpath"
+            return $false
+        }
+        $p = Get-Disk -number $o.DiskNumber | Get-Partition | sort -Property size -Descending | select -first 1
+        $p | Add-PartitionAccessPath -AccessPath Z:
+
+        $ok = apply-specialization Z:
+
+        Remove-PartitionAccessPath -AccessPath Z: -InputObject $p
+        Dismount-VHD -DiskNumber $o.DiskNumber
+
+        return $ok
+    }
+
+    $csvs = Get-ClusterSharedVolume
+
+    # handle restore cases by mapping the csv to the friendly name of the volume
+    # don't rely on the csv name to contain this data
+
+    $vh = @{}
+    Get-Volume |? FileSystem -eq CSVFS |% { $vh[$_.Path] = $_ }
+
+    $csvs |% {
+        $v = $vh[$_.SharedVolumeInfo.Partition.Name] 
+        if ($v -ne $null) {
+            $_ | Add-Member -NotePropertyName VDName -NotePropertyValue $v.FileSystemLabel
+        }
+    }
+
+    foreach ($csv in $csvs) {
+
+        if ($($using:groups).Length -eq 0) {
+            $groups = @( 'base' )
+        } else {
+            $groups = $using:groups
+        }
+
+        # identify the CSvs for which this node should create its VMs
+        # the trailing characters (if any) are the group prefix
+        if ($csv.VDName -match "^$env:COMPUTERNAME(?:-.+){0,1}") {
+
+            foreach ($group in $groups) {
+
+                if ($csv.VDName -match "^$env:COMPUTERNAME-([^-]+)$") {
+                    $g = $group+$matches[1]
+                } else {
+                    $g = $group
+                }
+
+                foreach ($vm in 1..$using:vms) {
+
+                    $stop = $false
+
+                    $newvm = $false
+                    $name = "vm-$g-$env:COMPUTERNAME-$vm"
+                    $path = Join-Path $csv.SharedVolumeInfo.FriendlyVolumeName $name
+
+                    # place vhdx in subdirectory, per scvmm layout defaults
+                    # $vhd = $path+".vhdx"
+                    $vhd = join-path $path "$name.vhdx"
+
+                    # if the vm cluster group exists, we are already deployed
+                    if (-not (Get-ClusterGroup -Name $name -ErrorAction SilentlyContinue)) {
+
+                        if (-not $stop) {
+                            $stop = Stop-After "AssertComplete"
+                        }
+
+                        if ($stop) {
+
+                            Write-Host -ForegroundColor Red "vm $name not deployed"
+
+                        } else {
+
+                            Write-Host -ForegroundColor Yellow "create vm $name @ metadata path $path with vhd $vhd"
+
+                            # create vm if not already done
+                            # note that when restarting interrupted creation, the vm could have moved elsewhere
+                            # under cluster control.
+
+                            $o = Get-ClusterGroup |? GroupType -eq VirtualMachine |? Name -eq $name
+
+                            if (-not $o) {
+
+                                # force re-specialization
+                                $newvm = $true
+
+                                # if the cluster group doesn't exist, we're on the canonical node to create the vm
+                                # if the vm exists, tear it down and refresh
+
+                                $o = get-vm -Name $name -ErrorAction SilentlyContinue
+
+                                if ($o) {
+
+                                    # interrupted between vm creation and role creation; redo it
+                                    write-host "REMOVING vm $name for re-creation"
+
+                                    if ($o.State -ne 'Off') {
+                                        Stop-VM -Name $name -Force -Confirm:$false
+                                    }
+                                    Remove-VM -Name $name -Force -Confirm:$false
+
+                                } else {
+
+                                    # scrub and re-create the vm metadata path and vhd
+                                    rmdir -ErrorAction SilentlyContinue -Recurse $path
+                                    $null = mkdir -ErrorAction SilentlyContinue $path
+
+                                    cp $using:basevhd $vhd
+                                }
+
+                                #### STOPAFTER
+                                if (-not $stop) {
+                                    $stop = Stop-After "CopyVHD"
+                                }
+
+                                if (-not $stop) {
+                                    $o = New-VM -VHDPath $vhd -Generation 2 -SwitchName Internal -Path $path -Name $name
+
+                                    # create A1 VM. use set-vmfleet to alter fleet sizing post-creation.
+                                    $o | Set-VM -ProcessorCount 1 -MemoryStartupBytes 1.75GB -StaticMemory
+
+                                    # do not monitor the internal switch connection; this allows live migration
+                                    $o | Get-VMNetworkAdapter| Set-VMNetworkAdapter -NotMonitoredInCluster $true
+                                }
+                            }
+
+                            #### STOPAFTER
+                            if (-not $stop) {
+                                $stop = Stop-After "CreateVM"
+                            }
+
+                            if (-not $stop) {
+                                
+                                # create clustered vm role and assign default owner node
+                                $o | Add-ClusterVirtualMachineRole
+                                Set-ClusterOwnerNode -Group $o.VMName -Owners $env:COMPUTERNAME
+                            }
+                        }
+
+                    } else {
+                        Write-Host -ForegroundColor Green "vm $name already deployed"
+                    }
+
+                    #### STOPAFTER
+                    if (-not $stop) {
+                        $stop = Stop-After "CreateVMGroup"
+                    }
+
+                    if (-not $stop -or ($using:specialize -eq 'Force')) {
+                        # specialize as needed
+                        # auto only specializes new vms; force always; none skips it
+                        if (($using:specialize -eq 'Auto' -and $newvm) -or ($using:specialize -eq 'Force')) {
+                            write-host -fore yellow specialize $vhd
+                            if (-not (specialize-vhd $vhd)) {
+                                write-host -fore red "Failed specialize of $vhd, halting."
+                            }
+                        } else {
+                            write-host -fore green skip specialize $vhd
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/demo.ps1

@@ -0,0 +1,72 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param([int] $interval = 60)
+
+$cnodes = (Get-ClusterNode).Count
+$cvms = (Get-ClusterGroup |? GroupType -eq 'VirtualMachine').Count/$cnodes
+
+# loop through the set of run scripts
+$f = gi C:\clusterstorage\collect\control\run-demo-*.ps1
+
+# qos policies to loop
+$policy = 'SilverVM','GoldVM','PlatinumVM',$null,$null
+
+while ($true) {
+
+    $null = & update-csv.ps1
+
+    foreach ($run in $f) {
+
+        cls
+
+        # touch the current run file to trigger VM updates
+        # see the master script
+        (gi $run).LastWriteTime = (get-date)
+
+        # pull the show-me line of the run script (comment, trimmed)
+        # format is a single line, with # standing in for newline for multi-line output
+        # substitute in the configuration's node and vms/node count
+        # TBD: autodetect vd configuration
+        $comm = (gc $run | sls '^#-#').Line
+
+        write-host -fore green $($comm.trimstart('#- ') -replace '__CVMS__',$cvms -replace '__CNODES__',$cnodes -replace '#',"`n")
+        
+        # apply random QoS policy
+        $p = $policy[(0..($policy.count - 1) | Get-Random)]
+        $null = & set-storageqos.ps1 -policyname $p 2>&1
+
+        write-host -fore yellow `nActive QoS Policy
+        if ($p) {
+            Get-StorageQosPolicy -Name $p | ft -AutoSize
+        } else {
+            write-host NONE
+        }
+
+        sleep $interval
+    }
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/destroy-vmfleet.ps1

@@ -0,0 +1,102 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    [string[]] $vms = $null
+    )
+
+if ($vms) {
+    write-host -ForegroundColor Yellow "Destroying VM Fleet content for: $vms"
+} else {
+    write-host -ForegroundColor Yellow "Destroying VM Fleet"
+}
+
+# stop and remove clustered vm roles
+write-host -ForegroundColor Green "Removing VM ClusterGroups"
+Get-ClusterGroup |? GroupType -eq VirtualMachine |? {
+    if ($vms) {
+        $_.Name -in $vms
+    } else {
+        $_.Name -like 'vm-*'
+    }
+} |% {
+    write-host "Removing ClusterGroup for $($_.Name)"
+    $_ | Stop-ClusterGroup
+    $_ | Remove-ClusterGroup -RemoveResources -Force
+}
+
+# remove all vms
+write-host -ForegroundColor Green "Removing VMs"
+icm (Get-ClusterNode) {
+    Get-VM |? {
+        if ($using:vms) {
+            $_.Name -in $using:vms
+        } else {
+            $_.Name -like 'vm-*'
+        }
+    } |% {
+        write-host "Removing VM for $($_.Name) @ $($env:COMPUTERNAME)"
+        $_ | Remove-VM -Confirm:$false -Force
+    }
+
+    # do not remove the internal switch if teardown is partial
+    if ($using:vms -eq $null) {
+        write-host -ForegroundColor Green "Removing Internal VMSwitch"
+        Get-VMSwitch -SwitchType Internal | Remove-VMSwitch -Confirm:$False -Force
+    }
+}
+
+# handle restore cases by mapping the csv to the friendly name of the volume
+# don't rely on the csv name to contain this data
+$csv = Get-ClusterSharedVolume
+
+$vh = @{}
+Get-Volume |? FileSystem -eq CSVFS |% { $vh[$_.Path] = $_ }
+
+$csv |% {
+    $v = $vh[$_.SharedVolumeInfo.Partition.Name]
+    if ($v -ne $null) {
+        $_ | Add-Member -NotePropertyName VDName -NotePropertyValue $v.FileSystemLabel
+    }
+}
+
+# now delete content from csvs corresponding to the cluster nodes
+write-host -ForegroundColor Green "Removing CSV content for VMs"
+Get-ClusterNode |% {
+    $csv |? VDName -match "$($_.Name)(-.+)?"
+} |% {
+    dir -Directory $_.sharedvolumeinfo.friendlyvolumename |? {
+        if ($vms) {
+            $_.Name -in $vms
+        } else {
+            $_.Name -like 'vm-*'
+        }
+    } |% {
+        write-host "Removing CSV content for $($_.BaseName) @ $($_.FullName)"
+        del -Recurse -Force $_.FullName
+    }
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/get-cluspc.ps1

@@ -0,0 +1,281 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+##
+## netsh advfirewall firewall set rule group="Performance Logs and Alerts" new enable=yes
+##
+[CmdletBinding( DefaultParameterSetName = "BySeconds" )]
+param(
+
+    [Parameter(
+        ParameterSetName                = 'BySeconds',
+        ValueFromPipeline               = $false,
+        ValueFromPipelineByPropertyName = $false,
+        Mandatory                       = $false)]
+    [ValidateRange(1,[int]::MaxValue)]
+    [int] $Seconds = 1,
+
+    [Parameter(
+        ParameterSetName                = 'ByStart',
+        ValueFromPipeline               = $false,
+        ValueFromPipelineByPropertyName = $false,
+        Mandatory                       = $false)]
+    [switch] $Start,
+    
+    [Parameter(
+        ParameterSetName                = 'ByStop',
+        ValueFromPipeline               = $false,
+        ValueFromPipelineByPropertyName = $false,
+        Mandatory                       = $false)]
+    [switch] $Stop = $false,
+
+    <# --------- common #>
+
+    [Parameter(
+        ParameterSetName                = 'BySeconds',
+        Mandatory                       = $false)]
+    [Parameter(
+        ParameterSetName                = 'ByStart',
+        Mandatory                       = $false)]
+    [Parameter(
+        ParameterSetName                = 'ByStop',
+        Mandatory                       = $false)]
+    [string] $Cluster = ".",
+
+    <# --------- start-time parameters #>
+
+    [Parameter(
+        ParameterSetName                = 'BySeconds',
+        Mandatory                       = $false)]
+    [Parameter(
+        ParameterSetName                = 'ByStart',
+        Mandatory                       = $false)]
+    [ValidateSet('PhysicalDisk','CPU','SMB','SMBD','SSB','SSB Cache','CSVFS','Storage','Spaces')]
+    [string[]] $Set = '*',
+
+    [Parameter(
+        ParameterSetName                = 'BySeconds',
+        Mandatory                       = $false)]
+    [Parameter(
+        ParameterSetName                = 'ByStart',
+        Mandatory                       = $false)]
+    [string] $AddSpec,
+
+    [Parameter(
+        ParameterSetName                = 'BySeconds',
+        Mandatory                       = $false)]
+    [Parameter(
+        ParameterSetName                = 'ByStart',
+        Mandatory                       = $false)]
+    [int] $SampleInterval = 1,
+
+    <# --------- stop-time parameters #>
+
+    [Parameter(
+        ParameterSetName                = 'BySeconds',
+        Mandatory                       = $false)]
+    [Parameter(
+        ParameterSetName                = 'ByStop',
+        Mandatory                       = $false)]
+    [switch] $Force = $false,
+
+    [Parameter(
+        ParameterSetName                = 'BySeconds',
+        Mandatory                       = $true)]
+    [Parameter(
+        ParameterSetName                = 'ByStop',
+        Mandatory                       = $true)]
+    [string] $Destination
+    )
+
+if ($psCmdlet.ParameterSetName -ne 'ByStart' -and
+    (gi -ErrorAction SilentlyContinue $Destination)) {
+
+    if (-not $force) {
+        Write-Error "$Destination already exists, please delete or use -Force to overwrite"
+        return
+    } else {
+        del -ErrorAction SilentlyContinue $Destination
+    }
+}
+
+$sets = @{
+    'PhysicalDisk' = '\PhysicalDisk(*)\*','+getclusport';
+    'CSVFS' = '\Cluster CSVFS(*)\*','\Cluster CSV Volume Cache(*)\*','\Cluster CSV Volume Manager(*)\*','\Cluster CSVFS Block Cache(*)\*','\Cluster CSVFS Direct IO(*)\*','\Cluster CSVFS Redirected IO(*)\*','+getcsv';
+    'SSB' = '\Cluster Disk Counters(*)\*','+getclusport';
+    'SMB' = '\SMB Client Shares(*)\*','\SMB Server Shares(*)\*';
+    'SMBD' = '\SMB Direct Connection(*)\*';
+    'SSB Cache' = '\Cluster Storage Hybrid Disks(*)\*','\Cluster Storage Cache Stores(*)\*';
+    'Spaces' = '\Storage Spaces Write Cache(*)\*','\Storage Spaces Tier(*)\*','\Storage Spaces Virtual Disk(*)\*'
+    'ReFS' = '\ReFS(*)\*';
+    'CPU' = '\Hyper-V Hypervisor Logical Processor(*)\*','\Processor Information(*)\*';
+
+    'Storage' = 'PhysicalDisk','CSVFS','SSB','SSB Cache','ReFS','Spaces';
+}
+
+$special = @{
+    "getclusport" = $false;
+    "getcsv" = $false;
+}
+
+$cleanup = @()
+
+$pc = @()
+if ($Set.count -eq 1 -and $Set[0] -eq '*') {
+    # list of all keys
+    $pc = $sets.keys |% { $_ }
+} else {
+    $pc = $Set
+}
+
+# repeat expansion (sets in sets, possibly containing special collection rules)
+do {
+    $expansion = $false
+    $pc = $pc |% {
+        $n = $_
+        switch ($n[0]) {
+            # performance counter - passthru
+            '\' { $n }
+            # special collection (sets variable -> $true)
+            '+' {
+
+                if ($special.ContainsKey($n.Substring(1))) {
+                    $special[$n.Substring(1)] = $true
+                } else {
+                    throw "unrecognized special gather command $($n.Substring(1))"
+                }
+            }
+            # set expansion
+            default {
+                $sets[$n]
+                $expansion = $true
+            }
+        }
+    }
+} while ($expansion)
+
+# uniq the counters
+$pc = ($pc | group -NoElement).Name
+
+# --
+
+function start-logman(
+    [string] $name,
+    [string[]] $counters,
+    [int] $sampleinterval
+    )
+{
+    $computer = $env:COMPUTERNAME
+    $f = "c:\perfctr-$name-$computer.blg"
+
+    $null = logman create counter "perfctr-$name" -o $f -f bin -si $sampleinterval --v -c $counters
+    $null = logman start "perfctr-$name"
+    write-host "performance counters on: $computer"
+}
+
+function stop-logman(
+    [string] $name
+    )
+{
+    $computer = $env:COMPUTERNAME
+    $f = "c:\perfctr-$name-$computer.blg"
+    
+    $null = logman stop "perfctr-$name"
+    $null = logman delete "perfctr-$name"
+    write-host "performance counters off: $computer"
+
+    echo $("\\$computer\$f" -replace ':','$')
+}
+
+if (-not $Stop) {
+
+    icm (get-clusternode -cluster $Cluster |? State -eq Up) -ArgumentList (get-command start-logman) {
+
+        param($fn)
+        set-item -path function:\$($fn.name) -value $fn.definition
+
+        start-logman $using:AddSpec $using:pc -sampleinterval $using:SampleInterval
+    }
+
+    if ($Start) {
+        write-host -ForegroundColor Yellow INFO: captures started - reinvoke with `-stop to complete capture
+        return
+    }
+
+    sleep $Seconds
+
+} else {
+
+    write-host -ForegroundColor Yellow INFO: completing previously started capture
+}
+
+# now capture all counter files
+$f = @()
+$f += icm (get-clusternode -cluster $Cluster |? State -eq Up) -ArgumentList (get-command stop-logman) {
+
+    param($fn)
+    set-item -path function:\$($fn.name) -value $fn.definition
+
+    stop-logman $using:AddSpec $using:Destination
+}
+
+# add all counter blg to the cleanup step
+$cleanup += $f
+
+#--
+# specials
+#--
+
+# make capture directory, and add to cleanup list
+# note that all specials are generated into this directory,
+# and will be automatically cleaned up when it is deleted
+$t = New-TemporaryFile
+del $t
+$null = md $t
+$cleanup += $t
+
+if ($special['getclusport']) {
+
+    get-clusternode -cluster $Cluster |? State -eq Up |% {
+
+        $exp = "$t\clusport-$_.xml"
+        gwmi -ComputerName $_ -Namespace root\wmi ClusPortDeviceInformation | export-clixml -Path $exp
+        $f += $exp
+    }
+}
+
+if ($special['getcsv']) {
+
+    $exp = "$t\csv.xml"
+    Get-ClusterSharedVolume -cluster $Cluster | export-clixml -Path $exp
+    $f += $exp
+}
+
+compress-archive -DestinationPath $Destination -Path $f
+del -Force -Recurse $cleanup
+
+write-host "performance counters: $Destination"

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/get-linfit.ps1

@@ -0,0 +1,143 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    $csvfile,
+    [string]$delimeter = "`t",
+    [string]$xcol = $(throw "please specify column containing x values"),
+    [string]$ycol = $(throw "please specify column containing y values"),
+    [string[]]$idxcol,
+    [switch]$zerointercept = $false
+    )
+
+# given the input data, produce the linear fit coefficients for
+#
+#     zerointercept = true: y = bx
+#    !zerointercept = true: y = a + bx
+#
+# specify the column containing the x and y measurements
+# a single index column can be used to differentiate rows which should be
+# used for the fit (i.e., "test1", "test2", ...).
+#
+# method: ordinary least squares
+
+function do-linfit(
+    [string] $xcol,
+    [string] $ycol,
+    [string] $key
+    )
+{
+    BEGIN
+    {
+        $sumxy = [decimal]0
+        $sumx2 = [decimal]0
+        $sumx = [decimal]0
+        $sumy = [decimal]0
+
+        $n = 0
+
+        $pipe = @()
+    }
+
+    PROCESS
+    {
+        $x = [decimal]$_.$xcol
+        $y = [decimal]$_.$ycol
+
+        $sumxy += $x*$y
+        $sumx2 += $x*$x
+        $sumx += $x
+        $sumy += $y
+
+        $n += 1
+
+        # accumulate pipeline for second pass
+        $pipe += $_
+    }
+
+    END
+    {
+        if ($n -eq 0) {
+            Write-Error "ERROR: no measurements matched"
+            return
+        }
+
+        # perform requested fit
+
+        $a = 0
+        $b = 0
+
+        if ($zerointercept) {
+
+            $a = 0
+            $b = ($sumxy/$sumx2)
+
+        } else {
+
+            $b = ($sumxy - (($sumx*$sumy)/$n)) / ($sumx2 - (($sumx*$sumx)/$n))
+            $a = ($sumy - $b*$sumx)/$n
+        }
+
+        # calculate r2 (coefficient of determination) with respect to the fit
+        $meany = $sumy/$n
+
+        # total sum of squares
+        $sstot = [decimal]0
+        $pipe |% {
+            $v = [decimal]$_.$ycol - $meany
+            $sstot += $v*$v
+        }
+
+        # residual sum of squares
+        $ssres = [decimal]0
+        $pipe |% {
+            $v = [decimal]$_.$ycol - ($a + $b*[decimal]$_.$xcol)
+            $ssres += $v*$v
+        }
+
+        $r2 = 1 - ($ssres/$sstot)
+
+        new-object -TypeName psobject -Property @{ 'A' = $a; 'B' = $b; 'R2' = $r2; 'N' = $n; 'Key' = $key; 'X' = $xcol; 'Y' = $ycol }
+    }
+}
+
+# process the results into a hash keyed by the index columns
+$h = @{}
+import-csv -Path $csvfile -Delimiter $delimeter |% {
+
+    $key = ""
+    foreach ($col in $idxcol) {
+        $key += "$col $($_.$col)"
+    }
+
+    $h[$key] += ,$_
+}
+
+$h.Keys |% {
+
+    $h[$_] | do-linfit -xcol $xcol -ycol $ycol -key $_
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/get-log.ps1

@@ -0,0 +1,80 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    [string] $zip = $(throw "please provide zip filename for compressed logs"),
+    [ValidateSet("*","HyperV","FailoverClustering","SMB")][string[]] $set = "*",
+    [int] $timespan = 0
+    )
+
+# convert minutes to milliseconds
+if ($timespan -gt 0) {
+    $timespan *= 60*1000
+}
+
+$logs = icm (get-clusternode) {
+
+    $map = @{
+        "HyperV" = "Microsoft-Windows-Hyper-V";
+        "FailoverClustering" = "Microsoft-Windows-FailoverClustering"
+        "SMB" = "Microsoft-Windows-SMB"
+    }
+
+    if ($using:set -eq "*") {
+        $lset = $map.Keys |% { $_ }
+    } else {
+        $lset = $using:set
+    }
+
+$q = @"
+/q:"*[System[TimeCreated[timediff(@SystemTime) <= __TIME__]]]"
+"@
+
+    if ($using:timespan -gt 0) {
+        $timefilt = $q -replace '__TIME__',$using:timespan
+    } else {
+        $timefilt = $null
+    }
+
+    $lset |% {
+
+        # get logs and normalize to legal filenames
+        $prov = wevtutil el | sls $map[$_]
+        $provf = $prov -replace '/','-'
+
+        foreach ($i in 0..($prov.Count - 1)) {
+
+            $localpath = "$($provf[$i])--$env:computername.evtx"
+            del -Force $localpath -ErrorAction SilentlyContinue
+            wevtutil epl $prov[$i] "c:\$localpath" $timefilt
+            write-output "\\$env:computername\c$\$localpath"
+        }
+    }
+}
+
+compress-archive -Path $logs -DestinationPath $zip
+del -force $logs

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/install-vmfleet.ps1

@@ -0,0 +1,74 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    $source = $(throw "Must specify the source directory for the vmfleet scripts")
+    )
+
+$col = get-clustersharedvolume |? Name -match '(collect)'
+if ($col -eq $null) {
+    write-error "The collect CSV is not present"
+    return
+}
+
+$fqsrc = gi $source
+if ($fqsrc -eq $null) {
+    write-error "The source directory for the vmfleet scripts is not accessible"
+}
+
+# update the csv mounts to their normalized names and install scripts and directory structure
+# remove internet-download blocks on ps1 scripting, if present
+& $fqsrc\update-csv.ps1 -renamecsvmounts
+$control = 'C:\ClusterStorage\collect\control'
+
+cp -r $fqsrc $control
+dir $control\*.ps1 |% { Unblock-File $_ }
+
+mkdir $control\result
+mkdir $control\flag
+mkdir $control\tools
+
+
+# put the control directory onto the path
+if (-not ([System.Environment]::GetEnvironmentVariable("path") -split ';' |? { $_ -eq $control })) {
+    $newpath = [System.Environment]::GetEnvironmentVariable("path") + ";$control"
+    [System.Environment]::SetEnvironmentVariable("path",
+                                                 $newpath,
+                                                 [System.EnvironmentVariableTarget]::Process)
+    [System.Environment]::SetEnvironmentVariable("path",
+                                                 $newpath,
+                                                 [System.EnvironmentVariableTarget]::User)
+}
+
+# disable the csv balancer so that motion is under fleet control
+(get-cluster).CsvBalancer = 0
+
+# finally set fleet pause and touch the base run file so that we ensure
+# the fleet will start with it.
+set-pause
+(gi $control\run.ps1).IsReadOnly = $false
+(gi $control\run.ps1).LastWriteTime = (Get-Date)

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/launch-template.ps1

@@ -0,0 +1,34 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+$script = 'c:\run\master.ps1'
+
+while ($true) {
+    Write-Host -fore Green Launching $script `@ $(Get-Date)
+    & $script -connectuser __CONNECTUSER__ -connectpass __CONNECTPASS__
+    sleep -Seconds 1
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/master.ps1

@@ -0,0 +1,225 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+#
+# VM Master Script
+#
+# This script must be executed on autologon by the VM. It establishes a mapping a location containing the Run Script
+# and then repeatedly polls/executes the latest one it finds. This allows the runner to inject new run scripts on the
+# fly and/or touch pre-existing ones to bump them to the top of the list.
+#
+# In the initial form, it assumes the VM is connected by an internal switch and it just polls all the node names to
+# find the one which will work. This could be improved by using specialization to inject the specific path to a
+# configuration file.
+#
+
+param(
+    [string] $connectuser,
+    [string] $connectpass
+    )
+
+$null = net use l: /d
+
+if ($(Get-SmbMapping) -eq $null) {
+    New-SmbMapping -LocalPath l: -RemotePath \\169.254.1.1\c$\clusterstorage\collect\control -UserName $connectuser -Password $connectpass -ErrorAction SilentlyContinue
+}
+
+# update tooling
+cp l:\tools\* c:\run -Force
+
+$run = 'c:\run\run.ps1'
+$master = 'c:\run\master.ps1'
+
+$mypause = "l:\flag\pause-$(gc c:\vmspec.txt)"
+$mydone = "l:\flag\done-$(gc c:\vmspec.txt)"
+
+$done = $false
+$donetouched = $false
+$pause = $false
+
+$tick = 0
+
+function get-newfile(
+    $sourcepat,
+    $dest,
+    $clean = $false,
+    $silent = $false
+    )
+{
+    $sf = dir $sourcepat | sort -Property LastWriteTime -Descending | select -first 1
+    $df = gi $dest -ErrorAction SilentlyContinue
+
+    # no source?
+    if ($sf -eq $null) {
+        write-host -fore green NO source present
+        # clean if needed
+        if ($clean) {
+            rm $dest
+        }
+        # no new file is an update condition
+        $true
+    } elseif ($df -eq $null -or $sf.lastwritetime -gt $df.lastwritetime) {
+        write-host -fore green NEWER source $sf.fullname '=>' $dest
+        # update with newer source file and indicate update condition
+        cp $sf.fullname $dest -Force
+        $true
+    } else {
+        if (-not $silent) {
+            write-host -fore green NO newer source $sf.lastwritetime $sf.fullname '**' $df.LastWriteTime $df.fullname
+        }
+        # already have latest, indicate no new
+        $false
+    }
+}
+
+function get-flagfile(
+    [string] $flag,
+    [switch] $gc = $false
+    )
+{
+    if ($gc) {
+        gc "l:\flag\$flag" -ErrorAction SilentlyContinue   
+    } else {
+        gi "l:\flag\$flag" -ErrorAction SilentlyContinue
+    }
+}
+
+while ($true) {
+
+    # update master control?
+    # assume runners have a simple loop to re-execute on exit
+    if (get-newfile l:\master*.ps1 $master -silent:$true) {
+        break
+    }
+
+    # check and acknowledge pause - only drop flag once
+    $pauseepoch = get-flagfile pause -gc:$true
+    if ($pauseepoch -ne $null) {
+
+        # pause clears done
+        $done = $false
+
+        # drop into pause flagfile if needed
+        if ($pause -eq $false) {
+            write-host -fore red PAUSE IN FORCE
+            $pause = $true
+            echo $pauseepoch > $mypause
+        } else {
+            if ($tick % 10 -eq 0) {
+                write-host -fore red -NoNewline '.'
+            }
+        }
+
+    } elseif ($done) {
+
+        # drop epoch into done flagfile if needed
+        if (-not $donetouched) {
+            echo $goepoch > $mydone
+            $donetouched = $true
+        }
+
+        # if go flag is now different, release for the next go around
+        if ($goepoch -ne (get-flagfile go -gc:$true)) {
+            $done = $false
+            write-host -fore cyan `nReleasing from Done
+        } else {
+            if ($tick % 10 -eq 0) {
+                write-host -fore yellow -NoNewline '.'
+            }
+        }
+
+    } else {
+
+        # clear pause & donetouched flags
+        $pause = $false
+        $donetouched = $false
+
+        # update run script?
+        $null = get-newfile l:\run*.ps1 $run -clean:$true
+        $runf = gi $run -ErrorAction SilentlyContinue
+
+        if ($runf -eq $null) {
+            write-host -fore yellow no control file found
+            sleep 30
+            continue
+        }
+
+        # update go epoch - a change to this (if present) will be what
+        # allows us to proceed past a done flag
+        $goepoch = get-flagfile go -gc:$true
+        if ($goepoch -ne $null) {
+            write-host -fore cyan Go Epoch acknowledged at: $goepoch
+        }
+
+        # acknowledge script
+        write-host -fore cyan Run Script $runf.lastwritetime "`n$("*"*20)"
+        gc $runf
+        write-host -fore cyan ("*"*20)
+
+        # launch and monitor pause and new run file
+        $j = start-job -arg $run { param($run) & $run }
+        while (($jf = wait-job $j -Timeout 1) -eq $null) {
+
+            $halt = $null
+
+            # check pause or new run file: if so, stop and loop
+            if (get-flagfile pause) {
+                $halt = "pause set"
+            }
+
+            if (get-newfile l:\run*.ps1 $run -clean:$true -silent:$true) {
+                $halt = "new run file"
+            }
+
+            if ($halt -ne $null) {
+                write-host -fore yellow STOPPING CURRENT "(reason: $halt)"
+                $j | stop-job
+                $j | remove-job
+                break
+            }
+        }
+
+        # job finished?
+        if ($jf -ne $null) {
+            $result = $jf | receive-job
+
+            if ($result -eq "done") {
+                write-host -fore yellow DONE CURRENT
+                $done = $true
+            }
+
+            $jf | remove-job
+        }
+
+        write-host -fore cyan ("*"*20)
+    }
+
+    # force gc to teardown potentially conflicting handles and enforce min pause
+    [system.gc]::Collect()
+    sleep 1
+    $tick += 1
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/run-demo-100r.ps1

@@ -0,0 +1,39 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+#-# System Config:    __CNODES__ Systems x __CVMS__ VMs/System#Storage Config:   3-way S2D Mirror#Current Workload: 100% 4K Random Read
+
+[string](get-date)
+
+$b = 4
+$t = 8
+$o = 20
+$w = 0
+
+C:\run\diskspd.exe -n -h `-t$t `-o$o `-b$($b)k `-r$($b)k `-w$w -W10 -d60 -C10 -D -L (dir C:\run\testfile?.dat)
+
+[string](get-date)

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/run-demo-7030.ps1

@@ -0,0 +1,38 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+#-# System Config:    __CNODES__ Systems x __CVMS__ VMs/System#Storage Config:   3-way S2D Mirror#Current Workload: 70:30 4K Random Read/Write
+[string](get-date)
+
+$b = 4
+$t = 8
+$o = 20
+$w = 30
+
+C:\run\diskspd.exe -n -h `-t$t `-o$o `-b$($b)k `-r$($b)k `-w$w -W10 -d60 -C10 -D -L (dir C:\run\testfile?.dat)
+
+[string](get-date)

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/run-demo-9010.ps1

@@ -0,0 +1,38 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+#-# System Config:    __CNODES__ Systems x __CVMS__ VMs/System#Storage Config:   3-way S2D Mirror#Current Workload: 90:10 4K Random Read/Write
+[string](get-date)
+
+$b = 4
+$t = 8
+$o = 20
+$w = 10
+
+C:\run\diskspd.exe -n -h `-t$t `-o$o `-b$($b)k `-r$($b)k `-w$w -W10 -d60 -C10 -D -L (dir C:\run\testfile?.dat)
+
+[string](get-date)

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/run-sweeptemplate.ps1

@@ -0,0 +1,78 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+[string](get-date)
+
+# buffer size/alighment, threads/target, outstanding/thread, write%
+$b = __b__; $t = __t__; $o = __o__; $w = __w__
+
+# optional - specify rate limit in iops, translated to bytes/ms for DISKSPD
+$iops = __iops__
+if ($iops -ne $null) { $g = $iops * $b * 1KB / 1000 }
+
+# io pattern, (r)andom or (s)equential (si as needed for multithread)
+$p = '__p__'
+
+# durations of test, cooldown, warmup
+$d = __d__; $cool = __Cool__; $warm = __Warm__
+
+# sweep template always captures
+$addspec = '__AddSpec__'
+$gspec = $null
+if ($g -ne $null) { $gspec = "g$($g)" }
+$result = "result-b$($b)t$($t)o$($o)w$($w)p$($p)$($gspec)-$($addspec)-$(gc c:\vmspec.txt).xml"
+$dresult = "l:\result"
+$lresultf = join-path "c:\run" $result
+$dresultf = join-path $dresult $result
+
+### prior to this is template
+
+if (-not (gi $dresultf -ErrorAction SilentlyContinue)) {
+
+    $res = 'xml'
+    $gspec = $null
+    if ($g -ne $null) { $gspec = "-g$($g)" }
+
+    C:\run\diskspd.exe -Z20M -z -h `-t$t `-o$o $gspec `-b$($b)k `-$($p)$($b)k `-w$w `-W$warm `-C$cool `-d$($d) -D -L `-R$res (dir C:\run\testfile?.dat) > $lresultf
+
+    # export result and indicate done flag to master
+    # use unbuffered copy to force IO downstream
+    xcopy /j $lresultf $dresult
+    del $lresultf
+    Write-Output "done"
+
+} else {
+
+    write-host -fore green already done $dresultf
+
+    # indicate done flag to master
+    # this should only occur if controller does not change variation
+    Write-Output "done"
+}
+
+[system.gc]::Collect()
+[string](get-date)

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/run.ps1

@@ -0,0 +1,94 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+[string](get-date)
+
+# buffer size/alighment, threads/target, outstanding/thread, write%
+$b = 4; $t = 1; $o = 8; $w = 10
+
+# optional - specify rate limit in iops, translated to bytes/ms for DISKSPD
+$iops = 500
+if ($iops -ne $null) { $g = $iops * $b * 1KB / 1000 }
+
+# io pattern, (r)andom or (s)equential (si as needed for multithread)
+$p = 'r'
+
+# durations of test, cooldown, warmup
+$d = 30*60; $cool = 30; $warm = 60
+
+$addspec = 'base'
+$gspec = $null
+if ($g -ne $null) { $gspec = "g$($g)" }
+$result = "result-b$($b)t$($t)o$($o)w$($w)p$($p)$($gspec)-$($addspec)-$(gc c:\vmspec.txt).xml"
+$dresult = "l:\result"
+$lresultf = join-path "c:\run" $result
+$dresultf = join-path $dresult $result
+
+# cap -> true to capture xml results, otherwise human text
+$cap = $false
+
+### prior to this is template
+
+if (-not (gi $dresultf -ErrorAction SilentlyContinue)) {
+
+    if ($cap) {
+        $res = 'xml'
+    } else {
+        $res = 'text'
+    }
+
+    $gspec = $null
+    if ($g -ne $null) { $gspec = "-g$($g)" }
+    $o = C:\run\diskspd.exe -Z20M -z -h `-t$t `-o$o $gspec `-b$($b)k `-$($p)$($b)k `-w$w `-W$warm `-C$cool `-d$($d) -D -L `-R$res (dir C:\run\testfile?.dat)
+
+    if ($cap) {
+
+        # export result and indicate done flag to master
+        # use unbuffered copy to force IO downstream
+
+        $o | Out-File $lresultf -Encoding ascii -Width 9999
+        xcopy /j $lresultf $dresult
+        del $lresultf
+        Write-Output "done"
+
+    } else {
+
+        #emit to human watcher
+        $o | Out-Host
+    }
+
+} else {
+
+    write-host -fore green already done $dresultf
+
+    # indicate done flag to master
+    # this should only occur if controller does not change variation
+    Write-Output "done"
+}
+
+[system.gc]::Collect()
+[string](get-date)

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/set-pause.ps1

@@ -0,0 +1,36 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+$pause = "C:\ClusterStorage\collect\control\flag\pause"
+$p = gi $pause -ErrorAction SilentlyContinue
+
+if ($p) {
+    write-host -fore green Pause already set $p.CreationTime
+} else {
+    echo (get-random) > $pause
+    write-host -fore red Pause set `@ (get-date)
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/set-storageqos.ps1

@@ -0,0 +1,54 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+# apply given QoS policy (by name) to all VMs on specified nodes
+param (
+    $policyname = $null,
+    [object[]] $node = $(get-clusternode |? State -eq Up)
+)
+
+if ($policyname -ne $null) {
+
+    # QoS policy must exist, else error out
+    $qosp = get-storageqospolicy -name $policyname
+    if ($qosp -eq $null) {
+        # cmdlet error sufficient
+        return
+    }
+    $id = $qosp.PolicyId
+
+} else {
+
+    # clears QoS policy
+    $id = $null
+}
+
+icm $node {
+
+    # note: set-vhdqos should be replaced with set-vmharddiskdrive
+    get-vm |% { get-vmharddiskdrive $_ |% { Set-VMHardDiskDrive -QoSPolicyID $using:id -VMHardDiskDrive $_ }}
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/set-vmfleet.ps1

@@ -0,0 +1,157 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param( [Parameter(ParameterSetName = 'FullSpec', Mandatory = $true)]
+        [int] $ProcessorCount,
+       [Parameter(ParameterSetName = 'FullSpec', Mandatory = $true)]
+        [int64] $MemoryStartupBytes,
+       [Parameter(ParameterSetName = 'FullSpec')]
+        [int64] $MemoryMaximumBytes = 0,
+       [Parameter(ParameterSetName = 'FullSpec')]
+        [int64] $MemoryMinimumBytes = 0,
+       [Parameter(ParameterSetName = 'FullSpec')]
+        [switch]$DynamicMemory = $true,
+       [Parameter(ParameterSetName = 'SizeSpec', Mandatory = $true)]
+        [ValidateSet('A0','A1','A1v2','A2','A2mv2','A2v2','A3','A4','A4mv2','A4v2','A5','A6','A7','A8mv2','A8v2','D1','D11','D11v2','D12','D12v2','D13','D13v2','D14','D14v2','D1
+5v2','D1v2','D2','D2v2','D3','D3v2','D4','D4v2','D5v2','DS11','DS11v2','DS12','DS12v2','DS13','DS13v2','DS14','DS14v2','DS15v2')]
+        [string]$Compute = 'A1'
+    )
+
+# to regenerate validateset
+# ($vmsize.keys | sort |% { "'$_'" }) -join ','
+
+# c = compute cores
+# m = memory
+# a = alias for another (ex: d -> ds, d -> dv2)
+# note that alias is only chased once
+
+ $vmsize = @{
+
+    # general purpose table
+    'A0' = @{ 'c' = 1; 'm' = 0.75GB; };
+    'A1' = @{ 'c' = 1; 'm' = 1.75GB; };
+    'A2' = @{ 'c' = 2; 'm' = 3.5GB; };
+    'A3' = @{ 'c' = 4; 'm' = 7GB; };
+    'A4' = @{ 'c' = 8; 'm' = 14GB; };
+    'A5' = @{ 'c' = 2; 'm' = 14GB };
+    'A6' = @{ 'c' = 4; 'm' = 28GB };
+    'A7' = @{ 'c' = 8; 'm' = 56GB };
+
+    'A1v2' = @{ 'c' = 1; 'm' = 2GB; };
+    'A2v2' = @{ 'c' = 2; 'm' = 4GB; };
+    'A4v2' = @{ 'c' = 4; 'm' = 8GB; };
+    'A8v2' = @{ 'c' = 8; 'm' = 16GB; };
+    'A2mv2' = @{ 'c' = 2; 'm' = 16GB; }
+    'A4mv2' = @{ 'c' = 4; 'm' = 32GB; };
+    'A8mv2' = @{ 'c' = 8; 'm' = 64GB; };
+
+    'D1' = @{ 'c' = 1; 'm' = 3.5GB };
+    'D2' = @{ 'c' = 2; 'm' = 7GB };
+    'D3' = @{ 'c' = 4; 'm' = 14GB };
+    'D4' = @{ 'c' = 8; 'm' = 28GB };
+
+    'D1v2' = @{ 'a' = 'D1' }
+    'D2v2' = @{ 'a' = 'D2' }
+    'D3v2' = @{ 'a' = 'D3' }
+    'D4v2' = @{ 'a' = 'D4' }
+    'D5v2' = @{ 'c' = 16; 'm' = 56GB };
+
+    # memory optimized table (just the d's)
+    'D11' = @{ 'c' = 2; 'm' = 14GB };
+    'D12' = @{ 'c' = 4; 'm' = 28GB };
+    'D13' = @{ 'c' = 8; 'm' = 56GB };
+    'D14' = @{ 'c' = 16; 'm' = 112GB };
+
+    'DS11' = @{ 'a' = 'D11' };
+    'DS12' = @{ 'a' = 'D12' };
+    'DS13' = @{ 'a' = 'D13' };
+    'DS14' = @{ 'a' = 'D14' };
+
+    'D11v2' = @{ 'a' = 'D11' };
+    'D12v2' = @{ 'a' = 'D12' };
+    'D13v2' = @{ 'a' = 'D13' };
+    'D14v2' = @{ 'a' = 'D14' };
+    'D15v2' = @{ 'c' = 20; 'm' = 140GB };
+
+    'DS11v2' = @{ 'a' = 'D11v2' };
+    'DS12v2' = @{ 'a' = 'D12v2' };
+    'DS13v2' = @{ 'a' = 'D13v2' };
+    'DS14v2' = @{ 'a' = 'D14v2' };
+    'DS15v2' = @{ 'a' = 'D15v2' };
+}
+
+$g = Get-ClusterGroup |? GroupType -eq VirtualMachine | group -Property OwnerNode -NoElement
+
+icm $g.Name {
+
+    # import vmsize hash (cannot $using[$using])
+    $vmsize = $using:vmsize
+
+    Get-ClusterGroup |? GroupType -eq VirtualMachine |? OwnerNode -eq $env:COMPUTERNAME |% {
+
+        $g = $_
+
+        switch ($using:PSCmdlet.ParameterSetName) {
+
+            'FullSpec' {
+
+                if ($using:MemoryMaximumBytes -eq 0) {
+                    $MemoryMaximumBytes = $MemoryStartupBytes
+                } else {
+                    $MemoryMaximumBytes = $using:MemoryMaximumBytes
+                }
+                if ($using:MemoryMinimumBytes -eq 0) {
+                    $MemoryMinimumBytes = $MemoryStartupBytes
+                } else {
+                    $MemoryMinimumBytes = $using:MemoryMinimumBytes
+                }
+
+                $memswitch = '-StaticMemory'
+                $dynamicMemArg = ""
+                if ($using:DynamicMemory) {
+                    $memswitch = '-DynamicMemory'
+                    $dynamicMemArg += "-MemoryMinimumBytes $MemoryMinimumBytes -MemoryMaximumBytes $MemoryMaximumBytes"
+                }
+
+                if ($g.State -ne 'Offline') {
+                    write-host -ForegroundColor Yellow Cannot alter VM sizing on running VMs "($($_.Name))"
+                } else {
+                    iex "Set-VM -ComputerName $($g.OwnerNode) -Name $($g.Name) -ProcessorCount $using:ProcessorCount -MemoryStartupBytes $using:MemoryStartupBytes $dynamicMemArg $memswitch"
+                }
+            }
+
+            'SizeSpec' {
+                $a = $vmsize[$using:compute].a
+                if ($a -eq $null) {
+                    $a = $using:compute
+                }
+
+                Set-VM -ComputerName $($g.OwnerNode) -Name $($g.Name) -ProcessorCount $vmsize[$a].c -MemoryStartupBytes $vmsize[$a].m -StaticMemory
+            }       
+        }
+    }
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/start-sweep.ps1

@@ -0,0 +1,506 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    [string] $addspec = "base",
+    [string] $runtemplate = "c:\clusterstorage\collect\control\run-sweeptemplate.ps1",
+    [string] $runfile = "c:\clusterstorage\collect\control\run-sweep.ps1",
+    [string[]] $labeltemplate = @('b','t','o','w','p','-$addspec'),
+    [Parameter(Mandatory =$true)]
+        [int[]] $b,
+    [Parameter(Mandatory =$true)]
+        [int[]] $t,
+    [Parameter(Mandatory =$true)]
+        [int[]] $o,
+    [Parameter(Mandatory =$true)]
+        [int[]] $w,
+    [int[]] $iops = $null,
+    [ValidateSet('r','s','si')]
+        [string[]] $p = 'r',
+    [ValidateRange(1,[int]::MaxValue)]
+        [int] $d = 60,
+    [ValidateRange(0,[int]::MaxValue)]
+        [int] $warm = 60,
+    [ValidateRange(0,[int]::MaxValue)]
+        [int] $cool = 60,
+    [string[]] $pc = $null,
+    [switch] $midcheck = $false
+    )
+
+#############
+
+# a single named variable with a set of values
+class variable {
+
+    [int] $_ordinal
+    [object[]] $_list
+    [string] $_label
+
+    variable([string] $label, [object[]] $list)
+    {
+        $this._list = $list
+        $this._ordinal = 0
+        $this._label = $label
+    }
+
+    # current value of the variable ("red"/"blue"/"green")
+    [object] value()
+    {
+        if ($this._list.count -gt 0) {
+            return $this._list[$this._ordinal]
+        } else {
+            return $null
+        }
+    }
+
+    # label/name of the variable ("color")
+    [object] label()
+    {
+        return $this._label
+    }
+
+    # increment to the next member, return whether a logical carry
+    # has occured (overflow)
+    [bool] increment()
+    {
+        # empty list passes through
+        if ($this._list.Count -eq 0) {
+            return $true
+        }
+
+        # non-empty list, increment
+        $this._ordinal += 1
+        if ($this._ordinal -eq $this._list.Count) {
+            $this._ordinal = 0
+            return $true
+        }
+        return $false
+    }
+
+    # back to initial state
+    [void] reset()
+    {
+        $this._ordinal = 0
+    }
+}
+
+# a set of variables which allows enumeration of their combinations
+# this behaves as a numbering system indexing the respective variables
+# order is not specified
+class variableset {
+
+    [hashtable] $_set = @{}
+
+    variableset([variable[]] $list)
+    {
+        $list |% { $this._set[$_.label()] = $_ }
+        $this._set.Values |% { $_.reset() }
+    }
+
+    # increment the enumeration
+    # returns true if the enumeration is complete
+    [bool] increment()
+    {
+        $carry = $false
+        foreach ($v in $this._set.Values) {
+
+            # if the variable returns the carry flag, increment
+            # the next, and so forth
+            $carry = $v.increment()
+            if (-not $carry) { break }
+        }
+
+        # done if the most significant carried over
+        return $carry
+    }
+
+    # enumerator of all variables
+    [object] getset()
+    {
+        return $this._set.Values
+    }
+
+    # return value of specific variable
+    [object] get([string]$label)
+    {
+        return $this._set[$label].value()
+    }
+
+    # return a label representing the current value of the set, following the input label template
+    # add a leading '-' to get a seperator
+    # use a leading '$' to eliminate repetition of label (just produce value)
+    [string] label([string[]] $template)
+    {
+        return $($template |% {
+
+            $str = $_
+            $pfx = ''
+            $done = $false
+            $norep = $false
+            do {
+                switch ($str[0])
+                {
+                    '-' {
+                        $pfx = '-'
+                        $str = $str.TrimStart('-')
+                    }
+                    '$' {
+                        $norep = $true
+                        $str = $str.TrimStart('$')
+                    }
+                    default {
+                        $done = $true
+                    }
+                }
+            } while (-not $done)
+
+            $lookstr = $str
+            if ($norep) {
+                $str = ''
+            }
+
+            # only produce labels for non-null values
+            if ($this._set[$lookstr].value() -ne $null) {
+                "$pfx$str" + $this._set[$lookstr].value()
+            }
+        }) -join $null
+    }
+}
+
+#############
+
+function start-logman(
+    [string] $computer,
+    [string] $name,
+    [string[]] $counters
+    )
+{
+    $f = "c:\perfctr-$name-$computer.blg"
+
+    $null = logman create counter "perfctr-$name" -o $f -f bin -si 1 --v -c $counters -s $computer
+    $null = logman start "perfctr-$name" -s $computer
+    write-host "performance counters on: $computer"
+}
+
+function stop-logman(
+    [string] $computer,
+    [string] $name,
+    [string] $path
+    )
+{
+    $f = "c:\perfctr-$name-$computer.blg"
+    
+    $null = logman stop "perfctr-$name" -s $computer
+    $null = logman delete "perfctr-$name" -s $computer
+    xcopy /j $f $path
+    del -force $f
+    write-host "performance counters off: $computer"
+}
+
+function new-runfile(
+    [variableset] $vs
+    )
+{
+    # apply current subsitutions to produce a new run file
+    gc $runtemplate |% {
+
+        $line = $_
+
+        foreach ($v in $vs.getset()) {
+            # non-null goes in as is, null goes in as evaluatable $null
+            if ($v.value() -ne $null) {
+                $vsub = $v.value()
+            } else {
+                $vsub = '$null'
+            }
+            $line = $line -replace "__$($v.label())__",$vsub
+        }
+
+        $line
+
+    } | out-file $runfile -Encoding ascii -Width 9999
+}
+
+function show-run(
+    [variableset] $vs
+    )
+{
+    # show current substitions (minus the underscore bracketing)
+    write-host -fore green RUN SPEC `@ (get-date)
+    foreach ($v in $vs.getset()) {
+            if ($v.value() -ne $null) {
+                $vsub = $v.value()
+            } else {
+                $vsub = '$null'
+            }
+            write-host -fore green "`t$($v.label()) = $($vsub)"
+    }
+}
+
+function get-runduration(
+    [variableset] $vs
+    )
+{
+    $vs.get('d') + $vs.get('Warm') + $vs.get('Cool')
+}
+
+function get-doneflags(
+    [switch] $assertnone = $false
+    )
+{
+    $assert = $false
+
+    $tries = 0
+    do {
+
+        if ($tries -gt 0) {
+            sleep 1
+        }
+
+        # increment attempts
+        $tries += 1
+
+        # capture start of the iteration
+        $t0 = (get-date)
+
+        # count number of done flags which agree with completion of the current go epoch
+        $good = 0
+        dir $done |% {
+            $thisdone = gc $_ -ErrorAction SilentlyContinue
+            if ($thisdone -eq $goepoch) {
+                # if asserting that none should be complete, this would be an error!
+                if ($assertnone) {
+                    write-host -fore red ERROR: $_.basename is already done
+                    $assert = $true
+                }
+                $good += 1
+            }
+        }
+
+        # color status message per condition
+        if ($assert -or $good -ne $vms) {
+            $color = 'yellow'
+        } else {
+            $color = 'green'
+        }
+
+        $t1 = (get-date)
+        $tdur = $t1 - $t0
+
+        write-host -fore $color done check iteration $tries with "$good/$vms" `@ $t1 "($('{0:F2}' -f $tdur.totalseconds) seconds)"
+
+    # loop if not asserting, not all vms are done, and still have timeout to use
+    } while ((-not $assertnone) -and $good -ne $vms -and $tries -lt $timeout)
+
+    # return assertion status?
+    if ($assertnone) {
+        return (-not $assert)
+    }
+
+    # return incomplete run failure
+    if ($good -ne $vms) {
+        write-host -fore red ABORT: only received "$good/$vms" completions in $timeout attempts `@ (get-date)
+        return $false
+    }
+
+    # all worked!
+    return $true
+}
+
+function do-run(
+    [variableset] $vs
+    )
+{
+    # apply specified run parameters. note null is ignored.
+    show-run $vs
+
+    write-host -fore yellow Generating new runfile `@ (get-date)
+    new-runfile $vs
+
+    # if we do not have a pause to clear, need the sleep here since go
+    # will release the fleet.
+    # smb fileinfo cache +5 seconds (so that fleet will see updated timestamp)
+    if (-not ($checkpause -and (check-pause -isactive:$true))) {
+        $script:checkpause = $false
+        sleep 15
+    }
+
+    write-host START Go Epoch: $goepoch `@ (get-date)
+    echo $goepoch > $go
+
+    # release any active pause on first loop
+    if ($checkpause) {
+        write-host CLEAR PAUSE `@ (get-date)
+
+        # same sleep need, pause will release the fleet
+        $script:checkpause = $false
+        sleep 15
+        
+        # capture time zero prior to clear - can take time
+        $t0 = get-date
+
+        clear-pause
+    } else {
+    
+        # capture time zero
+        $t0 = get-date
+    }
+
+    # start performance counter capture
+    if ($pc -ne $null) {
+        $curpclabel = $vs.label($labeltemplate)
+        icm (get-clusternode) -ArgumentList (get-command start-logman) {
+
+            param($fn)
+            set-item -path function:\$($fn.name) -value $fn.definition
+
+            start-logman $env:COMPUTERNAME $using:curpclabel $using:pc
+        }
+    }
+
+    ######
+
+    # sleep half, check for false done if possible (clear can take time/short runs), continue
+    $sleep = get-runduration $vs
+
+    $t1 = get-date
+    $td = $t1 - $t0
+
+    if ($midcheck) {
+
+        $remainingsleep = $sleep/2 - $td.TotalSeconds
+        if ($remainingsleep -gt 0) {
+            write-host SLEEP TO MID-RUN "($('{0:F2}' -f $remainingsleep) seconds)" `@ (get-date)
+            sleep $remainingsleep
+        }
+
+        if ($td.TotalSeconds -lt ($sleep - 5)) {
+            write-host MID-RUN CHECK Go Epoch: $goepoch `@ (get-date)
+            # check for early completions, assert none are done yet
+            if (-not (get-doneflags -assertnone:$true)) {
+                return $false
+            }
+            write-host -fore green MID-RUN CHECK PASS Go Epoch: $goepoch `@ (get-date)
+        }
+
+        # capture time and sleep for the remaining interval
+        $t1 = get-date
+        $td = $t1 - $t0
+
+        $remainingsleep = $sleep - $td.TotalSeconds
+        if ($remainingsleep -gt 0) {
+            write-host SLEEP TO END "($('{0:F2}' -f $remainingsleep) seconds)" `@ (get-date)
+            sleep $remainingsleep
+        }
+
+    } else {
+
+        sleep ($sleep - $td.TotalSeconds)
+    }
+
+    ######
+
+    # stop performance counter capture
+    if ($pc -ne $null) {
+        icm (get-clusternode) -ArgumentList (get-command stop-logman) {
+
+            param($fn)
+            set-item -path function:\$($fn.name) -value $fn.definition
+
+            stop-logman $env:COMPUTERNAME $using:curpclabel "C:\ClusterStorage\collect\control\result"
+        }
+    }
+
+    if (-not (get-doneflags)) {
+        return $false
+    }
+
+    # advance go epoch
+    $script:goepoch += 1
+
+    return $true
+}
+
+#############
+
+$vms = (get-clustergroup |? GroupType -eq VirtualMachine |? Name -like "vm-*" |? State -ne Offline).count
+
+# spec location of control files
+$go = "c:\clusterstorage\collect\control\flag\go"
+$done = "c:\clusterstorage\collect\control\flag\done-*"
+
+$timeout = 120
+$checkpause = $true
+
+# ensure we start a new go epoch
+$goepoch = 0
+$gocontent = gc $go -ErrorAction SilentlyContinue
+if ($gocontent -eq '0') {
+    $goepoch = 1
+}
+
+# construct the variable list describing the sweep
+
+############################
+############################
+## Modify from here down
+############################
+############################
+
+# add any additional sweep parameters here to match those specified on the command line
+# ensure your sweep template script contains substitutable elements for each
+#
+#      __somename__
+#
+# bracketed by two underscore characters. Consider adding your new parameters to
+# the label template so that result files are well-named and distinguishable.
+
+$v = @()
+$v += [variable]::new('b',$b)
+$v += [variable]::new('t',$t)
+$v += [variable]::new('o',$o)
+$v += [variable]::new('w',$w)
+$v += [variable]::new('p',$p)
+$v += [variable]::new('iops',$iops)
+$v += [variable]::new('d',$d)
+$v += [variable]::new('Warm',$warm)
+$v += [variable]::new('Cool',$cool)
+$v += [variable]::new('AddSpec',$addspec)
+
+$sweep = [variableset]::new($v)
+
+do {
+
+    write-host -ForegroundColor Cyan '---'
+
+    $r = do-run $sweep
+
+    if (-not $r) {
+        return
+    }
+
+} while (-not $sweep.increment())

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/start-vmfleet.ps1

@@ -0,0 +1,53 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    [string[]] $group = "*",
+    [int] $number = 0
+    )
+
+icm (get-clusternode |? State -eq Up) {
+
+    $n = $using:number
+
+    # failed is an unclean offline tbd root causes (can usually be recovered)
+
+    $using:group |% {
+
+        # sorted list of vms by vm number
+        $vms = @(Get-ClusterGroup |? OwnerNode -eq $env:COMPUTERNAME |? GroupType -eq VirtualMachine |? Name -like "vm-$_-*" | sort -Property @{ Expression = { $null = $_.Name -match '-(\d+)$'; [int] $matches[1] }})
+
+        # start limited number, if specified, else all
+        $(if ($n -gt 0 -and $vms.Count -gt $n) {
+            $vms[0..($n - 1)]
+          } else {
+            $vms
+          }) |? {
+            $_.State -eq 'Offline' -or $_.State -eq 'Failed'
+        } | Start-ClusterGroup
+    }
+} | ft -AutoSize

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/stop-vmfleet.ps1

@@ -0,0 +1,53 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    [string[]] $group = @('*'),
+    [ValidateSet('Save','Shutdown','TurnOff')][string] $method = 'Shutdown'
+    )
+
+icm (get-clusternode |? State -eq Up) -arg $group,$method {
+    param([string[]] $group, $method)
+
+    $group |% {
+
+        $g = Get-ClusterGroup |? GroupType -eq VirtualMachine |? OwnerNode -eq $env:COMPUTERNAME |? Name -like "vm-$_-*" |? State -ne 'Offline'
+
+        if ($g) {
+
+            # stop-clustergroup currently defaults to vm save
+            # use remoted stop-vm for the shutdown case
+            if ($method -eq 'Save') {
+                $g | Stop-ClusterGroup
+            } elseif ($method -eq 'TurnOff') {
+                Stop-VM -ComputerName $env:COMPUTERNAME -Name $g.Name -Force -TurnOff
+            } else {
+                Stop-VM -ComputerName $env:COMPUTERNAME -Name $g.Name -Force
+            }
+        }
+    }
+} | ft -AutoSize

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/sweep-cputarget.ps1

@@ -0,0 +1,212 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    $outfile = "result-cputarget.tsv",
+    $cputargets = $(throw "please specify a set of cpu percentage targets"),
+    $addspec = ""
+    )
+
+$result = "C:\ClusterStorage\collect\control\result"
+
+# count the number of vms in the configuration
+$vms = (Get-ClusterResource |? ResourceType -eq 'Virtual Machine' | measure).Count
+
+# target cpu utilization and qos to +/- given percentage
+$cputargetwindow = 5
+$qoswindow = 5
+
+# clean result file and set column headers
+del -Force $outfile -ErrorAction SilentlyContinue
+'WriteRatio','QOS','AVCPU','IOPS','AVRLat','AVWLat' -join "`t" > $outfile
+
+# make qos policy and reset
+Get-StorageQosPolicy -Name SweepQos -ErrorAction SilentlyContinue | Remove-StorageQosPolicy -Confirm:$false
+New-StorageQosPolicy -Name SweepQoS -MinimumIops $qos -MaximumIops $qos -PolicyType Dedicated
+set-storageqos -policyname SweepQoS
+
+function is-within(
+    $value,
+    $target,
+    $percentage
+    )
+{
+    ($value -ge ($target - ($target*($percentage/100))) -and
+     $value -le ($target + ($target*($percentage/100))))
+}
+
+function get-pc(
+    [string] $blg,
+    [int] $center,
+    [string] $ctr
+)
+{
+    # get central n samples of a performance counter's sample
+    $pc = Import-Counter -Path $blg -Counter "\\*\$ctr"
+
+    $t0 = ($pc.length - $center)/2
+    $t1 = $t0 + $center - 1
+    ($pc[$t0 .. $t1].CounterSamples.CookedValue | measure -Average).Average
+}
+
+$pc = @('\Hyper-V Hypervisor Logical Processor(_Total)\% Total Run Time',
+        '\Processor Information(_Total)\% Processor Performance',
+        '\Cluster CSVFS(_Total)\reads/sec',
+        '\Cluster CSVFS(_Total)\avg. sec/read',
+        '\Cluster CSVFS(_Total)\writes/sec',
+        '\Cluster CSVFS(_Total)\avg. sec/write')
+
+# limit the number of attempts per sweep (mix) to 4 per targeted cpu util
+$sweeplimit = ($cputargets.count * 4)
+
+foreach ($w in 0,10,30) {
+
+    # track measured qos points, starting at given value
+    $h = @{}
+    $qosinitial = $qos = 400
+
+    foreach ($cputarget in $cputargets) {
+
+        # move the qos window using previous run information
+        if ($cputarget -ne $cputargets[0]) {
+            $nextqos = [int](($cputarget*$iops/$avcpu)/$vms)
+            $qos = $nextqos
+        }
+
+        write-host -ForegroundColor Cyan Starting outer loop with CPU target $cputarget and initial QoS $qos
+
+        do {
+
+            # failsafes
+            if ($h[$qos]) { write-host -ForegroundColor Red already measured $qos; break }
+            if ($h.Keys.Count -ge $sweeplimit) { write-host -ForegroundColor Red $sweeplimit tries giving up; break }
+   
+            Set-StorageQosPolicy -Name SweepQoS -MaximumIops $qos
+            write-host -fore Cyan Starting loop with QoS target $qos
+
+            $curaddspec = "$($addspec)w$($w)qos$qos"
+            start-sweep.ps1 -addspec $curaddspec -b 4 -o 32 -t 1 -w $w -p r -d 60 -warm 15 -cool 15 -pc $pc
+
+            # HACKHACK bounce collect
+            Get-ClusterSharedVolume |? { $_.SharedVolumeInfo.FriendlyVolumeName -match 'collect' } | Move-ClusterSharedVolume
+            sleep 1
+
+            # get average IOPS at DISKSPD
+
+            $iops = $(dir $result\*.xml |% {
+                $x = [xml](gc $_)
+                ($x.Results.TimeSpan.Iops.Bucket | measure -Property Total -Average).Average
+            } | measure -Sum).Sum
+
+            # get average cpu utilization for central 60 seconds of each node
+            # get average of all nodes
+
+            $avcpu = $(dir $result\*.blg |% {
+
+                $center = 60
+                $trt = get-pc $_ $center '\Hyper-V Hypervisor Logical Processor(_Total)\% Total Run Time'
+                $ppc = get-pc $_ $center '\Processor Information(_Total)\% Processor Performance'
+                $trt*$ppc/100
+
+
+            } | measure -Average).Average
+
+            # get average latency for central 60 seconds, all nodes
+            # note we must aggregate the product of av latency and iops per node to get total
+            # latency, and then divide by total iops to get whole-cluster average.
+
+            $csvrtotal = 0
+            $csvwtotal = 0
+
+            ($avrlat,$avwlat) = $(dir $result\*.blg |% {
+
+                $csvrlat = get-pc $_ $center '\Cluster CSVFS(_Total)\avg. sec/read'
+                $csvwlat = get-pc $_ $center '\Cluster CSVFS(_Total)\avg. sec/write'
+                $csvr = get-pc $_ $center '\Cluster CSVFS(_Total)\reads/sec'
+                $csvw = get-pc $_ $center '\Cluster CSVFS(_Total)\writes/sec'
+
+                $csvrtotal += $csvr
+
+                $csvwtotal += $csvw
+
+                [pscustomobject] @{ 'avrtime' = $csvrlat*$csvr; 'avwtime' = $csvwlat*$csvw }
+
+            } | measure -Sum -Property avrtime,avwtime).Sum
+
+            $avrlat /= $csvrtotal
+            $avwlat /= $csvwtotal
+
+            # capture results
+            $w,$qos,$avcpu,$iops,$avrlat,$avwlat -join "`t" >> $outfile
+
+            # archive results
+            compress-archive -Path $(dir $result\* -Exclude *.zip) -DestinationPath $result\$curaddspec.zip
+            dir $result\* -Exclude *.zip | del
+            write-host Archived results to $result\$curaddspec.zip
+
+            # stop within targetwindow% of cpu (+/- % of target)
+            if (is-within $avcpu $cputarget $cputargetwindow) {
+                write-host -ForegroundColor Green "Stopping in target window at $('{0:N2}' -f $avcpu) with QoS $qos"
+                break
+            }
+    
+            # assume cpu and qos have a linear relationship - extrapolate to target
+            # could do a direct linear fit of measurements so far
+            $nextqos = [int]($cputarget*$qos/$avcpu)
+
+            # stop if next qos target is within qoswindow% of any previous measurement
+            $inwindow = $false
+            foreach ($previous in $h.keys) {
+
+                if (is-within $nextqos $previous $qoswindow) {
+                    $inwindow = $true
+                    break
+                }
+            }
+
+            if ($inwindow) {
+                write-host -ForegroundColor Yellow "Stopping in window of prior measurement at $('{0:N2}' -f $avcpu) with QoS $qos"
+                break
+            }
+
+            # stop if next qos target is less than initial
+            if ($nextqos -lt $qosinitial) {
+                write-host -ForegroundColor Red "Stopping with underflow targeting $nextqos less than initial $qosinitial"
+                break
+            }
+
+            write-host -ForegroundColor Cyan "Loop acheived $('{0:N2}' -f $avcpu) @ QoS $qos v. target $cputarget - next loop targeting QoS $nextqos"
+
+            # record this datapoint as measured, move along to the next
+            $h[$qos] = 1
+            $qos = $nextqos
+
+        } while ($true)
+    }
+}
+
+set-storageqos -policyname $null

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/test-clusterhealth.ps1

@@ -0,0 +1,637 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    [switch] $CleanOperationalIssues = $false
+    )
+
+function new-namedblock(
+    [string] $name,
+    [scriptblock] $block,
+    [switch] $nullpass = $true,
+    [int] $mustbe = -1
+    )
+{
+    new-object psobject -Property @{
+        'Name' = $name;
+        'Block' = $block;
+        'NullPass' = $nullpass;
+        'MustBe' = $mustbe
+    }
+}
+
+function write-blocktitle(
+    [string[]] $s
+    )
+{
+    write-host -fore cyan ('*'*20) $s
+}
+
+function display-jobs()
+{
+    BEGIN { $j = @() }
+    PROCESS { $j += $_ }
+    END {
+        # consume job results
+        $null = $j | wait-job
+        $j | sort -Property Name |% {
+            write-blocktitle $_.Name,('({0:F1}s)' -f ($_.PsEndTime - $_.PsBeginTime).TotalSeconds)
+            $_ | receive-job
+            $_ | remove-job
+        }
+    }
+}
+
+# script block containers for helper functions
+
+$evfns = {
+
+    function get-fltevents(
+        [decimal] $timedeltams,
+        [string] $provider,
+        [string[]] $evid,
+        [scriptblock] $flt = { $true },
+        [string] $source = $null
+        )
+    {
+        # simple query vs. a provider for a single event within some timedelta of current (in ms)
+        # optional addition of a source provider name filter (for system log filtering)
+$qstr = @"
+<QueryList>
+    <Query Id="0" Path="_PROV_">
+    <Select Path="_PROV_">*[System[_SOURCE_(_EVENTS_) and TimeCreated[timediff(@SystemTime) &lt;= _MS_]]]</Select>
+    </Query>
+</QueryList>
+"@
+
+$srcstr = @"
+Provider[@Name='_SOURCE_'] and 
+"@
+
+        $events = ($evid |% {
+            "EventID=$_"
+        }) -join " or "
+
+        $query = $qstr -replace '_MS_',$timedeltams -replace '_PROV_',$provider -replace '_EVENTS_',$events
+        if ($source) {
+            $query = $query -replace '_SOURCE_',($srcstr -replace '_SOURCE_',$source)
+        } else {
+            $query = $query -replace '_SOURCE_',''
+        }
+
+        Get-WinEvent -FilterXml $query -ErrorAction SilentlyContinue | Where-Object -FilterScript $flt
+    }
+}
+
+$fns = {
+
+    function do-clustersymmetry(
+        [object] $gather,
+        [object[]] $filters,
+        [switch] $saygather = $false
+        )
+    {
+        # deserialization note: the scriptblocks on the inputs are downconverted to strings
+        # so as a result we must reinstantiate
+
+        # assert all nodes agree on object counts from the provided gather element
+        $data = icm (get-clusternode |? State -eq up) ([scriptblock]::create($gather.block))
+
+        if ($saygather) {
+            write-host -ForegroundColor yellow ('*'*15) $gather.name
+        }
+
+        foreach ($f in $filters) {
+
+            write-host -fore yellow ('*'*10) $f.name
+            $r = $data | where-object -FilterScript ([scriptblock]::create($f.block))
+
+            # group results by node
+            $nodeg = $r | group -property pscomputername -NoElement
+
+            # regular symmetery
+            # if grouping by count (per node) yields more than one element, we know some nodes have different counts
+            # i.e., not all are 60: perhaps one is 58, etc.
+            # this is always failure.
+            if ($nodeg -ne $null -and ($nodeg | group -Property Count | measure).count -ne 1) {
+                write-host -ForegroundColor Red Fail
+                $nodeg | sort -Property Name,Count | ft -autosize
+            } else {
+                if ($nodeg -ne $null) {
+                    # if no enforced count or count is correct, pass
+                    if ($f.mustbe -lt 0 -or $f.mustbe -eq $nodeg[0].Count) {
+                        write-host -ForegroundColor Green Pass with $nodeg[0].Count per node
+                    } else {
+                        # enforced count not correct
+                        write-host -ForegroundColor Red Fail - required count of $f.mustbe not consistent on each node
+                        $nodeg | sort -Property Name,Count | ft -autosize
+                    }
+                } elseif ($f.nullpass) {
+                    write-host -ForegroundColor Green Pass with none on any node
+                } else {
+                    write-host -ForegroundColor Red Fail with none on any node
+                }
+            }
+        }
+    }
+}
+
+# Detect RDMA its type (by manufacturer) so that, if needed, we can assert QoS/Cos for RoCE
+
+$netadapters = Get-NetAdapterRdma |? Enabled | Get-NetAdapter
+
+$rdma = $false
+$roce = $false
+$rocematch = 'Mellanox'
+
+if ($netadapters) {
+    write-host -fore green Detected RDMA adapters: will require RDMA
+    $rdma = $true
+
+    # will need a tweak as additional non-Mellanox RoCE arrive (and/or if IB)
+    $qroce = $netadapters |? DriverProvider -match $rocematch
+    if ($qroce) {
+        $drvdesc = $qroce[0].DriverDescription
+
+        write-host -fore green Detected $qroce[0].DriverProvider RDMA adapters: will require RoCE configuration
+        write-host -fore green Adapter Description: $drvdesc
+        $roce = $true
+    }
+}
+
+### Basic Health Checks: serialized
+
+$j = @()
+
+$j += start-job -Name 'Basic Health Checks' {
+
+    # nodes up
+    $cn = Get-ClusterNode
+
+    if ($cn.count -eq $($cn |? State -eq Up).count) {
+        write-host -ForegroundColor Green All cluster nodes Up
+    } else {
+        write-host -ForegroundColor Red Following cluster nodes not Up:
+        $cn |? State -ne Up
+    }
+
+    # node uptime
+    $o = icm ($cn |? State -eq Up) {
+        $w = gwmi win32_operatingsystem
+        $w.ConvertToDateTime($w.localdatetime) - $w.ConvertToDateTime($w.lastbootuptime)
+    }
+
+    $reboots = $o |? TotalHours -lt 1
+
+    if ($reboots.length -and $reboots.length -ne $o.length) {
+        write-host -ForegroundColor Yellow WARNING: $reboots.length nodes have rebooted in the last hour. Ensure that
+        write-host -ForegroundColor Yellow `t no unexpected events are occuring in the cluster.
+    }
+
+    write-host -ForegroundColor Green Cluster node uptime:
+    $o | sort PsComputerName | ft PsComputerName,@{ Label="Uptime"; Expression={"{0}d:{1:00}h:{2:00}m.{3:00}s" -f $_.Days,$_.Hours,$_.Minutes,$_.Seconds}}
+
+    # subsystem check
+    $ss = Get-StorageSubSystem |? Model -eq 'Clustered Windows Storage'
+
+    if (($ss | measure).count -ne 1) {
+        write-host -ForegroundColor Red Expected single clustered storage subsystem, found:
+        $ss | ft -autosize
+        return
+    }
+
+    $ssuh = $ss |? HealthStatus -ne Healthy
+
+    if ($ssuh) {
+        write-host -ForegroundColor Red WARNING: clustered storage subsystem is not healthy
+        $ssuh | ft -AutoSize
+
+        write-host -ForegroundColor Red Output of Debug-StorageSubSystem follows
+        $ssuh | Debug-StorageSubSystem | fl
+    } else {
+        write-host -ForegroundColor Green Clustered storage subsystem Healthy
+    }
+
+    # pool health
+    $p = $ss | Get-StoragePool |? IsPrimordial -ne $true |? HealthStatus -ne Healthy
+
+    if ($p -eq $null) {
+        write-host -ForegroundColor Green All pools Healthy
+    } else {
+        write-host -ForegroundColor Red Following pools not Healthy:
+        $p | ft -autosize
+    }
+}
+
+# vd health
+
+$j += start-job -name 'Virtual Disk Health' {
+
+    $ss = Get-StorageSubSystem |? Model -eq 'Clustered Windows Storage'
+
+    $vd = $ss | Get-VirtualDisk |? HealthStatus -ne Healthy
+
+    if ($vd -eq $null) {
+        write-host -fore green All operational virtual disks Healthy
+    } else {
+        write-host -fore red Following virtual disks not Healthy:
+        $vd | ft -autosize
+    }
+}
+
+# disk state
+
+$j += start-job -name 'Physical Disk Health' {
+    $pd = Get-StorageSubSystem |? Model -eq 'Clustered Windows Storage' | Get-PhysicalDisk
+
+    $nonauto = $pd |? Usage -notmatch 'Journal|Auto-Select'
+    if ($nonauto) {
+        write-host -fore yellow WARNING: there are physical disks which are not auto-select/journal for usage.
+        write-host -fore yellow `t It is possible that while storage resilience has been restored from
+        write-host -fore yellow `t a failure, it is no longer evenly distributed between cluster nodes.
+        write-host -fore yellow `t Consider recovering before doing performance/operational work.
+        $nonauto | ft -autosize
+    } else {
+        write-host -fore green All physical disks are in normal auto-select or journal state
+    }
+}
+
+# consolidated op issues - should logically split?
+
+$j += start-job -name 'Operational Issues and Storage Jobs' -ArgumentList $CleanOperationalIssues {
+
+    param( $CleanOperationalIssues )
+
+    $ev = icm (get-clusternode) {
+        get-winevent -LogName Microsoft-Windows-Storage-Storport/Operational |? Id -eq 502 |% {
+            $ex = [xml]$_.ToXml()
+            $guid = ($ex.Event.EventData.Data |? Name -eq ClassDeviceGuid).'#text'
+            $_ | Add-Member -NotePropertyName DeviceGuid -NotePropertyValue $guid -PassThru
+        }
+    }
+    if ($ev) {
+        write-host -fore yellow WARNING: unresponsive device events have been logged by storport.
+        write-host -fore yellow `tThese may correspond to retired devices, and should be investigated.
+        $ev | ft -autosize PsComputerName,TimeCreated,Id,DeviceGuid,Message
+
+        write-host -fore yellow Corresponding devices by Device GUID:
+        $d = Get-StorageSubSystem Cluster* | Get-StoragePool |? IsPrimordial -eq $false | Get-PhysicalDisk
+        $ev |% {
+            $d |? ObjectId -match "PD:$($_.DeviceGuid)"
+        } | ft -AutoSize
+    }
+
+    # look for livekernelreport and/or bugcheck dumps
+
+    $dmps = icm (get-clusternode |? State -eq Up) {
+        
+        $obj = @()
+
+        $obj += dir $($env:windir + "\livekernelreports")
+        $obj += dir $($env:windir + "\minidump") -ErrorAction SilentlyContinue
+        $obj += dir $($env:windir + "\memory.dmp") -ErrorAction SilentlyContinue
+
+        if ($using:CleanOperationalIssues -and $obj.count -gt 0) {
+            $obj | del -Force -Recurse -ErrorAction SilentlyContinue
+        }
+
+        $obj
+
+    } | sort -property PsParentPath,LastWriteTime,PsComputerName
+
+    if ($dmps) {
+        if ($CleanOperationalIssues) {
+            write-host -fore red NOTE: the following failure reports were forcibly removed
+        } else {
+            write-host -fore yellow WARNING: there are failure reports that may require triage
+        }
+        
+        $dmps | ft -AutoSize
+    }
+
+    # Storage Jobs
+
+    $sj = get-storagejob
+    if ($sj |? JobState -ne Completed) {
+        write-host -ForegroundColor red WARNING: there are active storage jobs running. Investigate the root cause before continuing.
+        $sj | ft -autosize
+    } else {
+        write-host -fore green No storage rebuild or regeneration jobs are active
+    }
+}
+
+# SMB Connectivity Error Check
+
+$fltblk = {
+
+    param( $ev, $warncol, $warn )
+
+    $r = icm (get-clusternode |? State -eq Up) -ArgumentList @((get-command get-fltevents), $ev) {
+
+        param($fn, $ev)
+
+        set-item -path function:\$($fn.name) -value $fn.definition
+
+        $flttcp = {
+            # report tcp (type 1) connectivity events
+            $x = [xml] $_.ToXml()
+            [int]($x.Event.EventData.Data |? Name -eq 'ConnectionType').'#text' -eq 1
+        }
+
+        $fltrdma = {
+            # report rdma (type 2) connectivity events
+            $x = [xml] $_.ToXml()
+            [int]($x.Event.EventData.Data |? Name -eq 'ConnectionType').'#text' -eq 2
+        }
+
+        $last5 = (1000*60*5)
+        $lasthour = (1000*60*60)
+        $lastday = (1000*60*60*24)
+
+        new-object psobject -Property @{
+            'RDMA Last5Min' = (get-fltevents -flt $fltrdma -timedeltams $last5 -provider "Microsoft-Windows-SmbClient/Connectivity" -evid $ev).count;
+            'RDMA LastHour' = (get-fltevents -flt $fltrdma -timedeltams $lasthour -provider "Microsoft-Windows-SmbClient/Connectivity" -evid $ev).count;
+            'RDMA LastDay' = (get-fltevents -flt $fltrdma -timedeltams $lastday -provider "Microsoft-Windows-SmbClient/Connectivity" -evid $ev).count;
+            'TCP Last5Min' = (get-fltevents -flt $flttcp -timedeltams $last5 -provider "Microsoft-Windows-SmbClient/Connectivity" -evid $ev).count;
+            'TCP LastHour' = (get-fltevents -flt $flttcp -timedeltams $lasthour -provider "Microsoft-Windows-SmbClient/Connectivity" -evid $ev).count;
+            'TCP LastDay' = (get-fltevents -flt $flttcp -timedeltams $lastday -provider "Microsoft-Windows-SmbClient/Connectivity" -evid $ev).count;
+        }
+    }
+
+    $hdr = (($r[0] | gm -MemberType NoteProperty |? Definition -like 'int*').Name | sort)
+    $rdmafail = ($r |% { $row = $_; $hdr |? {$_ -like 'RDMA*' } |% { $row.$_ }} | measure -sum).sum -ne 0
+
+    if ($rdmafail) {
+        write-host -ForegroundColor $warncol $warn
+    }
+
+    $r | sort -Property PsComputerName | ft -Property (@('PsComputerName') + $hdr)
+}
+
+$w = @"
+WARNING: the SMB Client is receiving RDMA disconnects. This is an error whose root"
+`t cause may be PFC/CoS misconfiguration (RoCE) on hosts or switches, physical"
+`t issues (ex: bad cable), switch or NIC firmware issues, and will lead to severely"
+`t degraded performance. Additional triage is included in other tests."
+"@
+
+$j += start-job -name 'SMB Connectivity Error Check - Disconnect Failures' -ArgumentList 30804,([ConsoleColor]'Red'),$w -InitializationScript $evfns $fltblk
+
+$w = @"
+WARNING: the SMB Client is receiving RDMA connect errors. This is an error whose root
+`t cause may be actual lack of connectivity or fundamental problems with the RDMA
+`t network fabric. Please inspect especially if in the Last5 bucket.
+"@
+
+$j += start-job -name 'SMB Connectivity Error Check - Connect Failures' -ArgumentList 30803,([ConsoleColor]'Yellow'),$w -InitializationScript $evfns $fltblk
+
+if ($roce) {
+
+    $j += start-job -name 'RoCE: Mellanox Disable Check' -InitializationScript $evfns {
+
+        $r = icm (get-clusternode |? State -eq Up) -ArgumentList (get-command get-fltevents) {
+
+            param($fn)
+
+            set-item -path function:\$($fn.name) -value $fn.definition
+
+            $r = get-fltevents -timedeltams (1000*60*60*24*30) -provider 'System' -source 'mlx4eth63' -evid 35
+        }
+
+        if ($r) {
+
+            write-host -ForegroundColor red WARNING: Mellanox indicates that RDMA has been disabled due to mis/non-configuration
+            write-host -ForegroundColor red `t of Priority Flow Control at some point in the past 30 days. Unless this has been recently
+            write-host -ForegroundColor red `t "corrected," RDMA may be down.
+            write-host -ForegroundColor red Most recent event "(System log)" follows
+            $r[0] | fl
+        } else {
+            write-host -ForegroundColor Green Pass
+        }
+    }
+
+    $j += start-job -name 'RoCE: Mellanox Error Check' {
+        
+        $r = $null
+        $pc = $null
+
+        switch ($using:drvdesc) {
+            "Mellanox ConnectX-3 Pro Ethernet Adapter" {
+                $pc = @{
+                    '\Mellanox Adapter Diagnostic Counters(_Total)\Responder Out-of-order Sequence Received' = 'Out Of Order';
+                    '\Mellanox Adapter Traffic Counters(_Total)\Packets Received Bad CRC Error' = 'Rec BadCRC';
+                    '\Mellanox Adapter Traffic Counters(_Total)\Packets Received Frame Length Error' = 'Rec FrmLenErr';
+                    '\Mellanox Adapter Traffic Counters(_Total)\Packets Received Symbol Error' = 'Rec SymlErr';
+                    '\Mellanox Adapter Traffic Counters(_Total)\Packets Received Discarded' = 'Rec Discard';
+                    '\Mellanox Adapter Traffic Counters(_Total)\Packets Outbound Discarded' = 'Outbnd Discard'
+                    '\Mellanox Adapter Traffic Counters(_Total)\Packets Outbound Errors' = 'Outbnd Err';
+                }
+            }
+            "Mellanox ConnectX-4 Adapter" {
+                $pc = @{
+                    '\Mellanox WinOF-2 Diagnostics(_Total)\Responder out of order sequence received' = 'Out Of Order';
+                    '\Mellanox WinOF-2 Port Traffic(_Total)\Packets Received Bad CRC Error' = 'Rec BadCRC';
+                    '\Mellanox WinOF-2 Port Traffic(_Total)\Packets Received Frame Length Error' = 'Rec FrmLenErr';
+                    '\Mellanox WinOF-2 Port Traffic(_Total)\Packets Received Symbol Error' = 'Rec SymlErr';
+                    '\Mellanox WinOF-2 Port Traffic(_Total)\Packets Received Discarded' = 'Rec Discard';
+                    '\Mellanox WinOF-2 Port Traffic(_Total)\Packets Outbound Discarded' = 'Outbnd Discard'
+                    '\Mellanox WinOF-2 Port Traffic(_Total)\Packets Outbound Errors' = 'Outbnd Err';
+                }
+            }
+            default {
+                write-host -ForegroundColor Red "Unknown adapter type: $($using:drvdesc)"
+            }
+        }
+
+        # no counters, no results
+
+        if ($pc -ne $null) {
+
+            $r = icm (get-clusternode |? State -eq Up) -ArgumentList $pc {
+
+                param($pc)
+
+                $c = get-counter ($pc.Keys |% { $_ }) -ErrorAction SilentlyContinue
+                if ($c) {
+
+                    $o = new-object psobject -Property @{ 'Errors' = $false }
+                    $c.CounterSamples | sort -Property Path |% {
+                        if ($_.path -match '\\\\[^\\]+(\\.*$)') {
+                            $o | Add-Member -NotePropertyName $pc[$matches[1]] -NotePropertyValue $_.CookedValue
+                            if ($_.CookedValue -ne 0) { $o.Errors = $true }
+                        }
+                    }
+                    $o
+                }
+            }
+        }
+
+        if ($r.length -ne (get-clusternode |? State -eq Up).length) {
+
+            write-host -ForegroundColor Yellow WARNING: retransmit statistics not available from all nodes. Ensure driver updates applied.
+            write-host -ForegroundColor Yellow `t $r.length nodes responded out of $((get-clusternode |? Up).length)
+
+        }
+
+        if ($r |? Errors) {
+
+            write-host -ForegroundColor Red "WARNING: Any non-zero error counters may indicate physical or switch/NIC"
+            write-host -ForegroundColor Red "`t issues, likely leading to packet drops and retransmits, which will degrade"
+            write-host -ForegroundColor Red "`t performance. At high enough rates they can lead to SMB connection drops."
+        } else {
+            write-host -ForegroundColor Green "Pass - no errors detected"
+        }
+
+        $hdr = @( 'PsComputerName' )
+        $hdr += $($pc.Values |% { $_ })
+
+        $r | sort -property PsComputerName | ft -Property $hdr
+    }
+}
+
+## Begin Symmetry Checks
+
+$totalf = new-namedblock 'Total' { $true }
+$totalf_nonull = new-namedblock 'Total' { $true } -nullpass:$false
+
+###
+$t = new-namedblock 'Clusport Device Symmetry Check' { gwmi -namespace root\wmi ClusportDeviceInformation }
+$f = @($totalf)
+$f += ,(new-namedblock 'Disk Type' { $_.DeviceType -eq 0} -nullpass:$false)
+# temporarily remove hybrid check - the attribute is not synchronously updated
+# and so may be (harmlessly) inaccurate for a period of time in early configuration
+#$f += ,(new-namedblock 'Hybrid Media' { $_.DeviceAttribute -band 0x4})
+$f += ,(new-namedblock 'Solid/Non-Rotational Media' { $_.DeviceAttribute -band 0x8})
+$f += ,(new-namedblock 'Enclosure Type' { $_.DeviceType -eq 1} -nullpass:$false)
+$f += ,(new-namedblock 'Virtual' { $_.DeviceAttribute -band 0x1} -nullpass:$true)
+
+$j += start-job -InitializationScript $fns -Name $t.name { do-clustersymmetry $using:t $using:f }
+
+###
+$t = new-namedblock 'Physical Disk View Symmetry Check' { Get-StorageSubSystem |? Model -eq 'Clustered Windows Storage' | Get-PhysicalDisk }
+$f = @($totalf)
+
+$j += start-job -InitializationScript $fns -Name $t.name { do-clustersymmetry $using:t $using:f }
+
+###
+$t = new-namedblock 'Enclosure View Symmetry Check' { Get-StorageSubSystem |? Model -eq 'Clustered Windows Storage' | Get-StorageEnclosure }
+$f = @($totalf)
+
+$j += start-job -InitializationScript $fns -Name $t.name { do-clustersymmetry $using:t $using:f }
+
+###
+$t = new-namedblock 'SMB SBL Multichannel Symmetry Check' { Get-SmbMultichannelConnection -SmbInstance SBL }
+$f = @($totalf)
+$f += ,(new-namedblock 'RDMA Capable' { $_.ClientRdmaCapable -and $_.ServerRdmaCapable } -nullpass:$(-not $rdma))
+$f += ,(new-namedblock 'Selected & Non-Failed' { $_.Selected -and -not $_.Failed } -nullpass:$(-not $rdma))
+
+$j += start-job -InitializationScript $fns -Name $t.name { do-clustersymmetry $using:t $using:f }
+
+###
+$t = new-namedblock 'SMB CSV Multichannel Symmetry Check' { Get-SmbMultichannelConnection -SmbInstance CSV }
+$f = @($totalf)
+$f += ,(new-namedblock 'RDMA Capable' { $_.ClientRdmaCapable -and $_.ServerRdmaCapable } -nullpass:$(-not $rdma))
+$f += ,(new-namedblock 'Selected & Non-Failed' { $_.Selected -and -not $_.Failed } -nullpass:$(-not $rdma))
+
+$j += start-job -InitializationScript $fns -Name $t.name { do-clustersymmetry $using:t $using:f }
+
+###
+if ($rdma) {
+
+    # rdma gives us an easy way of identifying a set of adapters to do this test with.
+    # it would be good to extend this more generally
+
+    $t = @()
+    $t += new-namedblock 'RDMA Adapter IP Check' { Get-NetAdapterRdma |? Enabled | Get-NetAdapter |? HardwareInterface | Get-NetIPAddress -ErrorAction SilentlyContinue |? AddressState -eq 'Preferred' }
+    $t += new-namedblock 'RDMA Adapter (Virtual) IP Check' { Get-NetAdapterRdma |? Enabled | Get-NetAdapter |? { -not $_.HardwareInterface } | Get-NetIPAddress -ErrorAction SilentlyContinue |? AddressState -eq 'Preferred' }
+    $t += new-namedblock 'RDMA Adapter (Physical) IP Check' { Get-NetAdapterRdma |? Enabled | Get-NetAdapter |? HardwareInterface | Get-NetIPAddress -ErrorAction SilentlyContinue |? AddressState -eq 'Preferred' }
+    $f = @($totalf)
+
+    $j += start-job -InitializationScript $fns -Name $t[0].name {
+    
+        $using:t |% { do-clustersymmetry $_ $using:f -saygather:$true }
+    }
+}
+
+###
+$t = new-namedblock 'RDMA Adapters Symmetry Check' { Get-NetAdapterRdma |? Enabled | Get-NetAdapter }
+$f = @($totalf)
+$f += ,(new-namedblock 'Operational' { $_.Speed -gt 0 } -nullpass:$(-not $rdma))
+$f += ,(new-namedblock 'Up' { $_.ifOperStatus -eq 'Up' } -nullpass:$(-not $rdma))
+
+$j += start-job -InitializationScript $fns -Name $t.name { do-clustersymmetry $using:t $using:f }
+
+###
+if ($roce) {
+    
+    # assert SMB Direct policy defined
+    $t = new-namedblock 'RoCE/QoS Configuration for SMB Direct' { Get-NetQosPolicy }
+    $f = @($totalf_nonull)
+    $f += ,(new-namedblock 'SMB Direct' { $_.NetDirectPort -eq 445 -and $_.PriorityValue -ne 0 } -nullpass:$false)
+
+    $j += start-job -InitializationScript $fns -Name $t.name { do-clustersymmetry $using:t $using:f }
+    
+    # this is strictly insufficient, should ensure the enabled one is the same as that defined for SMB Direct
+    $t = new-namedblock 'RoCE/CoS Definitions' { Get-NetQosFlowControl }
+    $f = @($totalf_nonull)
+    $f += ,(new-namedblock 'Enabled' { $_.Enabled } -nullpass:$false)
+    $f += ,(new-namedblock 'Disabled' { -not $_.Enabled } -nullpass:$false)
+
+    $j += start-job -InitializationScript $fns -Name $t.name { do-clustersymmetry $using:t $using:f }
+
+    <#
+    #
+    $t = new-namedblock 'RoCE/CoS Applied to RoCE RNICs' { Get-NetAdapterRdma | get-netadapter -Physical -ErrorAction SilentlyContinue |? DriverProvider -match $using:rocematch }
+    $f = @($totalf_nonull)
+    $f += ,(new-namedblock 'Operational FlowControl Specs' { $_.OperationalFlowControl } -nullpass:$false)
+    $f += ,(new-namedblock 'Operational Port/Protocol Classification Specs' { $_.OperationalClassifications } -nullpass:$false)
+    $f += ,(new-namedblock 'Operational CoS Traffic Classes' { $_.OperationalTrafficClasses } -nullpass:$false)
+
+    $j += start-job -InitializationScript $fns -Name $t.name { do-clustersymmetry $using:t $using:f }
+    #>
+}
+
+###
+if (Get-StorageFileServer |? SupportsContinuouslyAvailableFileShare) {
+
+    $t = new-namedblock 'SMB Server CA FS Scope Net Interface Symmetry Check' {
+
+        $fs = Get-StorageFileServer |? SupportsContinuouslyAvailableFileShare
+        if ($fs) {
+            Get-SmbServerNetworkInterface |? ScopeName -eq $fs.FriendlyName
+        } else {
+            $null
+        }
+    }
+    $f = @()
+    $f = ,(new-namedblock 'Rdma Capable' { $_.RdmaCapable } -nullpass:$(-not $rdma))
+
+    $j += start-job -InitializationScript $fns -Name $t.name { do-clustersymmetry $using:t $using:f }
+}
+
+# consume job results
+$j | display-jobs

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/update-csv.ps1

@@ -0,0 +1,132 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    [switch]$disableintegrity = $false,
+    [switch]$renamecsvmounts = $false,
+    [switch]$movecsv = $true,
+    [switch]$movevm = $true,
+    [switch]$shiftcsv = $false
+)
+
+$csv = get-clustersharedvolume
+
+# handle restore cases by mapping the csv to the friendly name of the volume
+# don't rely on the csv name to contain this data
+
+$vh = @{}
+Get-Volume |? FileSystem -eq CSVFS |% { $vh[$_.Path] = $_ }
+
+$csv |% {
+    $v = $vh[$_.SharedVolumeInfo.Partition.Name] 
+    if ($v -ne $null) {
+        $_ | Add-Member -NotePropertyName VDName -NotePropertyValue $v.FileSystemLabel
+    }
+}
+
+if ($disableintegrity) {
+    $csv |% {
+        dir -r $_.SharedVolumeInfo.FriendlyVolumeName | Set-FileIntegrity -Enable:$false -ErrorAction SilentlyContinue
+    }
+}
+
+if ($renamecsvmounts) {
+    $csv |% {
+        if ($_.SharedVolumeInfo.FriendlyVolumeName -match 'Volume\d+$') {
+            ren $_.SharedVolumeInfo.FriendlyVolumeName $_.VDName
+        }
+    }
+}
+
+function move-csv(
+    $rehome = $true,
+    $shift = $false
+    )
+{
+    if ($shift) {
+
+        write-host -fore Yellow Shifting CSV owners
+
+        # rotation order (n0 -> n1, n1 -> n2, ... nn -> n0)
+        $nodes = (Get-ClusterNode |? State -eq Up | sort -Property Name).Name
+        $nh = @{}
+        foreach ($i in 1..($nodes.Length-1)) {
+            $nh[$nodes[$i-1]] = $nodes[$i]
+        }
+        $nh[$nodes[$nodes.Length-1]] = $nodes[0]
+
+        Get-ClusterNode |% {
+            $node = $_.Name
+            $csv |? VDName -match "$node(?:-.+){0,1}" |% {
+                $_ | Move-ClusterSharedVolume $nh[$_.OwnerNode.Name]
+            }
+        }
+
+    } elseif ($rehome) {
+
+        # write-host -fore Yellow Re-homing CSVs
+
+        # move all csvs named by node names back to their named node
+        get-clusternode |? State -eq Up |% {
+            $node = $_.Name
+            $csv |? VDName -match "$node(?:-.+){0,1}" |? OwnerNode -ne $node |% { $_ | Move-ClusterSharedVolume $node }
+        }
+    }
+}
+
+if ($shiftcsv) {
+    # shift rotates all csvs node ownership by one node, in lexical order of
+    # current node owner name. this is useful for forcing out-of-position ops.
+    move-csv -shift:$true
+} elseif ($movecsv) {
+    # move puts all csvs back on their home node
+    move-csv -rehome:$true
+}
+
+if ($movevm) {
+
+    icm (Get-ClusterNode |? State -eq Up) {
+
+        Get-ClusterGroup |? GroupType -eq VirtualMachine |% {
+
+            if ($_.Name -like "vm-*-$env:COMPUTERNAME-*") {
+                if ($env:COMPUTERNAME -ne $_.OwnerNode) {
+                    write-host -ForegroundColor yellow moving $_.name $_.OwnerNode '->' $env:COMPUTERNAME
+
+                    # the default move type is live, but live does not degenerately handle offline vms yet
+                    if ($_.State -eq 'Offline') {
+                        Move-ClusterVirtualMachineRole -Name $_.Name -Node $env:COMPUTERNAME -MigrationType Quick
+                    } else {
+                        Move-ClusterVirtualMachineRole -Name $_.Name -Node $env:COMPUTERNAME
+                    }
+                } else {
+                    # write-host -ForegroundColor green $_.name is on $_.OwnerNode
+                }
+            }
+        }
+    }
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/wait-result.ps1

@@ -0,0 +1,36 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+while ($true) {
+
+    $empty = dir C:\ClusterStorage\collect\control\result |? Length -eq 0
+    if ($empty) { continue }
+    sleep 20
+    break
+}
+
+write-host -fore green Done

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/watch-cluster.ps1

@@ -0,0 +1,531 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    $Cluster = ".",
+    $SampleInterval = 2,
+    [ValidateSet("CSV FS","SSB Cache","SBL","SBL Local","SBL Remote","SBL*","S2D BW","Hyper-V LCPU","SMB SRV","SMB Transport","*")]
+    [string[]] $Sets = "CSV FS",
+    $Log = $null
+)
+
+if ($null -ne $log) {
+    del -Force $log -ErrorAction SilentlyContinue
+}
+
+function write-log(
+    [string[]] $str
+    )
+{
+    if ($null -ne $log) {
+        $str |% {
+            "$(get-date) $_" | Out-File -Append -FilePath $log -Width 9999 -Encoding ascii
+        }
+    }
+}
+
+# display name
+# ctr name
+# display order
+# format string
+# scalar multiplier
+
+class CounterColumn {
+
+    [string] $displayname
+    [string] $setname
+    [string[]] $ctrname
+    [int] $width
+    [string] $fmt
+    [decimal] $multiplier
+    [ValidateSet("Average","AverageAggregate","Sum")][string] $aggregate
+    [boolean] $divider
+
+    CounterColumn(
+        [string] $displayname,
+        [string] $setname,
+        [string[]] $ctrname,
+        [int] $width,
+        [string] $fmt,
+        [decimal] $multiplier,
+        [string] $aggregate,
+        [boolean] $divider
+    )
+    {
+        $this.displayname = $displayname
+        $this.setname = $setname
+        $this.ctrname = $ctrname
+        $this.width = $width
+        $this.fmt = $fmt
+        $this.multiplier = $multiplier
+        $this.aggregate = $aggregate
+        $this.divider = $divider
+
+        if ($this.width -lt $this.displayname.length + 1) {
+            $this.width = $this.displayname.length + 1
+        }
+    }
+}
+
+class CounterColumnSet {
+
+    [CounterColumn[]] $columns
+    [string[]] $counters
+    [string] $topfmt
+    [string] $linfmt
+    [string] $name
+
+    [string] $totalline
+    [string[]] $nodelines
+
+    CounterColumnSet($name)
+    {
+        $this.columns = $null
+        $this.name = $name
+    }
+
+    [void] Add([CounterColumn] $c)
+    {
+        $this.columns += $c
+    }
+
+    [void] Seal()
+    {
+        # assemble the top-line fmt and per-line fmt strings
+        # top-line is just strings/width
+        # per-line adds the (likely numeric) format specifier
+        $n = 1
+        $this.topfmt = $this.linfmt = "{0,-16}"
+        foreach ($col in $this.columns) {
+            $str = ''
+            if ($col.divider) {
+                $str += '| '
+            }
+            $str += "{$n,-$($col.width)"
+            $this.topfmt += "$str}"
+            $this.linfmt += "$($str):$($col.fmt)}"
+
+            $n += 1
+        }
+
+        # assemble the list of counter instances that will be needed
+        # note that some may be internally aggregated (i.e., total = read + write)
+        # in cases where a counterset does not provide an explicit total
+        $this.counters = ($this.columns |% {
+            $s = $_.setname
+            $_.ctrname |% { "\$($s)(_Total)\$($_)" }
+        } | group -NoElement).Name
+    }
+
+    [void] DisplayPre(
+        [hashtable] $samples,
+        [hashtable] $psamples
+        )
+    {
+        # aggregate each column across all live sampled nodes
+        $this.totalline =  $this.linfmt -f $(
+            "Total"
+            foreach ($col in $this.columns) {
+
+                # average aggreate doesn't work across nodes if the base is not consistent
+                # for instance: cannot average latency safely, but can average cpu utilization
+                if ($col.aggregate -ne 'Average' -or $col.aggregate -eq 'AverageAggregate') {
+                    $(foreach ($node in $psamples.keys) {
+                        get-samples $psamples $node $col
+                    }) | get-aggregate $col
+                } else {
+                    $null
+                }
+            }
+        )
+
+        # individual nodes
+        # flags downed/non-responsive nodes by noting which are not
+        # present in the processed samples
+        $this.nodelines = foreach ($node in $samples.keys | sort) {
+            if ($psamples.ContainsKey($node)) {
+                $this.linfmt -f $(
+                    $node
+                    foreach ($col in $this.columns) {
+                        $s = get-samples $psamples $node $col
+                        if ($null -ne $s) {
+                            $a = $s | get-aggregate $col
+                            $a
+                        } else {
+                            "-"
+                        }
+                    }
+                )
+            } else {
+                $this.topfmt -f $(
+                    $node
+                    0..($this.columns.count - 1) |% { "-" }
+                )
+            }
+        }
+    }
+
+    [void] Display()
+    {
+        write-host ($this.topfmt -f (,$this.name + $this.columns.displayname))
+        write-host -fore green $this.totalline
+        $this.nodelines |% { write-host $_ }
+    }
+}
+
+function get-aggregate(
+    [CounterColumn] $col
+    )
+{
+    BEGIN {
+        $n = 0
+        $v = 0
+    }
+    PROCESS {
+        $n += 1
+        $v += $_
+    }
+    END {
+        if ($n -gt 0) {
+            switch -wildcard ($col.aggregate) {
+                'Sum' {
+                    #write-host $col.displayname $col.multipler $v
+                    $col.multiplier * $v
+                }
+                'Average*' {
+                    #write-host $col.displayname $col.multiplier $v $n
+                    $col.multiplier * $v / $n
+                }
+            }
+        } else {
+            $null
+        }
+    }
+}
+
+# get samples out of per-node hash of ctr hashes
+function get-samples(
+    [hashtable] $h,
+    [string] $node,
+    [CounterColumn] $col
+    )
+{
+    foreach ($i in $col.ctrname) {
+
+        $k = "$($col.setname)+$($i)"
+
+        if ($h.ContainsKey($node)) {
+            if ($h[$node].ContainsKey($k)) {
+                $h[$node][$k]
+            } else {
+                write-log "missing $node[$k] : $($h[$node].Keys.Count) total keys"
+            }
+        } else {
+            write-log "missing $node"
+        }
+    }
+}
+
+$allctrs = @()
+
+###
+$c = [CounterColumnSet]::new("CSV FS")
+$c.Add([CounterColumn]::new("IOPS", "Cluster CSVFS", @("Reads/sec","Writes/sec"), 12, '#,#', 1, 'Sum', $false))
+$c.Add([CounterColumn]::new("Reads", "Cluster CSVFS", @("Reads/sec"), 12, '#,#', 1, 'Sum',  $false))
+$c.Add([CounterColumn]::new("Writes", "Cluster CSVFS", @("Writes/sec"), 12, '#,#', 1, 'Sum',  $false))
+
+$c.Add([CounterColumn]::new("BW (MB/s)", "Cluster CSVFS", @("Read Bytes/sec","Write Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
+$c.Add([CounterColumn]::new("Read", "Cluster CSVFS", @("Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum',  $false))
+$c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+$c.Add([CounterColumn]::new("Read Lat (ms)", "Cluster CSVFS", @("Avg. sec/Read"), 15, '0.000', 1000, 'Average', $true))
+$c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Avg. sec/Write"), 8, '0.000', 1000, 'Average', $false))
+
+$c.Add([CounterColumn]::new("Read QAvg", "Cluster CSVFS", @("Avg. Read Queue Length"), 11, '0.000', 1, 'Average', $true))
+$c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Avg. Write Queue Length"), 8, '0.000', 1, 'Average', $false))
+
+$c.Seal()
+$allctrs += $c
+
+###
+$c = [CounterColumnSet]::new("SSB Cache")
+
+$c.Add([CounterColumn]::new("Hit/Sec", "Cluster Storage Hybrid Disks", @("Cache Hit Reads/Sec"), 12, '#,#', 1, 'Sum', $false))
+$c.Add([CounterColumn]::new("Miss/Sec", "Cluster Storage Hybrid Disks", @("Cache Miss Reads/Sec"), 12, '#,#', 1, 'Sum', $false))
+$c.Add([CounterColumn]::new("Remap/Sec" ,"Cluster Storage Cache Stores", @("Page ReMap/sec"), 12, '#,#', 1, 'Sum', $false))
+
+$c.Add([CounterColumn]::new("Cache (MB/s)", "Cluster Storage Hybrid Disks", @("Cache Populate Bytes/sec","Cache Write Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
+$c.Add([CounterColumn]::new("RdPop", "Cluster Storage Hybrid Disks", @("Cache Populate Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+$c.Add([CounterColumn]::new("WrPop", "Cluster Storage Hybrid Disks", @("Cache Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+$c.Add([CounterColumn]::new("Destage (MB/s)", "Cluster Storage Cache Stores", @("Destage Bytes/sec"), 15, '#,#', 0.000001, 'Sum', $true))
+$c.Add([CounterColumn]::new("Update", "Cluster Storage Cache Stores", @("Update Bytes/sec"), 7, '#,#', 0.000001, 'Sum', $false))
+
+$c.Add([CounterColumn]::new("Total (Pgs)", "Cluster Storage Cache Stores", @("Cache Pages"), 11, '0.00E+0', 1, 'Sum', $true))
+$c.Add([CounterColumn]::new("Standby", "Cluster Storage Cache Stores", @("Cache Pages StandBy"), 9, '0.00E+0', 1, 'Sum', $false))
+$c.Add([CounterColumn]::new("L0", "Cluster Storage Cache Stores", @("Cache Pages StandBy L0"), 9, '0.00E+0', 1, 'Sum', $false))
+$c.Add([CounterColumn]::new("L1", "Cluster Storage Cache Stores", @("Cache Pages StandBy L1"), 9, '0.00E+0', 1, 'Sum', $false))
+$c.Add([CounterColumn]::new("L2", "Cluster Storage Cache Stores", @("Cache Pages StandBy L2"), 9, '0.00E+0', 1, 'Sum', $false))
+$c.Add([CounterColumn]::new("Dirty", "Cluster Storage Cache Stores", @("Cache Pages Dirty"), 9, '0.00E+0', 1, 'Sum', $false))
+
+$c.Seal()
+$allctrs += $c
+
+###
+
+foreach ($subset in '','Local','Remote') {
+    $name = 'SBL'
+    $prefix = ''
+    if ($subset.Length) {
+        $name += " $subset"
+        $prefix = "$($subset): "
+    }
+
+    $c = [CounterColumnSet]::new($name)
+
+    $c.Add([CounterColumn]::new("IOPS", "Cluster Disk Counters", @(($prefix + "Read/sec"),($prefix + "Writes/sec")), 12, '#,#', 1, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Reads", "Cluster Disk Counters", @($prefix + "Read/sec"), 12, '#,#', 1, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Writes", "Cluster Disk Counters", @($prefix + "Writes/sec"), 12, '#,#', 1, 'Sum', $false))
+
+    $c.Add([CounterColumn]::new("BW (MB/s)", "Cluster Disk Counters", @(($prefix + "Read - Bytes/sec"),($prefix + "Write - Bytes/sec")), 13, '#,#', 0.000001, 'Sum', $true))
+    $c.Add([CounterColumn]::new("Read", "Cluster Disk Counters", @($prefix + "Read - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+    $c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @($prefix + "Write - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+    $c.Add([CounterColumn]::new("Read Lat (ms)", "Cluster Disk Counters", @($prefix + "Read Latency"), 15, '0.000', 1000, 'Average', $true))
+    $c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @($prefix + "Write Latency"), 8, '0.000', 1000, 'Average', $false))
+
+    $c.Add([CounterColumn]::new("Read QAvg", "Cluster Disk Counters", @($prefix + "Read Avg. Queue Length"), 11, '0.000', 1, 'Average', $true))
+    $c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @($prefix + "Write Avg. Queue Length"), 8, '0.000', 1, 'Average', $false))
+
+    $c.Seal()
+    $allctrs += $c
+}
+
+###
+$c = [CounterColumnSet]::new("SMB SRV")
+
+$c.Add([CounterColumn]::new("IOPS", "SMB Server Shares", @("Data Requests/sec"), 12, '#,#', 1, 'Sum', $false))
+$c.Add([CounterColumn]::new("Reads", "SMB Server Shares", @("Read Requests/sec"), 12, '#,#', 1, 'Sum', $false))
+$c.Add([CounterColumn]::new("Writes", "SMB Server Shares", @("Write Requests/sec"), 12, '#,#', 1, 'Sum', $false))
+
+$c.Add([CounterColumn]::new("Data BW (MB/s)", "SMB Server Shares", @("Data Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
+$c.Add([CounterColumn]::new("Read", "SMB Server Shares", @("Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+$c.Add([CounterColumn]::new("Write", "SMB Server Shares", @("Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+$c.Add([CounterColumn]::new("Total BW (MB/s)", "SMB Server Shares", @("Transferred Bytes/sec"), 13, '#,#', 0.000001, 'Sum', $true))
+$c.Add([CounterColumn]::new("Rcv", "SMB Server Shares", @("Received Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+$c.Add([CounterColumn]::new("Snd", "SMB Server Shares", @("Sent Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+$c.Seal()
+$allctrs += $c
+
+##
+$c = [CounterColumnSet]::new("S2D BW")
+
+$c.Add([CounterColumn]::new("CSV (MB/s)", "Cluster CSVFS", @("Read Bytes/sec","Write Bytes/sec"), 10, '#,#', 0.000001, 'Sum', $false))
+$c.Add([CounterColumn]::new("Read", "Cluster CSVFS", @("Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+$c.Add([CounterColumn]::new("Write", "Cluster CSVFS", @("Write Bytes/sec"), 8 ,'#,#', 0.000001, 'Sum', $false))
+
+$c.Add([CounterColumn]::new("SBL (MB/s)", "Cluster Disk Counters", @("Read - Bytes/sec","Write - Bytes/sec"), 10, '#,#', 0.000001, 'Sum', $true))
+$c.Add([CounterColumn]::new("Read", "Cluster Disk Counters", @("Read - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+$c.Add([CounterColumn]::new("Write", "Cluster Disk Counters", @("Write - Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+$c.Add([CounterColumn]::new("Cache (MB/s)", "Cluster Storage Hybrid Disks", @("Cache Hit Read Bytes/sec","Cache Write Bytes/sec"), 12, '#,#', 0.000001, 'Sum', $true))
+$c.Add([CounterColumn]::new("Read", "Cluster Storage Hybrid Disks", @("Cache Hit Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+$c.Add([CounterColumn]::new("Write", "Cluster Storage Hybrid Disks", @("Cache Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+$c.Add([CounterColumn]::new("Disk (MB/s)", "Cluster Storage Hybrid Disks", @("Disk Read Bytes/sec","Disk Write Bytes/sec"), 11, '#,#', 0.000001, 'Sum', $true))
+$c.Add([CounterColumn]::new("Read", "Cluster Storage Hybrid Disks", @("Disk Read Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+$c.Add([CounterColumn]::new("Write", "Cluster Storage Hybrid Disks", @("Disk Write Bytes/sec"), 8, '#,#', 0.000001, 'Sum', $false))
+
+$c.Seal()
+$allctrs += $c
+
+##
+$c = [CounterColumnSet]::new("Hyper-V LCPU")
+$c.Add([CounterColumn]::new("Logical Total%", "Hyper-V Hypervisor Logical Processor", @("% Total Run Time"), 8, "0.00", 1, 'AverageAggregate', $false))
+$c.Add([CounterColumn]::new("Guest%", "Hyper-V Hypervisor Logical Processor", @("% Guest Run Time"), 8, "0.00", 1, 'AverageAggregate', $false))
+$c.Add([CounterColumn]::new("Hypervisor%", "Hyper-V Hypervisor Logical Processor", @("% Hypervisor Run Time"), 13, "0.00", 1, 'AverageAggregate', $false))
+
+$c.Add([CounterColumn]::new("Root Total%", "Hyper-V Hypervisor Root Virtual Processor", @("% Total Run Time"), 12, "0.00", 1, 'AverageAggregate', $true))
+$c.Add([CounterColumn]::new("Guest%", "Hyper-V Hypervisor Root Virtual Processor", @("% Guest Run Time"), 8, "0.00", 1, 'AverageAggregate', $false))
+$c.Add([CounterColumn]::new("Hypervisor%", "Hyper-V Hypervisor Root Virtual Processor", @("% Hypervisor Run Time"), 12, "0.00", 1, 'AverageAggregate', $false))
+$c.Add([CounterColumn]::new("Remote%", "Hyper-V Hypervisor Root Virtual Processor", @("% Remote Run Time"), 7, "0.00", 1, 'AverageAggregate', $false))
+
+$c.Seal()
+$allctrs += $c
+
+##
+$c = [CounterColumnSet]::new("SMB Transport")
+$c.add([CounterColumn]::new("Read IOPS", "SMB Client Shares", @("Read Requests/sec"), 11, "#,#", 1, 'Sum', $false))
+$c.add([CounterColumn]::new("Write", "SMB Client Shares", @("Write Requests/sec"), 8, "#,#", 1, 'Sum', $false))
+
+$c.add([CounterColumn]::new("RDMA Read", "SMB Client Shares", @("Read Requests transmitted via SMB Direct/sec"), 11, "#,#", 1, 'Sum', $true))
+$c.add([CounterColumn]::new("Write", "SMB Client Shares", @("Write Requests transmitted via SMB Direct/sec"), 8, "#,#", 1, 'Sum', $false))
+
+$c.Seal()
+$allctrs += $c
+
+##
+if ($Sets.Count -eq 1 -and $Sets[0] -eq '*') {
+    $ctrs = $allctrs
+} else {
+    $ctrs = $Sets |% {
+        $s = $_
+        $allctrs |? { $_.name -like $s } # allows the SBL* wildcard
+    }
+}
+
+function start-sample(
+    [CounterColumnSet[]] $ctrs,
+    [int] $SampleInterval
+    )
+{
+    # clear any previous job instance
+    Get-Job -Name watch-cluster -ErrorAction SilentlyContinue | Stop-Job
+    Get-Job -Name watch-cluster -ErrorAction SilentlyContinue | Remove-Job
+
+    # flatten list of counters and uniquify for the total counter set
+    # some display counter sets may repeat specific values (which is fine)
+    $counters = ($ctrs.counters |% { $_ |% { $_ }} | group -NoElement).Name
+
+    icm -AsJob -JobName watch-cluster (Get-ClusterNode -Cluster $Cluster) {
+
+        # extract countersamples, the object does not survive transfer between powershell sessions
+        # extract as a list, not as the individual counters
+        get-counter -Continuous -SampleInterval $using:SampleInterval $using:ctrs.counters |% {
+            ,$_.countersamples
+        }
+    }
+}
+
+# start the first sample job and allow frame draw the first time through
+$j = start-sample $ctrs $SampleInterval
+$downtime = $null
+$skipone = $false
+$loops = 0
+$restart = $false
+
+# hash of most recent samples/node
+$samples = @{}
+Get-ClusterNode -Cluster $Cluster |% { $samples[$_.Name] = $null }
+
+while ($true) {
+
+    if (-not $restart) {
+        Start-Sleep -Seconds $SampleInterval
+
+        # sleep again if needed to prime the sample pipeline;
+        # there are no samples if we just restarted the sampling jobs
+        if ($skipone) {
+            $skipone = $false
+            continue
+        }
+
+        # receive updates into the per-node hash
+        foreach ($child in $j.ChildJobs) {
+            $samples[$child.Location] = $child | receive-job -ErrorAction SilentlyContinue
+        }
+
+        # null out downed nodes and remember first time we saw one drop out
+        $down = 0
+        $j.ChildJobs |? State -ne Running |% {
+            $samples[$_.Location] = $null
+            $down += 1
+        }
+        if ($down -and $null -eq $downtime) {
+            $downtime = get-date
+        }
+
+        # if everything is down, we will attempt restart
+        if ($down -eq $j.ChildJobs.Count) {
+            $restart = $true
+            break
+        }
+    }
+
+    # if explicit restart is required, or it has been 30 seconds with a downed node, restart the jobs to retry
+    if ($restart -or ($null -ne $downtime -and ((get-date)-$downtime).totalseconds -gt 30)) {
+        $j | stop-job
+        $j | remove-job
+        $j = start-sample $ctrs $SampleInterval
+
+        # force gc to clear out accumulated job state quickly
+        [system.gc]::Collect()
+
+        $downtime = $null
+        $skipone = $true
+        $restart = $false
+        continue
+    }
+
+    # now process samples into per-node hashes of set/ctr containing lists of the
+    # cooked values acrosss the (possible) multiple instances
+    $psamples = @{}
+    foreach ($node in $samples.keys) {
+
+        if ($samples[$node]) {
+
+            $nsamples = @{}
+
+            # flatten samples - if we are lagging, we'll have a list
+            # of consecutive (increasing by timestamp) samples
+            # we could try to be more efficient by dumping all but the
+            # final sample, but later ...
+            $samples[$node] |% { $_ } |% {
+
+                ($setinst,$ctr) = $($_.path -split '\\')[3..4]
+                $set = ($setinst -split '\(')[0]
+
+                $k = "$set+$ctr"
+                $nsamples[$k] = $_.cookedvalue
+            }
+
+            $psamples[$node] = $nsamples
+        }
+    }
+
+    # post-process the samples into the counterset, then clear and dump
+    $ctrs.DisplayPre($samples, $psamples)
+    Clear-Host
+    $drawsep = $false
+    $ctrs |% {
+        if ($drawsep) {
+            write-host -fore Green $('-'*20)
+        }
+        $drawsep = $true
+        $_.Display()
+
+    }
+
+    # restart the jobs every so many loops to prevent resource growth
+    $loops += 1
+    if ($loops -gt 100) {
+        $loops = 0
+        $restart = $true
+    }
+}

CristalDiskMark/source/diskspd22/Frameworks/VMFleet1.0/watch-cpu.ps1

@@ -0,0 +1,210 @@
+<#
+DISKSPD - VM Fleet
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+#>
+
+param(
+    $ComputerName = $env:COMPUTERNAME,
+    $SampleInterval = 2
+    )
+
+function div-to-width(
+    [int] $div
+    )
+{
+    # 0 - 100 scale
+    # ex: 4 -> 100/4 = 25 buckets + 1 more for == 100
+    1+100/$div
+}
+
+function center-pad(
+    [string] $s,
+    [int] $width
+    )
+{
+    if ($width -le $s.length) {
+        $s
+    } else {
+        (' ' * (($width - $s.length)/2) + $s)
+    }
+}
+
+function get-legend(
+    [int] $width,
+    [int] $div
+    )
+{
+    # now produce the scale, a digit at a time in vertical orientation
+    # at each multiple of 10% which aligns with a measurement bucket.
+    # the width is the width of the measured values
+    #
+    # 0 5 1
+    #   0 0
+    #     0
+
+    $lines = @()
+    $lines += ,('-' * $width)
+
+    foreach ($dig in 0..2) {
+         $o = foreach ($pos in 0..($width - 1)) {
+
+            $val = $pos * $div
+            if ($val % 10 -eq 0) {
+                switch ($dig) {
+                    0 { if ($val -eq 100) { 1 } else { $val / 10 }}
+                    1 { if ($val -ne 0) { 0 } else { ' ' }}
+                    2 { if ($val -eq 100) { 0 } else { ' ' }}
+                }
+            } else { ' ' }
+        }
+
+        $lines += ,($o -join '')
+    }
+
+    # trailing comments (horizontal scale name)
+    'Percent CPU Utilization' |% {
+        $lines += ,(center-pad $_ $width)
+    }
+
+    $lines
+}
+
+# these are the valid divisions, in units of percentage width.
+# they must evenly divide 100% and either 10% or 20% for scale markings.
+# determine which is the best fit based on window width.
+
+$div = 0
+foreach ($i in 1,2,4,5) {
+    if ((div-to-width $i) -le [console]::WindowWidth) {
+        $div = $i
+        break
+    }
+}
+
+# if nothing fit, widen to 4% divisions
+
+if ($div -eq 0) {
+    $div = 4
+}
+
+$width = div-to-width $div
+
+# get the constant legend; use the remaining height for the vertical cpu core bars.
+# note total height includes variable label line at bottom (instance + aggregagte)
+$legend = get-legend $width $div
+$clip = [console]::WindowHeight - $legend.Count - 1
+
+# insist on a clip no lower than 10
+
+if ($clip -lt 10) {
+    $clip = 10
+}
+
+# set window and buffer size simultaneously so we don't have extra scrollbars
+cls
+[console]::SetWindowSize($width,$clip + $legend.Count + 1)
+[console]::BufferWidth = [console]::WindowWidth
+[console]::BufferHeight = [console]::WindowHeight
+
+# scale divisions at x%
+# this should evenly divide 100%
+$m = [array]::CreateInstance([int],$width)
+
+# which processor counterset should we use?
+# pi is only the root partition if hv is active
+# hvlp is the host physical processors when hv is active
+# via ctrs, hv is active iff hvlp is present and has multiple instances
+$cset = get-counter -ComputerName $ComputerName -ListSet 'Hyper-V Hypervisor Logical Processor' -ErrorAction SilentlyContinue
+if ($cs -ne $null -and $cs.CounterSetType -eq [Diagnostics.PerformanceCounterCategoryType]::MultiInstance) {
+    $cpuset = '\Hyper-V Hypervisor Logical Processor(*)\% Total Run Time'
+} else {
+    $cpuset = '\Processor Information(*)\% Processor Time'
+}
+
+# processor performance counter (turbo/speedstep)
+$ppset = '\Processor Information(_Total)\% Processor Performance'
+
+while ($true) {
+
+    # reset measurements & the lines to output
+    $lines = @()
+    foreach ($i in 0..($m.length - 1)) {
+        $m[$i] = 0
+    }
+
+    # avoid remoting for the local case
+    if ($ComputerName -eq $env:COMPUTERNAME) {
+        $samp = (get-counter -SampleInterval $SampleInterval -Counter $cpuset,$ppset).Countersamples
+    } else {
+        $samp = (get-counter -SampleInterval $SampleInterval -Counter $cpuset,$ppset -ComputerName $ComputerName).Countersamples
+    }
+
+    # get all specific instances and count them into appropriate measurement bucket
+    $samp |% {
+    
+        if ($_.Path -like "*$ppset") { # scaling factor for total utility
+            $pperf = $_.CookedValue/100
+        } elseif ($_.InstanceName -notlike '*_Total') { # per cpu: ignore total and per-numa total
+            $m[[math]::Floor($_.CookedValue/$div)] += 1
+        } elseif ($_.InstanceName -eq '_Total') { # get total
+            $total = $_.CookedValue
+        }
+    }
+
+    # work down the veritical altitude of each strip, starting at vclip
+    $altitude = $clip
+    do {
+        $lines += ,($($m |% {
+
+            # if we are potentially clipped, handle
+            if ($altitude -eq $clip) {
+
+                # clipped?
+                # unclipped but at clip?
+                # nothing, less than altitude
+
+                if ($_ -gt $altitude) { 'x' }
+                elseif ($_ -eq $altitude) { '*' }
+                else { ' ' }
+
+            } else {
+                # normal
+                # >=, output
+                # <, nothing
+                if ($_ -ge $altitude) { '*' }
+                else { ' ' }
+            }
+        }) -join '')
+    } while (--$altitude)
+
+    cls
+    write-host -NoNewline ($lines + $legend -join "`n")
+    write-host -NoNewLine ("`n" + (center-pad ("{2} Total: {0:0.0}% Normalized: {1:0.0}%" -f $total,($total*$pperf),$ComputerName) $width))
+
+    # move the cursor to indicate average utilization
+    # column number is zero based, width is 1-based
+    [console]::SetCursorPosition([math]::Floor(($width - 1)*$total/100),[console]::CursorTop-$legend.Count)
+
+}

CristalDiskMark/source/diskspd22/IORequestGenerator/OverlappedQueue.cpp

@@ -0,0 +1,79 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#include "OverlappedQueue.h"
+#include <assert.h>
+
+OverlappedQueue::OverlappedQueue(void) :
+    _pHead(nullptr),
+    _pTail(nullptr),
+    _cItems(0)
+{
+}
+
+void OverlappedQueue::Add(OVERLAPPED *pOverlapped)
+{
+    pOverlapped->Internal = NULL;
+    if (_pHead == nullptr)
+    {
+        assert(_pTail == nullptr);
+        _pHead = pOverlapped;
+    }
+    else
+    {
+        assert(_pTail != nullptr);
+        _pTail->Internal = (ULONG_PTR)pOverlapped;
+    }
+    _pTail = pOverlapped;
+    _cItems++;
+}
+
+bool OverlappedQueue::IsEmpty(void) const
+{
+    return (_pHead == nullptr);
+}
+
+OVERLAPPED *OverlappedQueue::Remove(void)
+{
+    assert(!IsEmpty());
+
+    OVERLAPPED *pOverlapped = _pHead;
+    _pHead = (OVERLAPPED *)pOverlapped->Internal;
+    if (_pHead == nullptr)
+    {
+        _pTail = nullptr;
+    }
+    _cItems--;
+    return pOverlapped;
+}
+
+size_t OverlappedQueue::GetCount() const
+{
+    return _cItems;
+}

CristalDiskMark/source/diskspd22/IORequestGenerator/ThroughputMeter.cpp

@@ -0,0 +1,143 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#include "Common.h"
+#include "ThroughputMeter.h"
+
+ThroughputMeter::ThroughputMeter(void) :
+    _fRunning(false)
+{
+}
+
+bool ThroughputMeter::IsRunning(void) const
+{
+    return _fRunning;
+}
+
+void ThroughputMeter::Start(DWORD cBytesPerMillisecond, DWORD dwBlockSize, DWORD dwThinkTime, DWORD dwBurstSize)
+{
+    // Initialization
+    _cbCompleted = 0;
+    _cIO = 0; // number of completed IOs in the current burst
+    _cbBlockSize = dwBlockSize;
+
+    _fThrottle = false;
+    _cBytesPerMillisecond = 0;
+    _fThink = false;
+    _ullDelayUntil = 0;
+    _thinkTime = 0;
+    _burstSize = 0;
+    _fRunning = false;
+
+    _ullStartTimestamp = GetTickCount64();
+
+    if (0 != cBytesPerMillisecond)
+    {
+        _fThrottle = true;
+        _cBytesPerMillisecond = cBytesPerMillisecond;
+        _fRunning = true;
+    }
+    else if (0 != dwThinkTime)
+    {
+        _fThink = true;
+        _thinkTime = dwThinkTime;
+        _burstSize = dwBurstSize;
+        _fRunning = true;
+    }
+}
+
+DWORD ThroughputMeter::GetSleepTime(void) const
+{
+    if (_fThink)
+    {
+        ULONGLONG ullTimestamp = GetTickCount64();
+        if (ullTimestamp < _ullDelayUntil)
+        {
+            return (DWORD)(_ullDelayUntil - ullTimestamp);
+        }
+        else
+        {
+            return (_fThrottle) ? _GetThrottleTime() : 0;
+        }
+    }
+    else
+    {
+        if (_fThrottle) // think time has not been specified only check for throttling
+        {
+            return _GetThrottleTime();
+        }
+        else
+        {
+            return 0;
+        }
+    }
+}
+
+DWORD ThroughputMeter::_GetThrottleTime(void) const
+{
+    if ((g_ExperimentFlags & EXPERIMENT_TPUT_CALC) == 0)
+    {
+        ULONGLONG cbExpected = (GetTickCount64() - _ullStartTimestamp) * _cBytesPerMillisecond;
+        return cbExpected >= (_cbCompleted + _cbBlockSize) ? 0 : 1;
+    }
+    else
+    {
+        // prototype to calculate an actual sleep time
+        // under higher loads the ideal delay is likely in the microsecond range, but the minimum sleep is 1 ms
+        // however, at low rates it may be reasonable to calculate the delay and use it if > 1ms
+
+        ULONGLONG elapsed = GetTickCount64() - _ullStartTimestamp;
+        ULONGLONG bytesNext = _cbCompleted + _cbBlockSize;
+
+        if (elapsed * _cBytesPerMillisecond > bytesNext)
+        {
+            // below rate - no sleep
+            return 0;
+        }
+
+        // above rate - sleep at least 1 ms
+        ULONGLONG sleepTarget = (bytesNext / _cBytesPerMillisecond) - elapsed;
+
+        return max((DWORD)sleepTarget, 1);
+    }
+}
+
+void ThroughputMeter::Adjust(size_t cb)
+{
+    _cbCompleted += cb;
+    _cIO++;
+    if (_fThink)
+    {
+        if (_cIO >= _burstSize)
+        {
+            _cIO = 0;
+            _ullDelayUntil = GetTickCount64() + _thinkTime;
+        }
+    }
+}

CristalDiskMark/source/diskspd22/IORequestGenerator/etw.cpp

@@ -0,0 +1,498 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#include "etw.h"
+#include "common.h"
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <Wmistr.h>     //WNODE_HEADER
+
+#define INITGUID        //Include this #define to use SystemTraceControlGuid in Evntrace.h.
+#include <Evntrace.h>   //ETW
+
+#include "IORequestGenerator.h"
+
+// variables declared in IORequestGenerator.cpp
+extern struct ETWEventCounters g_EtwEventCounters;
+extern BOOL volatile g_bTracing;
+
+
+DEFINE_GUID ( /* 3d6fa8d4-fe05-11d0-9dda-00c04fd7ba7c */
+    DiskIoGuid,
+    0x3d6fa8d4,
+    0xfe05,
+    0x11d0,
+    0x9d, 0xda, 0x00, 0xc0, 0x4f, 0xd7, 0xba, 0x7c
+  );
+
+DEFINE_GUID ( /* 90cbdc39-4a3e-11d1-84f4-0000f80464e3 */
+    FileIoGuid,
+    0x90cbdc39,
+    0x4a3e,
+    0x11d1,
+    0x84, 0xf4, 0x00, 0x00, 0xf8, 0x04, 0x64, 0xe3
+  );
+
+DEFINE_GUID ( /* 2cb15d1d-5fc1-11d2-abe1-00a0c911f518 */
+    ImageLoadGuid,
+    0x2cb15d1d,
+    0x5fc1,
+    0x11d2,
+    0xab, 0xe1, 0x00, 0xa0, 0xc9, 0x11, 0xf5, 0x18
+  );
+
+DEFINE_GUID ( /* 3d6fa8d3-fe05-11d0-9dda-00c04fd7ba7c */
+    PageFaultGuid,
+    0x3d6fa8d3,
+    0xfe05,
+    0x11d0,
+    0x9d, 0xda, 0x00, 0xc0, 0x4f, 0xd7, 0xba, 0x7c
+  );
+
+DEFINE_GUID ( /* 9a280ac0-c8e0-11d1-84e2-00c04fb998a2 */
+    TcpIpGuid,
+    0x9a280ac0,
+    0xc8e0,
+    0x11d1,
+    0x84, 0xe2, 0x00, 0xc0, 0x4f, 0xb9, 0x98, 0xa2
+  );
+
+DEFINE_GUID ( /* bf3a50c5-a9c9-4988-a005-2df0b7c80f80 */
+    UdpIpGuid,
+    0xbf3a50c5,
+    0xa9c9,
+    0x4988,
+    0xa0, 0x05, 0x2d, 0xf0, 0xb7, 0xc8, 0x0f, 0x80
+  );
+
+DEFINE_GUID ( /* 3d6fa8d0-fe05-11d0-9dda-00c04fd7ba7c */
+    ProcessGuid,
+    0x3d6fa8d0,
+    0xfe05,
+    0x11d0,
+    0x9d, 0xda, 0x00, 0xc0, 0x4f, 0xd7, 0xba, 0x7c
+  );
+
+DEFINE_GUID ( /* AE53722E-C863-11d2-8659-00C04FA321A1 */
+    RegistryGuid, 
+    0xae53722e,
+    0xc863,
+    0x11d2,
+    0x86, 0x59, 0x0, 0xc0, 0x4f, 0xa3, 0x21, 0xa1
+);
+
+DEFINE_GUID ( /* 3d6fa8d1-fe05-11d0-9dda-00c04fd7ba7c */
+    ThreadGuid,
+    0x3d6fa8d1,
+    0xfe05,
+    0x11d0,
+    0x9d, 0xda, 0x00, 0xc0, 0x4f, 0xd7, 0xba, 0x7c
+  );
+
+/*****************************************************************************/
+// consumes events' data
+//
+BOOL TraceEvents()
+{
+    TRACEHANDLE handles[1];
+    EVENT_TRACE_LOGFILE logfile;
+
+    memset(&logfile, 0, sizeof(EVENT_TRACE_LOGFILE));
+
+    logfile.LoggerName = KERNEL_LOGGER_NAME;
+    logfile.LogFileName = NULL;
+    logfile.LogFileMode = EVENT_TRACE_REAL_TIME_MODE;
+
+    logfile.IsKernelTrace = true;
+
+    handles[0] = OpenTrace(&logfile);
+    if( (TRACEHANDLE)INVALID_HANDLE_VALUE == handles[0] )
+    {
+        PrintError("ETW ERROR: OpenTrace failed (error code: %d)\n", GetLastError());
+        return false;
+    }
+    else
+    {
+        ProcessTrace(handles, 1, 0, 0);
+        CloseTrace(handles[0]);
+    }
+
+    return true;
+}
+
+/*****************************************************************************/
+// allocates memory for ETW properties structure
+//
+PEVENT_TRACE_PROPERTIES allocateEventTraceProperties()
+{
+    PEVENT_TRACE_PROPERTIES pProperties = NULL;
+    size_t size = 0;
+
+
+    size = sizeof(EVENT_TRACE_PROPERTIES)+sizeof(KERNEL_LOGGER_NAME);
+    pProperties = (PEVENT_TRACE_PROPERTIES)malloc(size);
+    if( NULL == pProperties )
+    {
+        PrintError("FATAL ERROR: unable to allocate memory (error code: %d)\n", GetLastError());
+        return NULL;
+    }
+
+    memset(pProperties, 0, size);
+
+    pProperties->LoggerNameOffset = sizeof(EVENT_TRACE_PROPERTIES);
+    pProperties->Wnode.BufferSize = (ULONG)size;
+    pProperties->Wnode.Flags = WNODE_FLAG_TRACED_GUID;
+
+    strcpy_s((char *)pProperties+pProperties->LoggerNameOffset,
+        size-pProperties->LoggerNameOffset, KERNEL_LOGGER_NAME);
+
+    return pProperties;
+}
+
+/*****************************************************************************/
+// callback function for disk I/O events
+void WINAPI eventDiskIo(PEVENT_TRACE pEvent)
+{
+    if( EVENT_TRACE_TYPE_IO_READ == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullIORead;
+    }
+    else if( EVENT_TRACE_TYPE_IO_WRITE == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullIOWrite;
+    }
+}
+
+/*****************************************************************************/
+// callback function for LoadImage events
+//
+void WINAPI eventLoadImage(PEVENT_TRACE pEvent)
+{
+    if( EVENT_TRACE_TYPE_LOAD == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullImageLoad;
+    }
+}
+
+/*****************************************************************************/
+// callback function for memory events
+//
+void WINAPI eventPageFault(PEVENT_TRACE pEvent)
+{
+    if( EVENT_TRACE_TYPE_MM_COW == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullMMCopyOnWrite;
+    }
+    else if( EVENT_TRACE_TYPE_MM_DZF == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullMMDemandZeroFault;
+    }
+    else if( EVENT_TRACE_TYPE_MM_GPF == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullMMGuardPageFault;
+    }
+    else if( EVENT_TRACE_TYPE_MM_HPF == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullMMHardPageFault;
+    }
+    else if( EVENT_TRACE_TYPE_MM_TF == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullMMTransitionFault;
+    }
+}
+
+/*****************************************************************************/
+// callback function for network and TCP/IP events
+//
+void WINAPI eventTcpIp(PEVENT_TRACE pEvent)
+{
+    if( EVENT_TRACE_TYPE_ACCEPT == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullNetAccept;
+    }
+    else if( EVENT_TRACE_TYPE_CONNECT == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullNetConnect;
+    }
+    else if( EVENT_TRACE_TYPE_DISCONNECT == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullNetDisconnect;
+    }
+    else if( EVENT_TRACE_TYPE_RECEIVE == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullNetTcpReceive;
+    }
+    else if( EVENT_TRACE_TYPE_RECONNECT == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullNetReconnect;
+    }
+    else if( EVENT_TRACE_TYPE_RETRANSMIT == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullNetRetransmit;
+    }
+    else if( EVENT_TRACE_TYPE_SEND == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullNetTcpSend;
+    }
+}
+
+/*****************************************************************************/
+// callback function for UDP/IP events
+//
+void WINAPI eventUdpIp(PEVENT_TRACE pEvent)
+{
+    if( EVENT_TRACE_TYPE_SEND == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullNetUdpSend;
+    }
+    else if( EVENT_TRACE_TYPE_RECEIVE == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullNetUdpReceive;
+    }
+}
+
+/*****************************************************************************/
+// callback function for thread events
+//
+void WINAPI eventThread(PEVENT_TRACE pEvent)
+{
+    if( EVENT_TRACE_TYPE_START == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullThreadStart;
+    }
+    else if( EVENT_TRACE_TYPE_END == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullThreadEnd;
+    }
+}
+
+/*****************************************************************************/
+// callback function for process events
+//
+void WINAPI eventProcess(PEVENT_TRACE pEvent)
+{
+    if( EVENT_TRACE_TYPE_START == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullProcessStart;
+    }
+    else if( EVENT_TRACE_TYPE_END == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullProcessEnd;
+    }
+}
+
+/*****************************************************************************/
+// callback function for registry events
+//
+void WINAPI eventRegistry(PEVENT_TRACE pEvent)
+{
+    if( EVENT_TRACE_TYPE_REGCREATE == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegCreate;
+    }
+    else if( EVENT_TRACE_TYPE_REGDELETE == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegDelete;
+    }
+    else if( EVENT_TRACE_TYPE_REGDELETEVALUE == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegDeleteValue;
+    }
+    else if( EVENT_TRACE_TYPE_REGENUMERATEKEY == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegEnumerateKey;
+    }
+    else if( EVENT_TRACE_TYPE_REGENUMERATEVALUEKEY == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegEnumerateValueKey;
+    }
+    else if( EVENT_TRACE_TYPE_REGFLUSH == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegFlush;
+    }
+    else if( EVENT_TRACE_TYPE_REGOPEN == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegOpen;
+    }
+    else if( EVENT_TRACE_TYPE_REGQUERY == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegQuery;
+    }
+    else if( EVENT_TRACE_TYPE_REGQUERYMULTIPLEVALUE == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegQueryMultipleValue;
+    }
+    else if( EVENT_TRACE_TYPE_REGQUERYVALUE == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegQueryValue;
+    }
+    else if( EVENT_TRACE_TYPE_REGSETINFORMATION == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegSetInformation;
+    }
+    else if( EVENT_TRACE_TYPE_REGSETVALUE == pEvent->Header.Class.Type )
+    {
+        ++g_EtwEventCounters.ullRegSetValue;
+    }
+}
+
+/*****************************************************************************/
+// starts ETW session and sets callback functions for various groups of events
+//
+TRACEHANDLE StartETWSession(const Profile& profile)
+{
+    PEVENT_TRACE_PROPERTIES pProperties;
+
+    pProperties = allocateEventTraceProperties();
+    if (nullptr == pProperties)
+    {
+        return 0;
+    }
+
+    pProperties->LogFileMode = EVENT_TRACE_REAL_TIME_MODE;
+
+    //use paged memory
+    if (profile.GetEtwUsePagedMemory())
+    {
+        pProperties->LogFileMode |= EVENT_TRACE_USE_PAGED_MEMORY;
+    }
+
+    //
+    // event classes
+    //
+    if (profile.GetEtwProcess())
+    {
+        pProperties->EnableFlags |= EVENT_TRACE_FLAG_PROCESS;
+        SetTraceCallback(&ProcessGuid, eventProcess);
+    }
+
+    if (profile.GetEtwThread())
+    {
+        pProperties->EnableFlags |= EVENT_TRACE_FLAG_THREAD;
+        SetTraceCallback(&ThreadGuid, eventThread);
+    }
+
+    if (profile.GetEtwImageLoad())
+    {
+        pProperties->EnableFlags |= EVENT_TRACE_FLAG_IMAGE_LOAD;
+        SetTraceCallback(&ImageLoadGuid, eventLoadImage);
+    }
+
+    if (profile.GetEtwDiskIO())
+    {
+        pProperties->EnableFlags |= EVENT_TRACE_FLAG_DISK_IO;
+        SetTraceCallback(&DiskIoGuid, eventDiskIo);
+    }
+
+    if (profile.GetEtwMemoryPageFaults())
+    {
+        pProperties->EnableFlags |= EVENT_TRACE_FLAG_MEMORY_PAGE_FAULTS;
+        SetTraceCallback(&PageFaultGuid, eventPageFault);
+    }
+
+    if (profile.GetEtwMemoryHardFaults())
+    {
+        pProperties->EnableFlags |= EVENT_TRACE_FLAG_MEMORY_HARD_FAULTS;
+        SetTraceCallback(&PageFaultGuid, eventPageFault);
+    }
+
+    if (profile.GetEtwNetwork())
+    {
+        pProperties->EnableFlags |= EVENT_TRACE_FLAG_NETWORK_TCPIP;
+        SetTraceCallback(&TcpIpGuid, eventTcpIp);
+        SetTraceCallback(&UdpIpGuid, eventUdpIp);
+    }
+
+    if (profile.GetEtwRegistry())
+    {
+        pProperties->EnableFlags |= EVENT_TRACE_FLAG_REGISTRY;
+        SetTraceCallback(&RegistryGuid, eventRegistry);
+    }
+
+    //
+    // timer
+    //
+    if (profile.GetEtwUsePerfTimer())
+    {
+        pProperties->Wnode.ClientContext = 1;
+    }
+    if (profile.GetEtwUseSystemTimer())
+    {
+        pProperties->Wnode.ClientContext = 2;
+    }
+    if (profile.GetEtwUseCyclesCounter())
+    {
+        pProperties->Wnode.ClientContext = 3;
+    }
+
+    pProperties->Wnode.Guid = SystemTraceControlGuid;
+
+    TRACEHANDLE hTraceSession;
+    ULONG ret = StartTrace(&hTraceSession, KERNEL_LOGGER_NAME, pProperties);
+    free(pProperties);
+    if (ERROR_SUCCESS != ret)
+    {
+        PrintError("Error starting trace session\n");
+        return 0;
+    }
+
+    return hTraceSession;
+}
+
+/*****************************************************************************/
+// stops ETW session
+//
+PEVENT_TRACE_PROPERTIES StopETWSession(TRACEHANDLE hTraceSession)
+{
+    PEVENT_TRACE_PROPERTIES pProperties;
+
+    pProperties = allocateEventTraceProperties();
+    if( NULL == pProperties )
+    {
+        return NULL;
+    }
+
+    ULONG ret;
+
+    ret = ControlTrace(hTraceSession, NULL, pProperties, EVENT_TRACE_CONTROL_STOP);
+    if( ERROR_SUCCESS != ret )
+    {
+        PrintError("Error stopping trace session\n");
+        return NULL;
+    }
+
+    //wait
+    while(g_bTracing)
+    {
+        Sleep(10);  // TODO: remove active waiting
+    }
+
+    return pProperties;
+}

CristalDiskMark/source/diskspd22/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Microsoft
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

CristalDiskMark/source/diskspd22/Process-DiskSpd.ps1

@@ -0,0 +1,228 @@
+<#
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#>
+
+param(
+    [string] $xmlresultpath = ".",
+    [string] $outfile = "result.tsv"
+    )
+
+function Log-Host(
+    $ForegroundColor = 'Green',
+    [parameter(Position=0, ValueFromRemainingArguments=$true)] $args
+    )
+{
+    write-host -ForegroundColor $ForegroundColor $(get-date -DisplayHint Time) $args
+}
+
+function get-latency( $x ) {
+
+    $x.Results.TimeSpan.Latency.Bucket |% {
+        $_.Percentile,$_.ReadMilliseconds,$_.WriteMilliseconds -join "`t"
+    }
+}
+
+function process-result(
+    )
+{
+    PROCESS {
+
+        log-host -fore green $_
+
+        if ((Get-Content $_ -first 1) -ne '<Results>') {
+
+            write-host -ForegroundColor Red ERROR: $_ does not appear to be an DiskSpd XML result. Make sure `-Rxml was specified.
+
+        } else {
+
+            # count valid results
+            $script:valid += 1
+
+            $x = [xml](Get-Content $_)
+
+            $lf = $_.fullname -replace '.xml','.lat.tsv'
+
+            if (-not [io.file]::Exists($lf)) {
+                get-latency $x > $lf
+            }
+
+            $system = $x.Results.System.ComputerName
+            $t = $x.Results.TimeSpan.TestTimeSeconds
+
+            # sum read and write iops across all threads and targets
+            $ri = ($x.Results.TimeSpan.Thread.Target |
+                    measure -sum -Property ReadCount).Sum
+            $wi = ($x.Results.TimeSpan.Thread.Target |
+                    measure -sum -Property WriteCount).Sum
+            $rb = ($x.Results.TimeSpan.Thread.Target |
+                    measure -sum -Property ReadBytes).Sum
+            $wb = ($x.Results.TimeSpan.Thread.Target |
+                    measure -sum -Property WriteBytes).Sum
+
+            $riops = $ri / $t
+            $wiops = $wi / $t
+            $rbw = $rb / $t
+            $wbw = $wb / $t
+
+            # avoid undefined/div-by-zero when calculating coefficient of var
+            $ricv = $null
+            if ($riops -gt 0) { $ricv = $x.Results.TimeSpan.Iops.ReadIopsStdDev / $riops }
+            $wicv = $null
+            if ($wiops -gt 0) { $wicv = $x.Results.TimeSpan.Iops.WriteIopsStdDev / $wiops }
+
+            if ($ricv -gt 0.10 -or $wicv -gt 0.10) {
+                log-host -ForegroundColor red ("ricv {0:F3} wicv {1:F3}" -f $ricv,$wicv)
+            } else {
+                log-host ("ricv {0:F3} wicv {1:F3}" -f $ricv,$wicv)
+            }
+
+            # note that with runs specified on the command line, only a single write ratio,
+            # outstanding request count and blocksize can be specified, so sampling the one
+            # used for the first thread is sufficient.
+            #
+            # this script DOES NOT handle complex timespan specifications
+
+            if (($x.Results.Profile.TimeSpans.TimeSpan.Targets.Target.Random | select -first 1) -gt 0) {
+                $pat = "Random"
+                $align = ($x.Results.Profile.TimeSpans.TimeSpan.Targets.Target.Random | select -first 1)
+            } else {
+                $pat = "Sequential"
+                $align = ($x.Results.Profile.TimeSpans.TimeSpan.Targets.Target.StrideSize | select -first 1)
+            }
+
+            # convert to object form / export-csv
+
+            $o = @{
+                'ResultFileName' = $_.BaseName;
+                'Path' = ($x.Results.TimeSpan.Thread.target.path | select -first 1);
+                'Affinity' = ($x.Results.Profile.TimeSpans.TimeSpan.DisableAffinity -ne 'true');
+                'System' = $system;
+                'TestTime' = $t;
+                'WriteRatio' = ($x.Results.Profile.TimeSpans.TimeSpan.Targets.Target.WriteRatio |
+                    select -first 1);
+                'Pattern' = $pat;
+                'Align' = $align;
+                'Threads' = $x.Results.TimeSpan.ThreadCount;
+                'Outstanding' = ($x.Results.Profile.TimeSpans.TimeSpan.Targets.Target.RequestCount |
+                    select -first 1);
+                'Block' = ($x.Results.Profile.TimeSpans.TimeSpan.Targets.Target.BlockSize |
+                    select -first 1);
+                'CPU' = $x.Results.TimeSpan.CpuUtilization.Average.UsagePercent;
+                'CPUKern' = $x.Results.TimeSpan.CpuUtilization.Average.KernelPercent;
+                'CPUHotCore' = ($x.Results.TimeSpan.CpuUtilization.CPU.UsagePercent |% { [decimal] $_ } |
+                    sort -Descending | select -first 1);
+                'CPUHotCoreKern' = ($x.Results.TimeSpan.CpuUtilization.CPU.KernelPercent |% { [decimal] $_ } |
+                    sort -Descending | select -first 1);
+                'ReadIOPS' = $riops;
+                'ReadIOPSCoV' = $ricv;
+                'ReadBW' = $rbw;
+                'ReadBytes' = $rb;
+                'LatReadAV' = $x.Results.TimeSpan.Latency.AverageReadMilliseconds;
+                'WriteIOPS' = $wiops;
+                'WriteIOPSCoV' = $wicv;
+                'WriteBW' = $wbw;
+                'WriteBytes' = $wb;
+                'LatWriteAV' = $x.Results.TimeSpan.Latency.AverageWriteMilliseconds;
+                'TotIOPS' = $riops + $wiops;
+                'TotBW' = $rbw + $wbw;
+            }
+
+            # extract the subset of latency percentiles as specified above in $l
+            $h = @{}; $x.Results.TimeSpan.Latency.Bucket |% { $h[$_.Percentile] = $_ }
+
+            $ls = $script:l |% {
+                $b = $h[$_];
+                if ($b.ReadMilliseconds) { $v = $b.ReadMilliseconds } else { $v = "" }
+                $o["LatRead$_"] = $v
+                if ($b.WriteMilliseconds) { $v = $b.WriteMilliseconds } else { $v = "" }
+                $o["LatWrite$_"] = $v
+            }
+
+            # Parse additional fields from filename? other? here
+            $add = @()
+            if ($_.BaseName -match "-wc(\d+)wicpw(\d+)vm(\d+)") {
+                $add = $matches[1..3]
+            } elseif ($_.BaseName -match "-basevm(\d+)") {
+                $add = @('base','base',$matches[1])
+            } else {
+                $add = @('na','na','na')
+            }
+
+            if ($add.count -gt 0) {
+                write-host adding fields ($add -join ' : ')
+            }
+
+            # Place any addons. user to rename manually for the moment.
+            $n = 1
+            $add |% {
+                $o["Add$n"] = $_
+                $n++
+            }
+
+            # emit for export
+            $o
+        }
+    }
+}
+
+#########################
+#
+$delim = "`t"
+$l = @(); foreach ($i in 25,50,75,90,95,99,99.9,99.99,99.999,100) { $l += ,[string]$i }
+
+$files = dir (join-path $xmlresultpath *.xml)
+
+#########################
+# get schema and process/emit column headers
+$valid = 0
+$scratch = $files[0] | process-result
+
+# if no valid results found, stop now
+if ($valid -eq 0) {
+
+    write-host -ForegroundColor Red ERROR: `"$xmlresultpath`"` may not contain DISKSPD result files
+    return
+}
+
+# lexically order the column schema and emit with bounding quotes per field
+$fields = $scratch.Keys | sort
+
+# column headers to output
+$($fields |% { "`"$_`"" }) -join "$delim" | Out-File -FilePath $outfile
+
+#########################
+# now process all files for rows
+$valid = 0
+$files | process-result |% {
+    $row = $_
+    $($fields |% { "`"$($row[$_])`"" }) -join "$delim"
+} | Out-File -FilePath $outfile -Append
+
+# if some invalid files were found, warn
+if ($valid -ne $files.count) {
+
+    write-host -ForegroundColor Red WARNING: `"$xmlresultpath`"` contained $($files.count - $valid) non-DISKSPD xml files
+    return
+}

CristalDiskMark/source/diskspd22/README.md

@@ -0,0 +1,150 @@
+DiskSpd
+=======
+
+DiskSpd is a storage performance tool from the Windows, Windows Server and Cloud Server Infrastructure engineering teams at Microsoft. Please visit <https://github.com/Microsoft/diskspd/wiki> for updated documentation.
+
+In addition to the tool itself, this repository hosts measurement frameworks which utilize DiskSpd. The initial example is [VM Fleet](https://github.com/Microsoft/diskspd/blob/master/Frameworks/VMFleet), used for Windows Server Hyper-Converged environments with Storage Spaces Direct.
+
+This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
+
+Releases
+========
+
+The [Releases](https://github.com/Microsoft/diskspd/releases) page includes **pre-compiled binaries (ZIP) and source code** for the most current releases of the DiskSpd tool. The latest update to DiskSpd can always be downloaded from <https://github.com/Microsoft/diskspd/releases/latest/download/DiskSpd.zip> (aka <https://aka.ms/getdiskspd>).
+
+What's New?
+===========
+
+## DISKSPD
+
+# DISKSPD 2.2 6/3/2024
+
+**NOTE:** changes to the asynchronous IO loop will require rebaselining results with queue depths greater than 1.
+The new design drains the completion queue more aggressively, shrinking delays that impacted latency measurement
+especially on faster storage targeting higher queue depths. Latency measurement is improved at a very small cost
+to rates, as well as improving rates when latency measurement is not used (`-D` or `-L`) due to batch dequeue.
+
+Smaller IO sizes will see the most change.
+
+* New: Socket, NUMA, Core and Power Efficiency Class (big/little cores) added to processor topology reporting (XML and text output)
+  * topology elements only displayed in text results when > 1 are present (e.g. multi-socket systems)
+  * CPU numbering remains group relative, as is the new Core numbering
+  * highest Power Efficiency Class is marked with a `P` suffix (this will usually be `1P` v. `0`)
+  * **NOTE**: efficiency classes can have major impact; work on heterogenous systems **must** be aware of core properties in combination with thread affinity rules (see `-a` and `-n `)
+* New: active power scheme reporting
+* New: base target offset switch now allows range specification: `-Bbase[:length]`; replaces `-B` and `-f`
+* post-run latency histogram processing now significantly faster
+* verbose output is more consistent; includes actual warmup, measured and cooldown intervals v. expected
+
+Binary release supports down to Windows 8 and Windows Server 2012; now uses the Universal CRT.
+
+# DISKSPD 2.1 7/1/2021
+
+* New `-g<n>i` form allowing throughput limit specification in units of IOPS (per specified blocksize)
+* New `-rs<pct>` to specify mixed random/sequential operation (pct random); geometric distribution of run lengths
+* New `-rd<distribution>` to specify non-uniform IO distributions across target
+  * `pct` by target percentage
+  * `abs` by absolute offset
+* New `-Rp<text|xml>` to show specified parameter set in indicated profile output form; works with -X XML profiles and conventional command line
+* XML results/profiles are now indented for ease of review
+* Text result output updates
+  * now shows values in size units (K/M/G, and now TiB) to two decimals
+  * thread stride no longer shown unless specified
+  * -F/-O threadpool parameters shown
+* XML profiles can now be built more generically
+  * XML profiles can be stated in terms of templated target names (*1, *2), replaced in order from command line invocation
+  * the command line now allows options alongside -X: -v, -z, -R and -W/-d/-C along with template target specs
+
+# DISKSPD 2.0.21a 9/21/2018
+
+* Added support for memory mapped I/O:
+  * New `-Sm` option to enable memory mapped I/O
+  * New `-N<vni>` option to specify flush options for memory mapped I/O
+* Added support for providing Event Tracing for Windows (ETW) events
+* Included a Windows Performance Recorder (WPR) profile to enable ETW tracing
+* Added system information to the ResultParser output
+
+# DISKSPD 2.0.20a 2/28/2018
+
+* Changes that may require rebaselining of results:
+  * New random number generator that may show an observable decreased cost
+  * Switched to 512-byte aligned buffers with the `-Z` option to increase performance
+* New `-O` option for specifying the number of outstanding IO requests per thread
+* New `-Zr` option for per-IO randomization of write buffer content
+* XML: Adds a new `<ThreadTarget>` element to support target weighting schemes
+* Enhanced statistics captured from IOPS data
+* Added support for validating XML profiles using an in-built XSD
+* Added support for handling RAW volumes
+* Updated CPU statistics to work on > 64-core systems
+* Updated calculation and accuracy of CPU statistics
+* Re-enable support for ETW statistics
+
+# DISKSPD 2.0.18a 5/31/2016
+
+* update `/?` example to use `-Sh` v. deprecated `-h`
+* fix operation on volumes on GPT partitioned media (<driveletter>:)
+* fix IO priority hint to proper stack alignment (if not 8 byte, will fail)
+* use iB notation to clarify that text result output is in 2^n units (KiB/MiB/GiB)
+
+# DISKSPD 2.0.17a 5/01/2016
+
+* `-S` is expanded to control write-through independent of OS/software cache. Among other things, this allows buffered write-through to be specified (`-Sbw`).
+* XML: adds a new `<WriteThrough>` element to specify write-through
+* XML: `<DisableAllCache>` is no longer emitted (still parsed, though), in favor or `<WriteThrough>` and `<DisableOSCache>`
+* Text output: OS/software cache and write-through state are now documented separately (adjacent lines)
+* Latency histogram now reports to 9-nines (one part in one billion) in both text and XML output
+* Error message added for failure to open write-content source file (`-Z<size>,<file>`)
+
+## VM Fleet
+
+VM Fleet is a performance characterization and analyst framework for exploring the storage capabilities of Windows Server Hyper-Converged environments with Storage Spaces Direct.
+
+# VM Fleet 2.1.0.0 4/3/2024
+
+* Support for Arc VM management (only applicable to clusters managed by Arc)
+* `Set-FleetRunProfileScript` - produce a free-run script based on one of the defined workload profiles
+* `Watch-FleetCPU` - new support for monitoring guest VCPU utilization (-Guest); can handle data outages
+* Fix: performance counter handling now manages intermittent data dropouts (per conventional relog.exe)
+* Fix: mid-run vm health check now handles the possibility of many vms taking longer than intended runtime to validate; early exit to avoid false failures
+* Fix: ignore reboot required indication from cache layer when changing cache behavior; avoid false failure
+
+# VM Fleet 2.0.2.2 12/1/2021
+
+* Fix cluster remoting issue during New-Fleet caused by 2.0.2.1 work
+* Use timestamped logging in New-Fleet, simplify and de-colorize default output
+
+# VM Fleet 2.0.2.1 11/9/2021
+
+* Fix cluster remoting issues in Move-Fleet and Get-FleetDataDiskEstimate
+* Fix timing issue with Start-FleetSweep; always start from fleet pause to avoid triggering free run
+* Use uniqueness to guarantee Start-FleetSweep runs profile in case of repeat
+
+# VM Fleet 2.0.2 11/2/2021
+
+* Windows Server 2019/RS5 host operation now confirmed & supported
+* Read cache warmup for HDD capacity systems should now be faster
+
+`Set-FleetPause` will wait for VM responses before completion by default (see -Timeout)
+
+Several minor fixes including:
+
+* Disable Windows Recovery Console in fleet VMs
+* Fix: `Show-Fleet` IOPS view now aggregates all VM disk devices
+* Fix: clean up leaked/conflicting data collectors and blg automatically
+
+# VM Fleet 2.0 9/22/2021
+
+* major release and rewrite as a first class Powershell module
+* original script-based VM Fleet remains available at Frameworks/VMFleet1.0
+* see the [documentation](https://github.com/microsoft/diskspd/wiki/VMFleet) in the Wiki
+
+Source Code
+===========
+
+The source code for DiskSpd is hosted on GitHub at:
+
+<https://github.com/Microsoft/diskspd>
+
+Any issues with DiskSpd can be reported using the following link:
+
+<https://github.com/Microsoft/diskspd/issues>

CristalDiskMark/source/diskspd22/SECURITY.md

@@ -0,0 +1,41 @@
+<!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
+
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
+
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com).  If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+  * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+  * Full paths of source file(s) related to the manifestation of the issue
+  * The location of the affected source code (tag/branch/commit or direct URL)
+  * Any special configuration required to reproduce the issue
+  * Step-by-step instructions to reproduce the issue
+  * Proof-of-concept or exploit code (if possible)
+  * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
+
+<!-- END MICROSOFT SECURITY.MD BLOCK -->

CristalDiskMark/source/diskspd22/UnitTests/CmdLineParser/CmdLineParser.UnitTests.h

@@ -0,0 +1,117 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include "WexTestClass.h"
+
+namespace UnitTests
+{
+    BEGIN_MODULE()
+        MODULE_PROPERTY(L"Feature", L"CmdLineParser")
+    END_MODULE()
+
+    MODULE_SETUP(ModuleSetup);
+    MODULE_CLEANUP(ModuleCleanup);
+
+    class CmdLineParserUnitTests : public WEX::TestClass<CmdLineParserUnitTests>
+    {
+    private:
+        void VerifyParseCmdLineDisableAllCache(Profile &profile);
+        void VerifyParseCmdLineMappedIO(Profile &profile, MemoryMappedIoFlushMode FlushMode);
+        void VerifyParseCmdLineAccessHints(Profile &profile, bool RandomAccess, bool SequentialScan, bool TemporaryFile);
+
+    public:
+        TEST_CLASS(CmdLineParserUnitTests)
+
+        TEST_CLASS_SETUP(ClassSetup);
+        TEST_CLASS_CLEANUP(ClassCleanup);
+
+        TEST_METHOD_SETUP(MethodSetup);
+        TEST_METHOD_CLEANUP(MethodCleanup);
+
+        TEST_METHOD(Test_GetSizeInBytes);
+        TEST_METHOD(TestGetRandomDataWriteBufferData);
+        TEST_METHOD(TestParseCmdLine);
+        TEST_METHOD(TestParseCmdLineAssignAffinity);
+        TEST_METHOD(TestParseCmdLineBlockSize);
+        TEST_METHOD(TestParseCmdLineBufferedWriteThrough);
+        TEST_METHOD(TestParseCmdLineBurstSizeAndThinkTime);
+        TEST_METHOD(TestParseCmdLineConflictingCacheModes);
+        TEST_METHOD(TestParseCmdLineCreateFileAndMaxFileSize);
+        TEST_METHOD(TestParseCmdLineDisableAffinity);
+        TEST_METHOD(TestParseCmdLineDisableAffinityConflict);
+        TEST_METHOD(TestParseCmdLineDisableAllCacheMode1);
+        TEST_METHOD(TestParseCmdLineDisableAllCacheMode2);
+        TEST_METHOD(TestParseCmdLineDisableLocalCache);
+        TEST_METHOD(TestParseCmdLineDisableOSCache);
+        TEST_METHOD(TestParseCmdLineDurationAndProgress);
+        TEST_METHOD(TestParseCmdLineEtwDISK_IO);
+        TEST_METHOD(TestParseCmdLineEtwIMAGE_LOAD);
+        TEST_METHOD(TestParseCmdLineEtwMEMORY_HARD_FAULTS);
+        TEST_METHOD(TestParseCmdLineEtwMEMORY_PAGE_FAULTS);
+        TEST_METHOD(TestParseCmdLineEtwNETWORK);
+        TEST_METHOD(TestParseCmdLineEtwPROCESS);
+        TEST_METHOD(TestParseCmdLineEtwREGISTRY);
+        TEST_METHOD(TestParseCmdLineEtwTHREAD);
+        TEST_METHOD(TestParseCmdLineEtwUseCycleCount);
+        TEST_METHOD(TestParseCmdLineEtwUsePagedMemory);
+        TEST_METHOD(TestParseCmdLineEtwUsePerfTimer);
+        TEST_METHOD(TestParseCmdLineEtwUseSystemTimer);
+        TEST_METHOD(TestParseCmdLineGroupAffinity);
+        TEST_METHOD(TestParseCmdLineHintFlag);
+        TEST_METHOD(TestParseCmdLineBaseMaxTarget);
+        TEST_METHOD(TestParseCmdLineInterlockedSequential);
+        TEST_METHOD(TestParseCmdLineInterlockedSequentialWithStride);
+        TEST_METHOD(TestParseCmdLineIOPriority);
+        TEST_METHOD(TestParseCmdLineMappedIO);
+        TEST_METHOD(TestParseCmdLineMeasureLatency);
+        TEST_METHOD(TestParseCmdLineOverlappedCountAndBaseOffset);
+        TEST_METHOD(TestParseCmdLineRandomIOAlignment);
+        TEST_METHOD(TestParseCmdLineRandomSequentialMixed);
+        TEST_METHOD(TestParseCmdLineRandomWriteBuffers);
+        TEST_METHOD(TestParseCmdLineRandSeed);
+        TEST_METHOD(TestParseCmdLineRandSeedGetTickCount);
+        TEST_METHOD(TestParseCmdLineResultOutput);
+        TEST_METHOD(TestParseCmdLineStrideSize);
+        TEST_METHOD(TestParseCmdLineTargetDistribution);
+        TEST_METHOD(TestParseCmdLineTargetPosition);
+        TEST_METHOD(TestParseCmdLineThreadsPerFileAndThreadStride);
+        TEST_METHOD(TestParseCmdLineThroughput);
+        TEST_METHOD(TestParseCmdLineTotalThreadCountAndThroughput);
+        TEST_METHOD(TestParseCmdLineTotalThreadCountAndTotalRequestCount);
+        TEST_METHOD(TestParseCmdLineUseCompletionRoutines);
+        TEST_METHOD(TestParseCmdLineUseLargePages);
+        TEST_METHOD(TestParseCmdLineUseParallelAsyncIO);
+        TEST_METHOD(TestParseCmdLineVerbose);
+        TEST_METHOD(TestParseCmdLineWarmupAndCooldown);
+        TEST_METHOD(TestParseCmdLineWriteBufferContentRandomNoFilePath);
+        TEST_METHOD(TestParseCmdLineWriteBufferContentRandomWithFilePath);
+        TEST_METHOD(TestParseCmdLineZeroWriteBuffers);
+    };
+}

CristalDiskMark/source/diskspd22/UnitTests/CmdLineParser/CmdLineParser.rc

@@ -0,0 +1,17 @@
+#include <windows.h>
+#include "Version.h"
+
+#include <ntverp.h>
+
+#define VER_FILETYPE                VFT_APP
+#define VER_FILESUBTYPE             VFT2_UNKNOWN
+#define VER_FILEDESCRIPTION_STR     "DiskSpd Storage Performance Tool"
+#define VER_INTERNALNAME_STR        "CmdLineParser.UnitTests.dll"
+
+#undef VER_PRODUCTVERSION
+#define VER_PRODUCTVERSION DISKSPD_MAJOR,DISKSPD_MINOR,DISKSPD_BUILD,DISKSPD_QFE
+
+#undef VER_PRODUCTVERSION_STR
+#define VER_PRODUCTVERSION_STR      DISKSPD_NUMERIC_VERSION_STRING
+
+#include "common.ver"

CristalDiskMark/source/diskspd22/UnitTests/CmdLineParser/stdafx.h

@@ -0,0 +1,33 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+
+#include <windows.h>
+

CristalDiskMark/source/diskspd22/UnitTests/Common/Common.UnitTests.h

@@ -0,0 +1,130 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include "WexTestClass.h"
+
+namespace UnitTests
+{
+    BEGIN_MODULE()
+        MODULE_PROPERTY(L"Feature", L"Common")
+    END_MODULE()
+
+    class PerfTimerUnitTests : public WEX::TestClass<PerfTimerUnitTests>
+    {
+    public:
+        TEST_CLASS(PerfTimerUnitTests);
+        TEST_METHOD(Test_Freq);
+        TEST_METHOD(Test_GetTime);
+        TEST_METHOD(Test_PerfTimeToSeconds);
+        TEST_METHOD(Test_PerfTimeToMilliseconds);
+        TEST_METHOD(Test_PerfTimeToMicroseconds);
+        TEST_METHOD(Test_SecondsToPerfTime);
+        TEST_METHOD(Test_MillisecondsToPerfTime);
+        TEST_METHOD(Test_MicrosecondsToPerfTime);
+    };
+
+    class HistogramUnitTests : public WEX::TestClass<HistogramUnitTests>
+    {
+    public:
+        TEST_CLASS(HistogramUnitTests);
+        TEST_METHOD(Test_Empty);
+        TEST_METHOD(Test_Add);
+        TEST_METHOD(Test_Clear);
+        TEST_METHOD(Test_MinMax);
+        TEST_METHOD(Test_GetPercentile);
+        TEST_METHOD(Test_GetMean);
+        TEST_METHOD(Test_Merge);
+    };
+
+    class IoBucketizerUnitTests :  public WEX::TestClass<IoBucketizerUnitTests>
+    {
+    public:
+        TEST_CLASS(IoBucketizerUnitTests);
+        TEST_METHOD(Test_Empty);
+        TEST_METHOD(Test_Add);
+        TEST_METHOD(Test_Merge);
+        TEST_METHOD(Test_GetStandardDeviation);
+    };
+
+    class ProfileUnitTests : public WEX::TestClass<ProfileUnitTests>
+    {
+    public:
+        TEST_CLASS(ProfileUnitTests);
+        TEST_METHOD(Test_GetXmlEmptyProfile);
+        TEST_METHOD(Test_GetXmlPrecreateFilesUseMaxSize);
+        TEST_METHOD(Test_GetXmlPrecreateFilesOnlyFilesWithConstantSizes);
+        TEST_METHOD(Test_GetXmlPrecreateFilesOnlyFilesWithConstantOrZeroSizes);
+        TEST_METHOD(Test_MarkFilesAsCreated);
+        TEST_METHOD(Test_Validate);
+        TEST_METHOD(Test_ValidateSystem);
+    };
+
+    class TargetUnitTests : public WEX::TestClass<TargetUnitTests>
+    {
+    public:
+        TEST_CLASS(TargetUnitTests);
+        TEST_METHOD(TestGetSetRandomDataWriteBufferSize);
+        TEST_METHOD(TestGetSetRandomDataWriteBufferSourcePath);
+        TEST_METHOD(Test_TargetGetXmlWriteBufferContentSequential);
+        TEST_METHOD(Test_TargetGetXmlWriteBufferContentZero);
+        TEST_METHOD(Test_TargetGetXmlWriteBufferContentRandomNoFilePath);
+        TEST_METHOD(Test_TargetGetXmlWriteBufferContentRandomWithFilePath);
+        TEST_METHOD(Test_TargetGetXmlDisableAllCache);
+        TEST_METHOD(Test_TargetGetXmlDisableLocalCache);
+        TEST_METHOD(Test_TargetGetXmlDisableOSCache);
+        TEST_METHOD(Test_TargetGetXmlBufferedWriteThrough);
+        TEST_METHOD(Test_TargetGetXmlMemoryMappedIo);
+        TEST_METHOD(Test_TargetGetXmlMemoryMappedIoFlushModeViewOfFile);
+        TEST_METHOD(Test_TargetGetXmlMemoryMappedIoFlushModeNonVolatileMemory);
+        TEST_METHOD(Test_TargetGetXmlMemoryMappedIoFlushModeNonVolatileMemoryNoDrain);
+        TEST_METHOD(Test_TargetGetXmlRandomAccessHint);
+        TEST_METHOD(Test_TargetGetXmlSequentialScanHint);
+        TEST_METHOD(Test_TargetGetXmlCombinedAccessHint);
+        TEST_METHOD(Test_AllocateAndFillRandomDataWriteBuffer);
+        TEST_METHOD(Test_AllocateAndFillRandomDataWriteBufferFromFile);
+    };
+
+    class ThreadParametersUnitTests : public WEX::TestClass<ThreadParametersUnitTests>
+    {
+    public:
+        TEST_CLASS(ThreadParametersUnitTests);
+        TEST_METHOD(Test_AllocateAndFillBufferForTarget);
+    };
+
+    class TopologyUnitTests : public WEX::TestClass<TopologyUnitTests>
+    {
+    public:
+        TEST_CLASS(TopologyUnitTests);
+        TEST_METHOD(Test_MaskCount);
+    };
+}
+
+// TODO: ThreadParameters::GetWriteBuffer
+// TODO: Target::GetRandomDataWriteBuffer();

CristalDiskMark/source/diskspd22/UnitTests/Common/Common.rc

@@ -0,0 +1,17 @@
+#include <windows.h>
+#include "Version.h"
+
+#include <ntverp.h>
+
+#define VER_FILETYPE                VFT_APP
+#define VER_FILESUBTYPE             VFT2_UNKNOWN
+#define VER_FILEDESCRIPTION_STR     "DiskSpd Storage Performance Tool"
+#define VER_INTERNALNAME_STR        "Common.UnitTests.dll"
+
+#undef VER_PRODUCTVERSION
+#define VER_PRODUCTVERSION DISKSPD_MAJOR,DISKSPD_MINOR,DISKSPD_BUILD,DISKSPD_QFE
+
+#undef VER_PRODUCTVERSION_STR
+#define VER_PRODUCTVERSION_STR      DISKSPD_NUMERIC_VERSION_STRING
+
+#include "common.ver"

CristalDiskMark/source/diskspd22/UnitTests/Common/stdafx.h

@@ -0,0 +1,33 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+
+#include <windows.h>
+

CristalDiskMark/source/diskspd22/UnitTests/IORequestGenerator/IORequestGenerator.UnitTests.h

@@ -0,0 +1,64 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include "WexTestClass.h"
+
+namespace UnitTests
+{
+    BEGIN_MODULE()
+        MODULE_PROPERTY(L"Feature", L"IORequestGenerator")
+    END_MODULE()
+
+    class IORequestGeneratorUnitTests : public WEX::TestClass<IORequestGeneratorUnitTests>
+    {
+    public:
+        TEST_CLASS(IORequestGeneratorUnitTests);
+        TEST_METHOD(Test_GetFilesToPrecreate1);
+        TEST_METHOD(Test_GetFilesToPrecreate2);
+        TEST_METHOD(Test_GetFilesToPrecreateConstantSizes);
+        TEST_METHOD(Test_GetFilesToPrecreateConstantOrZeroSizes);
+        TEST_METHOD(Test_GetFilesToPrecreateUseMaxSize);
+
+        TEST_METHOD(Test_GetNextFileOffsetRandom);
+        TEST_METHOD(Test_GetNextFileOffsetSequential);
+        TEST_METHOD(Test_GetNextFileOffsetInterlockedSequential);
+        TEST_METHOD(Test_GetNextFileOffsetParallelAsyncIO);
+        TEST_METHOD(Test_SequentialWithStride);
+        TEST_METHOD(Test_SequentialWithStrideInterleaved);
+        TEST_METHOD(Test_SequentialWithStrideUneven);
+
+        TEST_METHOD(Test_GetThreadBaseFileOffset);
+        TEST_METHOD(Test_GetThreadBaseFileOffsetWithStride);
+
+        TEST_METHOD(Test_ThreadTargetStateInit);
+        TEST_METHOD(Test_ThreadTargetStateEffectiveDistPct);
+        TEST_METHOD(Test_ThreadTargetStateEffectiveDistAbs);
+    };
+}

CristalDiskMark/source/diskspd22/UnitTests/IORequestGenerator/IORequestGenerator.rc

@@ -0,0 +1,17 @@
+#include <windows.h>
+#include "Version.h"
+
+#include <ntverp.h>
+
+#define VER_FILETYPE                VFT_APP
+#define VER_FILESUBTYPE             VFT2_UNKNOWN
+#define VER_FILEDESCRIPTION_STR     "DiskSpd Storage Performance Tool"
+#define VER_INTERNALNAME_STR        "IORequestGenerator.UnitTests.dll"
+
+#undef VER_PRODUCTVERSION
+#define VER_PRODUCTVERSION DISKSPD_MAJOR,DISKSPD_MINOR,DISKSPD_BUILD,DISKSPD_QFE
+
+#undef VER_PRODUCTVERSION_STR
+#define VER_PRODUCTVERSION_STR      DISKSPD_NUMERIC_VERSION_STRING
+
+#include "common.ver"

CristalDiskMark/source/diskspd22/UnitTests/IORequestGenerator/stdafx.h

@@ -0,0 +1,33 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+
+#include <windows.h>
+

CristalDiskMark/source/diskspd22/UnitTests/ResultParser/ResultParser.UnitTests.cpp

@@ -0,0 +1,965 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#include "StdAfx.h"
+#include "ResultParser.UnitTests.h"
+#include "Common.h"
+#include "resultparser.h"
+#include <stdlib.h>
+#include <vector>
+
+using namespace WEX::TestExecution;
+using namespace WEX::Logging;
+using namespace std;
+
+namespace UnitTests
+{
+    void ResultParserUnitTests::Test_ParseResults()
+    {
+        Profile profile;
+        TimeSpan timeSpan;
+        ResultParser parser;
+
+        Results results;
+        results.fUseETW = false;
+        double fTime = 120.0;
+        results.ullTimeCount = PerfTimer::SecondsToPerfTime(fTime);
+
+        SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION systemProcessorInfo = {};
+        systemProcessorInfo.UserTime.QuadPart = static_cast<LONGLONG>(fTime * 30 * 100000);
+        systemProcessorInfo.IdleTime.QuadPart = static_cast<LONGLONG>(fTime * 45 * 100000);
+        systemProcessorInfo.KernelTime.QuadPart = static_cast<LONGLONG>(fTime * 70 * 100000);
+        results.vSystemProcessorPerfInfo.push_back(systemProcessorInfo);
+
+        TargetResults targetResults;
+        targetResults.sPath = "testfile1.dat";
+        targetResults.ullFileSize = 10 * 1024 * 1024;
+        targetResults.ullReadBytesCount = 4 * 1024 * 1024;
+        targetResults.ullReadIOCount = 6;
+        targetResults.ullWriteBytesCount = 2 * 1024 * 1024;
+        targetResults.ullWriteIOCount = 10;
+        targetResults.ullBytesCount = targetResults.ullReadBytesCount + targetResults.ullWriteBytesCount;
+        targetResults.ullIOCount = targetResults.ullReadIOCount + targetResults.ullWriteIOCount;
+
+        // TODO: Histogram<float> readLatencyHistogram;
+        // TODO: Histogram<float> writeLatencyHistogram;
+
+        // TODO: IoBucketizer writeBucketizer;
+
+        timeSpan.SetCalculateIopsStdDev(true);
+        targetResults.readBucketizer.Initialize(1000, timeSpan.GetDuration());
+        for (size_t i = 0; i < timeSpan.GetDuration(); i++)
+        {
+            // add an io halfway through the bucket's time interval
+            targetResults.readBucketizer.Add(i * 1000 + 500, 100);
+        }
+
+        ThreadResults threadResults;
+        threadResults.vTargetResults.push_back(targetResults);
+        results.vThreadResults.push_back(threadResults);
+
+        vector<Results> vResults;
+        vResults.push_back(results);
+
+        // just throw away the computername - for the ut, it's as useful (and simpler)
+        // to verify a static null as anything else.
+        SystemInformation system;
+        system.sComputerName.clear();
+        system.ResetTime();
+
+        // and power plan
+        system.sActivePolicyName.clear();
+        system.sActivePolicyGuid.clear();
+
+        system.processorTopology._ulProcessorCount = 1;
+        system.processorTopology._ubPerformanceEfficiencyClass = 0;
+        system.processorTopology._fSMT = false;
+
+        system.processorTopology._vProcessorGroupInformation.clear();
+        system.processorTopology._vProcessorGroupInformation.emplace_back((WORD)0, (BYTE)1, (BYTE)1, (KAFFINITY)0x1);
+
+        ProcessorNumaInformation node;
+        node._nodeNumber = 0;
+        node._vProcessorMasks.emplace_back((WORD)0, (KAFFINITY)0x1);
+        system.processorTopology._vProcessorNumaInformation.clear();
+        system.processorTopology._vProcessorNumaInformation.push_back(node);
+
+        ProcessorSocketInformation socket;
+        socket._vProcessorMasks.emplace_back((WORD)0, (KAFFINITY)0x1);
+        system.processorTopology._vProcessorSocketInformation.clear();
+        system.processorTopology._vProcessorSocketInformation.push_back(socket);
+
+        system.processorTopology._vProcessorCoreInformation.clear();
+        system.processorTopology._vProcessorCoreInformation.emplace_back((WORD)0, (KAFFINITY)0x1, (BYTE)0);
+
+        // finally, add the timespan to the profile and dump.
+        profile.AddTimeSpan(timeSpan);
+
+        string sResults = parser.ParseResults(profile, system, vResults);
+
+        // stringify random text, quoting "'s and adding newline/preserving tabs
+        // gc some.txt |% { write-host $("`"{0}\n`"" -f $($_ -replace "`"","\`"" -replace "`t","\t")) }
+
+        const char *pcszExpectedOutput = "\n"
+            "Command Line: \n"
+            "\n"
+            "Input parameters:\n"
+            "\n"
+            "\ttimespan:   1\n"
+            "\t-------------\n"
+            "\tduration: 10s\n"
+            "\twarm up time: 5s\n"
+            "\tcool down time: 0s\n"
+            "\tgathering IOPS at intervals of 1000ms\n"
+            "\trandom seed: 0\n"
+            "\n"
+            "System information:\n\n"
+            "\tcomputer name: \n"
+            "\tstart time: \n"
+            "\n"
+            "\tcpu count:\t\t1\n"
+            "\tcore count:\t\t1\n"
+            "\tgroup count:\t\t1\n"
+            "\tnode count:\t\t1\n"
+            "\tsocket count:\t\t1\n"
+            "\theterogeneous cores:\tn\n"
+            "\n"
+            "\tactive power scheme:\t\n"
+            "\n"
+            "Results for timespan 1:\n"
+            "*******************************************************************************\n"
+            "\n"
+            "actual test time:\t120.00s\n"
+            "thread count:\t\t1\n"
+            "\n"
+            "CPU |  Usage |  User  | Kernel |  Idle\n"
+            "----------------------------------------\n"
+            "   0|  55.00%|  30.00%|  25.00%|  45.00%\n"
+            "----------------------------------------\n"
+            "avg.|  55.00%|  30.00%|  25.00%|  45.00%\n"
+            "\n"
+            "Total IO\n"
+            "thread |       bytes     |     I/Os     |    MiB/s   |  I/O per s | IopsStdDev |  file\n"
+            "-------------------------------------------------------------------------------------------\n"
+            "     0 |         6291456 |           16 |       0.05 |       0.13 |       0.00 | testfile1.dat (10MiB)\n"
+            "-------------------------------------------------------------------------------------------\n"
+            "total:           6291456 |           16 |       0.05 |       0.13 |       0.00\n"
+            "\n"
+            "Read IO\n"
+            "thread |       bytes     |     I/Os     |    MiB/s   |  I/O per s | IopsStdDev |  file\n"
+            "-------------------------------------------------------------------------------------------\n"
+            "     0 |         4194304 |            6 |       0.03 |       0.05 |       0.00 | testfile1.dat (10MiB)\n"
+            "-------------------------------------------------------------------------------------------\n"
+            "total:           4194304 |            6 |       0.03 |       0.05 |       0.00\n"
+            "\n"
+            "Write IO\n"
+            "thread |       bytes     |     I/Os     |    MiB/s   |  I/O per s | IopsStdDev |  file\n"
+            "-------------------------------------------------------------------------------------------\n"
+            "     0 |         2097152 |           10 |       0.02 |       0.08 |       0.00 | testfile1.dat (10MiB)\n"
+            "-------------------------------------------------------------------------------------------\n"
+            "total:           2097152 |           10 |       0.02 |       0.08 |       0.00\n";
+        VERIFY_ARE_EQUAL(sResults, pcszExpectedOutput);
+    }
+
+    void ResultParserUnitTests::Test_ParseProfile()
+    {
+        Profile profile;
+        ResultParser parser;
+        TimeSpan timeSpan;
+        Target target;
+
+        timeSpan.AddTarget(target);
+        profile.AddTimeSpan(timeSpan);
+
+        string s = parser.ParseProfile(profile);
+        const char *pszExpectedResult = "\nCommand Line: \n"
+            "\n"
+            "Input parameters:\n"
+            "\n"
+            "\ttimespan:   1\n"
+            "\t-------------\n"
+            "\tduration: 10s\n"
+            "\twarm up time: 5s\n"
+            "\tcool down time: 0s\n"
+            "\trandom seed: 0\n"
+            "\tpath: ''\n"
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing sequential I/O (stride: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tthreads per file: 1\n"
+            "\t\tusing I/O Completion Ports\n"
+            "\t\tIO priority: normal\n\n";
+
+        VERIFY_ARE_EQUAL(strlen(pszExpectedResult), s.length());
+        VERIFY_IS_TRUE(!strcmp(pszExpectedResult, s.c_str()));
+    }
+
+    void ResultParserUnitTests::Test_PrintTarget()
+    {
+        ResultParser parser;
+        Target target;
+
+        parser._PrintTarget(target, false, true, false);
+        const char *pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing sequential I/O (stride: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+
+        parser._sResult.clear();
+        target.SetThreadStrideInBytes(100 * 1024);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing sequential I/O (stride: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tthread stride size: 100KiB\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+        target.SetThreadStrideInBytes(0);
+
+        parser._sResult.clear();
+        target.SetMaxFileSize(2000 * 1024);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing sequential I/O (stride: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tmax file size: 1.95MiB\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+        target.SetMaxFileSize(0);
+
+        parser._sResult.clear();
+        target.SetBaseFileOffsetInBytes(2 * 1024 * 1024);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing sequential I/O (stride: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tbase file offset: 2MiB\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+        target.SetBaseFileOffsetInBytes(0);
+
+        parser._sResult.clear();
+        target.SetThroughput(1000);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing sequential I/O (stride: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n"
+            "\t\tthroughput rate-limited to 1000 B/ms\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+        target.SetThroughput(0);
+
+        parser._sResult.clear();
+        target.SetThroughputIOPS(1000);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing sequential I/O (stride: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n"
+            "\t\tthroughput rate-limited to 1000 IOPS\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+        target.SetThroughputIOPS(0);
+
+        parser._sResult.clear();
+        target.SetWriteRatio(30);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming mix test (read/write ratio: 70/30)\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing sequential I/O (stride: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+        target.SetWriteRatio(0);
+
+        parser._sResult.clear();
+        target.SetRandomDataWriteBufferSize(12341234);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\twrite buffer size: 11.77MiB\n"
+            "\t\twrite buffer source: random fill\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing sequential I/O (stride: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+
+        parser._sResult.clear();
+        target.SetRandomDataWriteBufferSourcePath("x:\\foo\\bar.dat");
+        target.SetRandomRatio(100);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\twrite buffer size: 11.77MiB\n"
+            "\t\twrite buffer source: 'x:\\foo\\bar.dat'\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+        target.SetRandomDataWriteBufferSize(0);
+        target.SetRandomDataWriteBufferSourcePath("");
+
+        parser._sResult.clear();
+        target.SetCacheMode(TargetCacheMode::DisableOSCache);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tsoftware cache disabled\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+
+        parser._sResult.clear();
+        target.SetCacheMode(TargetCacheMode::DisableOSCache);
+        target.SetWriteThroughMode(WriteThroughMode::On);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tsoftware cache disabled\n"
+            "\t\thardware write cache disabled, writethrough on\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+
+        parser._sResult.clear();
+        target.SetCacheMode(TargetCacheMode::Cached);
+        target.SetWriteThroughMode(WriteThroughMode::On);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\thardware and software write caches disabled, writethrough on\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+        target.SetWriteThroughMode(WriteThroughMode::Undefined);
+
+        parser._sResult.clear();
+        target.SetCacheMode(TargetCacheMode::DisableLocalCache);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tlocal software cache disabled, remote cache enabled\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+
+        parser._sResult.clear();
+        target.SetCacheMode(TargetCacheMode::Cached);
+        target.SetMemoryMappedIoMode(MemoryMappedIoMode::On);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tmemory mapped I/O enabled\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+
+        parser._sResult.clear();
+        target.SetMemoryMappedIoFlushMode(MemoryMappedIoFlushMode::ViewOfFile);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tmemory mapped I/O enabled, flush mode: FlushViewOfFile\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+
+        parser._sResult.clear();
+        target.SetMemoryMappedIoFlushMode(MemoryMappedIoFlushMode::NonVolatileMemory);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tmemory mapped I/O enabled, flush mode: FlushNonVolatileMemory\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+
+        parser._sResult.clear();
+        target.SetMemoryMappedIoFlushMode(MemoryMappedIoFlushMode::NonVolatileMemoryNoDrain);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tmemory mapped I/O enabled, flush mode: FlushNonVolatileMemory with no drain\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+
+        parser._sResult.clear();
+        target.SetMemoryMappedIoMode(MemoryMappedIoMode::Off);
+        target.SetMemoryMappedIoFlushMode(MemoryMappedIoFlushMode::Undefined);
+        target.SetCacheMode(TargetCacheMode::DisableLocalCache);
+        target.SetTemporaryFileHint(true);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tlocal software cache disabled, remote cache enabled\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tusing FILE_ATTRIBUTE_TEMPORARY hint\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+
+        parser._sResult.clear();
+        target.SetRandomAccessHint(true);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tlocal software cache disabled, remote cache enabled\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tusing FILE_FLAG_RANDOM_ACCESS hint\n"
+            "\t\tusing FILE_ATTRIBUTE_TEMPORARY hint\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+
+        parser._sResult.clear();
+        target.SetRandomAccessHint(false);
+        target.SetTemporaryFileHint(false);
+        target.SetSequentialScanHint(true);
+        parser._PrintTarget(target, false, true, false);
+        pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tlocal software cache disabled, remote cache enabled\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing random I/O (alignment: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tusing FILE_FLAG_SEQUENTIAL_SCAN hint\n"
+            "\t\tIO priority: normal\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+    }
+
+    void ResultParserUnitTests::Test_PrintTargetDistributionPct()
+    {
+        ResultParser parser;
+        Target target;
+
+        vector<DistributionRange> v;
+
+        // these match the CmdLineParser UTs
+
+        // -rdpct10/10:10/10:0/10, though we need to produce the tail here
+        v.emplace_back(0, 10, make_pair(0, 10));
+        v.emplace_back(10, 10, make_pair(10, 10));
+        v.emplace_back(20, 0, make_pair(20, 10));   // zero IO% length hole
+        v.emplace_back(20, 80, make_pair(30, 70));
+        target.SetDistributionRange(v, DistributionType::Percent);
+
+        parser._PrintTarget(target, false, true, false);
+        const char *pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing sequential I/O (stride: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n"
+            "\t\tIO Distribution:\n"
+            "\t\t    10% of IO => [ 0% -  10%) of target\n"
+            "\t\t    10% of IO => [10% -  20%) of target\n"
+            "\t\t     0% of IO => [20% -  30%) of target\n"
+            "\t\t    80% of IO => [30% - 100%) of target\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+        v.clear();
+        parser._sResult.clear();
+    }
+
+    void ResultParserUnitTests::Test_PrintTargetDistributionAbs()
+    {
+        ResultParser parser;
+        Target target;
+
+        vector<DistributionRange> v;
+
+        // these match the CmdLineParser UTs
+
+        // -rdabs10/1G:10/1G:0/100G, again producing tail
+        v.emplace_back(0,10, make_pair(0, 1*GB));
+        v.emplace_back(10,10, make_pair(1*GB, 1*GB));
+        v.emplace_back(20,0, make_pair(2*GB, 100*GB));
+        v.emplace_back(20,80, make_pair(102*GB, 0));
+        target.SetDistributionRange(v, DistributionType::Absolute);
+
+        parser._PrintTarget(target, false, true, false);
+        const char *pszExpectedResult = "\tpath: ''\n" \
+            "\t\tthink time: 0ms\n"
+            "\t\tburst size: 0\n"
+            "\t\tusing software cache\n"
+            "\t\tusing hardware write cache, writethrough off\n"
+            "\t\tperforming read test\n"
+            "\t\tblock size: 64KiB\n"
+            "\t\tusing sequential I/O (stride: 64KiB)\n"
+            "\t\tnumber of outstanding I/O operations per thread: 2\n"
+            "\t\tIO priority: normal\n"
+            "\t\tIO Distribution:\n"
+            "\t\t    10% of IO => [     0    -      1GiB)\n"
+            "\t\t    10% of IO => [     1GiB -      2GiB)\n"
+            "\t\t     0% of IO => [     2GiB -    102GiB)\n"
+            "\t\t    80% of IO => [   102GiB -       end)\n";
+        VERIFY_ARE_EQUAL(parser._sResult, pszExpectedResult);
+        v.clear();
+        parser._sResult.clear();
+    }
+
+    void ResultParserUnitTests::Test_PrintEffectiveDistributionPct()
+    {
+        // the first matches the corresponding IORequestGenerator effdist UT
+        ResultParser parser;
+
+        Target target;
+        target.SetBlockAlignmentInBytes(4*KB);
+        target.SetBlockSizeInBytes(4*KB);
+
+        Random r;
+        ThreadParameters tp;
+        tp.pRand = &r;
+
+        vector<DistributionRange> v;
+
+        // -rdpct10/10:10/10:0/10 + tail
+        // this is the same distribution in the cmdlineparser UT
+        v.emplace_back(0, 10, make_pair(0, 10));
+        v.emplace_back(10, 10, make_pair(10, 10));
+        v.emplace_back(20, 0, make_pair(20, 10));   // zero IO% length hole
+        v.emplace_back(20, 80, make_pair(30, 70));
+        target.SetDistributionRange(v, DistributionType::Percent);
+        tp.vTargets.push_back(target);
+
+        {
+            ThreadTargetState tts(&tp, 0, 100*KB);
+
+            Results results;
+            ThreadResults threadResults;
+            TargetResults targetResults;
+
+            targetResults.sPath = "testfile.dat";
+            targetResults.vDistributionRange = tts._vDistributionRange;
+
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+
+            parser._PrintEffectiveDistributions(results);
+
+            const char* pcszResults = "\nEffective IO Distributions\n" \
+                "--------------------------\n"
+                "target: testfile.dat [thread: 0]\n"
+                "    10% of IO => [     0    -      8KiB)\n"
+                "    10% of IO => [     8KiB -     20KiB)\n"
+                "    80% of IO => [    28KiB -    100KiB)\n";
+            VERIFY_ARE_EQUAL(pcszResults, parser._sResult);
+
+            parser._sResult.clear();
+        }
+
+        //
+        // Tests of distribution deduplication.
+        //
+
+        // now repeat, duplicating the thread result for a second thread
+
+        {
+            ThreadTargetState tts(&tp, 0, 100*KB);
+
+            Results results;
+            ThreadResults threadResults;
+            TargetResults targetResults;
+
+            targetResults.sPath = "testfile.dat";
+            targetResults.vDistributionRange = tts._vDistributionRange;
+
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+            results.vThreadResults.push_back(threadResults);
+
+            parser._PrintEffectiveDistributions(results);
+
+            const char* pcszResults = "\nEffective IO Distributions\n" \
+                "--------------------------\n"
+                "target: testfile.dat [thread: 0 1]\n"
+                "    10% of IO => [     0    -      8KiB)\n"
+                "    10% of IO => [     8KiB -     20KiB)\n"
+                "    80% of IO => [    28KiB -    100KiB)\n";
+            VERIFY_ARE_EQUAL(pcszResults, parser._sResult);
+
+            parser._sResult.clear();
+        }
+
+        // now repeat, for a third thread - ellision
+
+        {
+            ThreadTargetState tts(&tp, 0, 100*KB);
+
+            Results results;
+            ThreadResults threadResults;
+            TargetResults targetResults;
+
+            targetResults.sPath = "testfile.dat";
+            targetResults.vDistributionRange = tts._vDistributionRange;
+
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+            results.vThreadResults.push_back(threadResults);
+            results.vThreadResults.push_back(threadResults);
+
+            parser._PrintEffectiveDistributions(results);
+
+            const char* pcszResults = "\nEffective IO Distributions\n" \
+                "--------------------------\n"
+                "target: testfile.dat [thread: 0 - 2]\n"
+                "    10% of IO => [     0    -      8KiB)\n"
+                "    10% of IO => [     8KiB -     20KiB)\n"
+                "    80% of IO => [    28KiB -    100KiB)\n";
+            VERIFY_ARE_EQUAL(pcszResults, parser._sResult);
+
+            parser._sResult.clear();
+        }
+
+        // now repeat, moving the third thread to a different target
+
+        {
+            ThreadTargetState tts(&tp, 0, 100*KB);
+
+            Results results;
+            ThreadResults threadResults;
+            TargetResults targetResults;
+
+            targetResults.vDistributionRange = tts._vDistributionRange;
+
+            targetResults.sPath = "testfile.dat";
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+            results.vThreadResults.push_back(threadResults);
+
+            targetResults.sPath = "testfile2.dat";
+            threadResults.vTargetResults.clear();
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+
+            parser._PrintEffectiveDistributions(results);
+
+            const char* pcszResults = "\nEffective IO Distributions\n" \
+                "--------------------------\n"
+                "target: testfile.dat [thread: 0 1]\n"
+                "target: testfile2.dat [thread: 2]\n"
+                "    10% of IO => [     0    -      8KiB)\n"
+                "    10% of IO => [     8KiB -     20KiB)\n"
+                "    80% of IO => [    28KiB -    100KiB)\n";
+            VERIFY_ARE_EQUAL(pcszResults, parser._sResult);
+
+            parser._sResult.clear();
+        }
+
+        // now repeat, four threads on the first target, three contiguous and one not
+        // the thread on the second target is used to create the gap - ellision will occur
+        // and the fourth thread will stand alone
+
+        {
+            ThreadTargetState tts(&tp, 0, 100*KB);
+
+            Results results;
+            ThreadResults threadResults;
+            TargetResults targetResults;
+
+            targetResults.vDistributionRange = tts._vDistributionRange;
+
+            targetResults.sPath = "testfile.dat";
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+            results.vThreadResults.push_back(threadResults);
+            results.vThreadResults.push_back(threadResults);
+
+            threadResults.vTargetResults.clear();
+            targetResults.sPath = "testfile2.dat";
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+
+            threadResults.vTargetResults.clear();
+            targetResults.sPath = "testfile.dat";
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+
+            parser._PrintEffectiveDistributions(results);
+
+            const char* pcszResults = "\nEffective IO Distributions\n" \
+                "--------------------------\n"
+                "target: testfile.dat [thread: 0 - 2 4]\n"
+                "target: testfile2.dat [thread: 3]\n"
+                "    10% of IO => [     0    -      8KiB)\n"
+                "    10% of IO => [     8KiB -     20KiB)\n"
+                "    80% of IO => [    28KiB -    100KiB)\n";
+            VERIFY_ARE_EQUAL(pcszResults, parser._sResult);
+
+            parser._sResult.clear();
+        }
+
+        // two distinct distributions which share the same IO%
+
+        {
+            ThreadTargetState tts1(&tp, 0, 100*KB);
+            ThreadTargetState tts2(&tp, 0, 1*MB);
+
+            Results results;
+            ThreadResults threadResults;
+            TargetResults targetResults;
+
+            targetResults.vDistributionRange = tts1._vDistributionRange;
+            targetResults.sPath = "testfile.dat";
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+
+            threadResults.vTargetResults.clear();
+
+            targetResults.vDistributionRange = tts2._vDistributionRange;
+            targetResults.sPath = "testfile2.dat";
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+
+            parser._PrintEffectiveDistributions(results);
+
+            const char* pcszResults = "\nEffective IO Distributions\n" \
+                "--------------------------\n"
+                "target: testfile.dat [thread: 0]\n"
+                "    10% of IO => [     0    -      8KiB)\n"
+                "    10% of IO => [     8KiB -     20KiB)\n"
+                "    80% of IO => [    28KiB -    100KiB)\n"
+                "target: testfile2.dat [thread: 1]\n"
+                "    10% of IO => [     0    -    100KiB)\n"
+                "    10% of IO => [   100KiB -    204KiB)\n"
+                "    80% of IO => [   304KiB -      1MiB)\n";
+            VERIFY_ARE_EQUAL(pcszResults, parser._sResult);
+
+            parser._sResult.clear();
+        }
+
+        tp.vTargets.clear();
+        v.clear();
+    }
+
+    void ResultParserUnitTests::Test_PrintEffectiveDistributionAbs()
+    {
+        // the first matches the corresponding IORequestGenerator effdist UT
+        ResultParser parser;
+
+        Target target;
+        target.SetBlockAlignmentInBytes(4*KB);
+        target.SetBlockSizeInBytes(4*KB);
+
+        Random r;
+        ThreadParameters tp;
+
+        vector<DistributionRange> v;
+
+        // -rdabs10/1G:10/1G:0/100G, again producing tail - with autoscale (0)
+        // this is the same distribution in the cmdlineparser UT
+        // aligned tail range
+        v.emplace_back(0,10, make_pair(0, 1*GB));
+        v.emplace_back(10,10, make_pair(1*GB, 1*GB));
+        v.emplace_back(20,0, make_pair(2*GB, 100*GB));
+        v.emplace_back(20,80, make_pair(102*GB, 0));
+        target.SetDistributionRange(v, DistributionType::Absolute);
+        tp.vTargets.push_back(target);
+
+        {
+            ThreadTargetState tts(&tp, 0, 200*GB);
+
+            Results results;
+            ThreadResults threadResults;
+            TargetResults targetResults;
+
+            targetResults.sPath = "testfile.dat";
+            targetResults.vDistributionRange = tts._vDistributionRange;
+
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+
+            parser._PrintEffectiveDistributions(results);
+
+            const char* pcszResults = "\nEffective IO Distributions\n" \
+                "--------------------------\n"
+                "target: testfile.dat [thread: 0]\n"
+                "    10% of IO => [     0    -      1GiB)\n"
+                "    10% of IO => [     1GiB -      2GiB)\n"
+                "    80% of IO => [   102GiB -    200GiB)\n";
+            VERIFY_ARE_EQUAL(pcszResults, parser._sResult);
+
+            tp.vTargets.clear();
+            v.clear();
+            parser._sResult.clear();
+        }
+
+        // -rdabs10/50k:20/10k:30/1G on 100KiB target - autoscale tail, but trimmed on last spec'd range
+        // this results in logical truncation since the covered ranges are only 60% of IO%, a case which
+        // is specific to absolute distributions.
+        v.emplace_back(0, 10, make_pair(0, 50*KB));
+        v.emplace_back(10, 20, make_pair(50*KB, 10*KB));
+        v.emplace_back(30, 30, make_pair(60*KB, 1*GB));
+        v.emplace_back(60, 40, make_pair(1*GB + 60*KB, 0));
+        target.SetDistributionRange(v, DistributionType::Absolute);
+        tp.vTargets.push_back(target);
+
+        {
+            ThreadTargetState tts(&tp, 0, 100*KB);
+
+            Results results;
+            ThreadResults threadResults;
+            TargetResults targetResults;
+
+            targetResults.sPath = "testfile.dat";
+            targetResults.vDistributionRange = tts._vDistributionRange;
+
+            threadResults.vTargetResults.push_back(targetResults);
+            results.vThreadResults.push_back(threadResults);
+
+            parser._PrintEffectiveDistributions(results);
+
+            const char* pcszResults = "\nEffective IO Distributions\n" \
+                "--------------------------\n"
+                "target: testfile.dat [thread: 0]\n"
+                "    16.7% of IO => [     0    -     50KiB)\n"
+                "    33.3% of IO => [    50KiB -     60KiB)\n"
+                "    50.0% of IO => [    60KiB -    100KiB)\n";
+            VERIFY_ARE_EQUAL(pcszResults, parser._sResult);
+
+            tp.vTargets.clear();
+            v.clear();
+            parser._sResult.clear();
+        }
+    }
+}

CristalDiskMark/source/diskspd22/UnitTests/ResultParser/ResultParser.UnitTests.h

@@ -0,0 +1,52 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include "WexTestClass.h"
+
+namespace UnitTests
+{
+    BEGIN_MODULE()
+        MODULE_PROPERTY(L"Feature", L"ResultParser")
+    END_MODULE()
+
+    class ResultParserUnitTests : public WEX::TestClass<ResultParserUnitTests>
+    {
+    public:
+        TEST_CLASS(ResultParserUnitTests);
+        TEST_METHOD(Test_ParseResults);
+        TEST_METHOD(Test_ParseProfile);
+        TEST_METHOD(Test_PrintTarget);
+        TEST_METHOD(Test_PrintTargetDistributionPct);
+        TEST_METHOD(Test_PrintTargetDistributionAbs);
+        TEST_METHOD(Test_PrintEffectiveDistributionPct);
+        TEST_METHOD(Test_PrintEffectiveDistributionAbs);
+    };
+}
+

CristalDiskMark/source/diskspd22/UnitTests/ResultParser/ResultParser.rc

@@ -0,0 +1,17 @@
+#include <windows.h>
+#include "Version.h"
+
+#include <ntverp.h>
+
+#define VER_FILETYPE                VFT_APP
+#define VER_FILESUBTYPE             VFT2_UNKNOWN
+#define VER_FILEDESCRIPTION_STR     "DiskSpd Storage Performance Tool"
+#define VER_INTERNALNAME_STR        "ResultParser.UnitTests.dll"
+
+#undef VER_PRODUCTVERSION
+#define VER_PRODUCTVERSION DISKSPD_MAJOR,DISKSPD_MINOR,DISKSPD_BUILD,DISKSPD_QFE
+
+#undef VER_PRODUCTVERSION_STR
+#define VER_PRODUCTVERSION_STR      DISKSPD_NUMERIC_VERSION_STRING
+
+#include "common.ver"

CristalDiskMark/source/diskspd22/UnitTests/ResultParser/stdafx.h

@@ -0,0 +1,33 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+
+#include <windows.h>
+

CristalDiskMark/source/diskspd22/UnitTests/XmlProfileParser/XmlProfileParser.UnitTests.h

@@ -0,0 +1,116 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include "WexTestClass.h"
+#include "XmlProfileParser.h"
+#include <string>
+using namespace std;
+
+namespace UnitTests
+{
+    BEGIN_MODULE()
+        MODULE_PROPERTY(L"Feature", L"XmlProfileParser")
+    END_MODULE()
+
+    MODULE_SETUP(ModuleSetup);
+    MODULE_CLEANUP(ModuleCleanup);
+
+    class XmlProfileParserUnitTests : public WEX::TestClass<XmlProfileParserUnitTests>
+    {
+    public:
+        TEST_CLASS(XmlProfileParserUnitTests)
+
+        TEST_CLASS_SETUP(ClassSetup);
+        TEST_CLASS_CLEANUP(ClassCleanup);
+
+        TEST_METHOD_SETUP(MethodSetup);
+        TEST_METHOD_CLEANUP(MethodCleanup);
+
+        TEST_METHOD(Test_ParseDistributionAbsolute);
+        TEST_METHOD(Test_ParseDistributionPercent);
+        TEST_METHOD(Test_ParseFile);
+        TEST_METHOD(Test_ParseFileGlobalRequestCount);
+        TEST_METHOD(Test_ParseFilePrecreateFilesOnlyFilesWithConstantOrZeroSizes);
+        TEST_METHOD(Test_ParseFilePrecreateFilesOnlyFilesWithConstantSizes);
+        TEST_METHOD(Test_ParseFilePrecreateFilesUseMaxSize);
+        TEST_METHOD(Test_ParseFileWriteBufferContentRandom);
+        TEST_METHOD(Test_ParseFileWriteBufferContentRandomNoFilePath);
+        TEST_METHOD(Test_ParseFileWriteBufferContentRandomWithFilePath);
+        TEST_METHOD(Test_ParseFileWriteBufferContentSequential);
+        TEST_METHOD(Test_ParseFileWriteBufferContentZero);
+        TEST_METHOD(Test_ParseGroupAffinity);
+        TEST_METHOD(Test_ParseNonGroupAffinity);
+        TEST_METHOD(Test_ParseRandomSequentialMixed);
+        TEST_METHOD(Test_ParseTemplateTargets);
+        TEST_METHOD(Test_ParseThroughput);
+
+        //
+        // Utility wrapping the specification and validation of a given distribution.
+        //
+        // Note that % and abs distributions are represented in the same way, only
+        // differing in the relative scale of the target spans.
+        //
+
+        #pragma warning(push)
+        #pragma warning(disable:4200) // zero length array
+
+        typedef struct {
+            UINT32 size;
+            struct {
+                UINT32 io;
+                UINT64 target;
+            } range[];
+        } RangePair;
+
+        typedef struct {
+            UINT32 size;
+            struct{
+                UINT32 ioBase, ioSpan;
+                UINT64 targetBase, targetSpan;
+            } range[];
+        } DistQuad;
+        
+        #pragma warning(pop)
+
+        void ValidateDistribution(
+            const PWCHAR desc,
+            boolean expectedParseResult,
+            boolean expectedValidate,
+            DistributionType type,
+            PCHAR xmlDoc,
+            const RangePair *insert,
+            const DistQuad *dist
+            );
+
+    private:
+        string _sTempFilePath;
+        HMODULE _hModule;
+    };
+}

CristalDiskMark/source/diskspd22/UnitTests/XmlProfileParser/XmlProfileParser.rc

@@ -0,0 +1,19 @@
+#include <windows.h>
+#include "Version.h"
+
+DISKSPD.XSD HTML "..\\XmlProfileParser\\diskspd.xsd"
+
+#include <ntverp.h>
+
+#define VER_FILETYPE                VFT_APP
+#define VER_FILESUBTYPE             VFT2_UNKNOWN
+#define VER_FILEDESCRIPTION_STR     "DiskSpd Storage Performance Tool"
+#define VER_INTERNALNAME_STR        "XmlProfileParser.UnitTests.dll"
+
+#undef VER_PRODUCTVERSION
+#define VER_PRODUCTVERSION DISKSPD_MAJOR,DISKSPD_MINOR,DISKSPD_BUILD,DISKSPD_QFE
+
+#undef VER_PRODUCTVERSION_STR
+#define VER_PRODUCTVERSION_STR      DISKSPD_NUMERIC_VERSION_STRING
+
+#include "common.ver"

CristalDiskMark/source/diskspd22/UnitTests/XmlProfileParser/stdafx.h

@@ -0,0 +1,32 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+
+#include <windows.h>

CristalDiskMark/source/diskspd22/UnitTests/XmlResultParser/XmlResultParser.UnitTests.cpp

@@ -0,0 +1,464 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#include "StdAfx.h"
+#include "XmlResultParser.UnitTests.h"
+#include "Common.h"
+#include "xmlresultparser.h"
+#include <stdlib.h>
+#include <vector>
+
+using namespace WEX::TestExecution;
+using namespace WEX::Logging;
+using namespace std;
+
+namespace UnitTests
+{
+    void XmlResultParserUnitTests::Test_ParseResults()
+    {
+        Profile profile;
+        TimeSpan timeSpan;
+        Target target;
+        XmlResultParser parser;
+
+        Results results;
+        results.fUseETW = false;
+        double fTime = 120.0;
+        results.ullTimeCount = PerfTimer::SecondsToPerfTime(fTime);
+
+        // First group has 1 active cpu
+        // 30% user, 45% idle, 25% non-idle kernel (45% + 25% = 70%)
+        SYSTEM_PROCESSOR_PERFORMANCE_INFORMATION systemProcessorInfo = { 0 };
+        systemProcessorInfo.UserTime.QuadPart = static_cast<LONGLONG>(fTime * 30 * 100000);
+        systemProcessorInfo.IdleTime.QuadPart = static_cast<LONGLONG>(fTime * 45 * 100000);
+        systemProcessorInfo.KernelTime.QuadPart = static_cast<LONGLONG>(fTime * 70 * 100000);
+        results.vSystemProcessorPerfInfo.push_back(systemProcessorInfo);
+
+        // Second group has 2 active
+        // 100% idle
+        systemProcessorInfo.UserTime.QuadPart = static_cast<LONGLONG>(fTime * 0 * 100000);
+        systemProcessorInfo.IdleTime.QuadPart = static_cast<LONGLONG>(fTime * 100 * 100000);
+        systemProcessorInfo.KernelTime.QuadPart = static_cast<LONGLONG>(fTime * 100 * 100000);
+        results.vSystemProcessorPerfInfo.push_back(systemProcessorInfo);
+        results.vSystemProcessorPerfInfo.push_back(systemProcessorInfo);
+
+        // TODO: multiple target cases, full profile/result variations
+        target.SetPath("testfile1.dat");
+        target.SetCacheMode(TargetCacheMode::DisableOSCache);
+        target.SetWriteThroughMode(WriteThroughMode::On);
+        target.SetThroughputIOPS(1000);
+
+        timeSpan.AddTarget(target);
+        timeSpan.SetCalculateIopsStdDev(true);
+
+        TargetResults targetResults;
+        targetResults.sPath = "testfile1.dat";
+        targetResults.ullFileSize = 10 * 1024 * 1024;
+        targetResults.ullReadBytesCount = 4 * 1024 * 1024;
+        targetResults.ullReadIOCount = 6;
+        targetResults.ullWriteBytesCount = 2 * 1024 * 1024;
+        targetResults.ullWriteIOCount = 10;
+        targetResults.ullBytesCount = targetResults.ullReadBytesCount + targetResults.ullWriteBytesCount;
+        targetResults.ullIOCount = targetResults.ullReadIOCount + targetResults.ullWriteIOCount;
+
+        // TODO: Histogram<float> readLatencyHistogram;
+        // TODO: Histogram<float> writeLatencyHistogram;
+        // TODO: IoBucketizer writeBucketizer;
+
+        targetResults.readBucketizer.Initialize(1000, timeSpan.GetDuration());
+        for (size_t i = 0; i < timeSpan.GetDuration(); i++)
+        {
+            // add an io halfway through the bucket's time interval
+            targetResults.readBucketizer.Add(i*1000 + 500, 0);
+        }
+
+        ThreadResults threadResults;
+        threadResults.vTargetResults.push_back(targetResults);
+        results.vThreadResults.push_back(threadResults);
+
+        vector<Results> vResults;
+        vResults.push_back(results);
+
+        // Just throw away the computername, pp and reset the timestamp - for the ut, it's
+        // as useful (and simpler) to verify statics as anything else. Reconstruct the
+        // processor topo to a fixed example as well. Note that the performance
+        // efficiency class must be placed since it is calculated on the fly during
+        // the actual GLPIEx enumeration. If we could shim GLPIEx ...
+        SystemInformation system;
+        system.ResetTime();
+        system.sComputerName.clear();
+        system.sActivePolicyName.clear();
+        system.sActivePolicyGuid.clear();
+
+        system.processorTopology._ulProcessorCount = 3;
+        system.processorTopology._ubPerformanceEfficiencyClass = 1;
+        system.processorTopology._fSMT = true;
+
+        system.processorTopology._vProcessorGroupInformation.clear();
+        system.processorTopology._vProcessorGroupInformation.emplace_back((WORD)0, (BYTE)1, (BYTE)1, (KAFFINITY)0x1);
+        system.processorTopology._vProcessorGroupInformation.emplace_back((WORD)1, (BYTE)4, (BYTE)2, (KAFFINITY)0x3);
+
+        ProcessorNumaInformation node;
+        node._nodeNumber = 0;
+        node._vProcessorMasks.emplace_back((WORD)0, (KAFFINITY)0x1);
+        node._vProcessorMasks.emplace_back((WORD)1, (KAFFINITY)0x3);
+        system.processorTopology._vProcessorNumaInformation.clear();
+        system.processorTopology._vProcessorNumaInformation.push_back(node);
+
+        ProcessorSocketInformation socket;
+        socket._vProcessorMasks.emplace_back((WORD)0, (KAFFINITY)0x1);
+        socket._vProcessorMasks.emplace_back((WORD)1, (KAFFINITY)0x3);
+        system.processorTopology._vProcessorSocketInformation.clear();
+        system.processorTopology._vProcessorSocketInformation.push_back(socket);
+
+        system.processorTopology._vProcessorCoreInformation.clear();
+        system.processorTopology._vProcessorCoreInformation.emplace_back((WORD)0, (KAFFINITY)0x1, (BYTE)0);
+        system.processorTopology._vProcessorCoreInformation.emplace_back((WORD)1, (KAFFINITY)0x3, (BYTE)1);
+
+        // finally, add the timespan to the profile and dump.
+        profile.AddTimeSpan(timeSpan);
+
+        string sResults = parser.ParseResults(profile, system, vResults);
+
+        // stringify random text, quoting "'s and adding newline/preserving tabs
+        // gc some.txt |% { write-host $("`"{0}\n`"" -f $($_ -replace "`"","\`"" -replace "`t","\t")) }
+
+        const char *pcszExpectedOutput = \
+            "<Results>\n"
+            "  <System>\n"
+            "    <ComputerName></ComputerName>\n"
+            "    <Tool>\n"
+            "      <Version>" DISKSPD_NUMERIC_VERSION_STRING "</Version>\n"
+            "      <VersionDate>" DISKSPD_DATE_VERSION_STRING "</VersionDate>\n"
+            "    </Tool>\n"
+            "    <RunTime></RunTime>\n"
+            "    <PowerScheme Name=\"\" Guid=\"\"/>\n"
+            "    <ProcessorTopology Heterogeneous=\"true\">\n"
+            "      <Group Group=\"0\" MaximumProcessors=\"1\" ActiveProcessors=\"1\" ActiveProcessorMask=\"0x1\"/>\n"
+            "      <Group Group=\"1\" MaximumProcessors=\"4\" ActiveProcessors=\"2\" ActiveProcessorMask=\"0x3\"/>\n"
+            "      <Node Node=\"0\">\n"
+            "        <Group Group=\"0\" Mask=\"0x1\"/>\n"
+            "        <Group Group=\"1\" Mask=\"0x3\"/>\n"
+            "      </Node>\n"
+            "      <Socket Socket=\"0\">\n"
+            "        <Group Group=\"0\" Mask=\"0x1\"/>\n"
+            "        <Group Group=\"1\" Mask=\"0x3\"/>\n"
+            "      </Socket>\n"
+            "      <Core Group=\"0\" Core=\"0\" Mask=\"0x1\" EfficiencyClass=\"0\"/>\n"
+            "      <Core Group=\"1\" Core=\"0\" Mask=\"0x3\" EfficiencyClass=\"1\"/>\n"
+            "    </ProcessorTopology>\n"
+            "  </System>\n"
+            "  <Profile>\n"
+            "    <Progress>0</Progress>\n"
+            "    <ResultFormat>text</ResultFormat>\n"
+            "    <Verbose>false</Verbose>\n"
+            "    <TimeSpans>\n"
+            "      <TimeSpan>\n"
+            "        <CompletionRoutines>false</CompletionRoutines>\n"
+            "        <MeasureLatency>false</MeasureLatency>\n"
+            "        <CalculateIopsStdDev>true</CalculateIopsStdDev>\n"
+            "        <DisableAffinity>false</DisableAffinity>\n"
+            "        <Duration>10</Duration>\n"
+            "        <Warmup>5</Warmup>\n"
+            "        <Cooldown>0</Cooldown>\n"
+            "        <ThreadCount>0</ThreadCount>\n"
+            "        <RequestCount>0</RequestCount>\n"
+            "        <IoBucketDuration>1000</IoBucketDuration>\n"
+            "        <RandSeed>0</RandSeed>\n"
+            "        <Targets>\n"
+            "          <Target>\n"
+            "            <Path>testfile1.dat</Path>\n"
+            "            <BlockSize>65536</BlockSize>\n"
+            "            <BaseFileOffset>0</BaseFileOffset>\n"
+            "            <SequentialScan>false</SequentialScan>\n"
+            "            <RandomAccess>false</RandomAccess>\n"
+            "            <TemporaryFile>false</TemporaryFile>\n"
+            "            <UseLargePages>false</UseLargePages>\n"
+            "            <DisableOSCache>true</DisableOSCache>\n"
+            "            <WriteThrough>true</WriteThrough>\n"
+            "            <WriteBufferContent>\n"
+            "              <Pattern>sequential</Pattern>\n"
+            "            </WriteBufferContent>\n"
+            "            <ParallelAsyncIO>false</ParallelAsyncIO>\n"
+            "            <StrideSize>65536</StrideSize>\n"
+            "            <InterlockedSequential>false</InterlockedSequential>\n"
+            "            <ThreadStride>0</ThreadStride>\n"
+            "            <MaxFileSize>0</MaxFileSize>\n"
+            "            <RequestCount>2</RequestCount>\n"
+            "            <WriteRatio>0</WriteRatio>\n"
+            "            <Throughput unit=\"IOPS\">1000</Throughput>\n"
+            "            <ThreadsPerFile>1</ThreadsPerFile>\n"
+            "            <IOPriority>3</IOPriority>\n"
+            "            <Weight>1</Weight>\n"
+            "          </Target>\n"
+            "        </Targets>\n"
+            "      </TimeSpan>\n"
+            "    </TimeSpans>\n"
+            "  </Profile>\n"
+            "  <TimeSpan>\n"
+            "    <TestTimeSeconds>120.00</TestTimeSeconds>\n"
+            "    <ThreadCount>1</ThreadCount>\n"
+            "    <RequestCount>0</RequestCount>\n"
+            "    <ProcCount>3</ProcCount>\n"
+            "    <CpuUtilization>\n"
+            "      <CPU>\n"
+            "        <Socket>0</Socket>\n"
+            "        <Node>0</Node>\n"
+            "        <Group>0</Group>\n"
+            "        <Core>0</Core>\n"
+            "        <EfficiencyClass>0</EfficiencyClass>\n"
+            "        <Id>0</Id>\n"
+            "        <UsagePercent>55.00</UsagePercent>\n"
+            "        <UserPercent>30.00</UserPercent>\n"
+            "        <KernelPercent>25.00</KernelPercent>\n"
+            "        <IdlePercent>45.00</IdlePercent>\n"
+            "      </CPU>\n"
+            "      <CPU>\n"
+            "        <Socket>0</Socket>\n"
+            "        <Node>0</Node>\n"
+            "        <Group>1</Group>\n"
+            "        <Core>0</Core>\n"
+            "        <EfficiencyClass>1</EfficiencyClass>\n"
+            "        <Id>0</Id>\n"
+            "        <UsagePercent>0.00</UsagePercent>\n"
+            "        <UserPercent>0.00</UserPercent>\n"
+            "        <KernelPercent>0.00</KernelPercent>\n"
+            "        <IdlePercent>100.00</IdlePercent>\n"
+            "      </CPU>\n"
+            "      <CPU>\n"
+            "        <Socket>0</Socket>\n"
+            "        <Node>0</Node>\n"
+            "        <Group>1</Group>\n"
+            "        <Core>0</Core>\n"
+            "        <EfficiencyClass>1</EfficiencyClass>\n"
+            "        <Id>1</Id>\n"
+            "        <UsagePercent>0.00</UsagePercent>\n"
+            "        <UserPercent>0.00</UserPercent>\n"
+            "        <KernelPercent>0.00</KernelPercent>\n"
+            "        <IdlePercent>100.00</IdlePercent>\n"
+            "      </CPU>\n"
+            "      <Average>\n"
+            "        <UsagePercent>18.33</UsagePercent>\n"
+            "        <UserPercent>10.00</UserPercent>\n"
+            "        <KernelPercent>8.33</KernelPercent>\n"
+            "        <IdlePercent>81.67</IdlePercent>\n"
+            "      </Average>\n"
+            "    </CpuUtilization>\n"
+            "    <Iops>\n"
+            "      <ReadIopsStdDev>0.000</ReadIopsStdDev>\n"
+            "      <IopsStdDev>0.000</IopsStdDev>\n"
+            "      <Bucket SampleMillisecond=\"1000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "      <Bucket SampleMillisecond=\"2000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "      <Bucket SampleMillisecond=\"3000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "      <Bucket SampleMillisecond=\"4000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "      <Bucket SampleMillisecond=\"5000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "      <Bucket SampleMillisecond=\"6000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "      <Bucket SampleMillisecond=\"7000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "      <Bucket SampleMillisecond=\"8000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "      <Bucket SampleMillisecond=\"9000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "      <Bucket SampleMillisecond=\"10000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "    </Iops>\n"
+            "    <Thread>\n"
+            "      <Id>0</Id>\n"
+            "      <Target>\n"
+            "        <Path>testfile1.dat</Path>\n"
+            "        <BytesCount>6291456</BytesCount>\n"
+            "        <FileSize>10485760</FileSize>\n"
+            "        <IOCount>16</IOCount>\n"
+            "        <ReadBytes>4194304</ReadBytes>\n"
+            "        <ReadCount>6</ReadCount>\n"
+            "        <WriteBytes>2097152</WriteBytes>\n"
+            "        <WriteCount>10</WriteCount>\n"
+            "        <Iops>\n"
+            "          <ReadIopsStdDev>0.000</ReadIopsStdDev>\n"
+            "          <IopsStdDev>0.000</IopsStdDev>\n"
+            "          <Bucket SampleMillisecond=\"1000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "          <Bucket SampleMillisecond=\"2000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "          <Bucket SampleMillisecond=\"3000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "          <Bucket SampleMillisecond=\"4000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "          <Bucket SampleMillisecond=\"5000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "          <Bucket SampleMillisecond=\"6000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "          <Bucket SampleMillisecond=\"7000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "          <Bucket SampleMillisecond=\"8000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "          <Bucket SampleMillisecond=\"9000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "          <Bucket SampleMillisecond=\"10000\" Read=\"1\" Write=\"0\" Total=\"1\" ReadMinLatencyMilliseconds=\"0.000\" ReadMaxLatencyMilliseconds=\"0.000\" ReadAvgLatencyMilliseconds=\"0.000\" ReadLatencyStdDev=\"0.000\" WriteMinLatencyMilliseconds=\"0.000\" WriteMaxLatencyMilliseconds=\"0.000\" WriteAvgLatencyMilliseconds=\"0.000\" WriteLatencyStdDev=\"0.000\"/>\n"
+            "        </Iops>\n"
+            "      </Target>\n"
+            "    </Thread>\n"
+            "  </TimeSpan>\n"
+            "</Results>";
+
+#if 0
+        HANDLE h;
+        DWORD written;
+        h = CreateFileW(L"g:\\xmlresult-received.txt", GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+        WriteFile(h, sResults.c_str(), (DWORD)sResults.length(), &written, NULL);
+        VERIFY_ARE_EQUAL(sResults.length(), written);
+        CloseHandle(h);
+
+        h = CreateFileW(L"g:\\xmlresult-expected.txt", GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+        WriteFile(h, pcszExpectedOutput, (DWORD)strlen(pcszExpectedOutput), &written, NULL);
+        VERIFY_ARE_EQUAL((DWORD)strlen(pcszExpectedOutput), written);
+        CloseHandle(h);
+
+        printf("--\n%s\n", sResults.c_str());
+        printf("-------------------------------------------------\n");
+        printf("--\n%s\n", pcszExpectedOutput);
+#endif
+
+        VERIFY_ARE_EQUAL(0, strcmp(sResults.c_str(), pcszExpectedOutput));
+    }
+
+    void XmlResultParserUnitTests::Test_ParseProfile()
+    {
+        Profile profile;
+        XmlResultParser parser;
+        TimeSpan timeSpan;
+        Target target;
+
+        timeSpan.AddTarget(target);
+        profile.AddTimeSpan(timeSpan);
+
+        string s = parser.ParseProfile(profile);
+        const char *pcszExpectedOutput = "<Profile>\n"
+            "  <Progress>0</Progress>\n"
+            "  <ResultFormat>text</ResultFormat>\n"
+            "  <Verbose>false</Verbose>\n"
+            "  <TimeSpans>\n"
+            "    <TimeSpan>\n"
+            "      <CompletionRoutines>false</CompletionRoutines>\n"
+            "      <MeasureLatency>false</MeasureLatency>\n"
+            "      <CalculateIopsStdDev>false</CalculateIopsStdDev>\n"
+            "      <DisableAffinity>false</DisableAffinity>\n"
+            "      <Duration>10</Duration>\n"
+            "      <Warmup>5</Warmup>\n"
+            "      <Cooldown>0</Cooldown>\n"
+            "      <ThreadCount>0</ThreadCount>\n"
+            "      <RequestCount>0</RequestCount>\n"
+            "      <IoBucketDuration>1000</IoBucketDuration>\n"
+            "      <RandSeed>0</RandSeed>\n"
+            "      <Targets>\n"
+            "        <Target>\n"
+            "          <Path></Path>\n"
+            "          <BlockSize>65536</BlockSize>\n"
+            "          <BaseFileOffset>0</BaseFileOffset>\n"
+            "          <SequentialScan>false</SequentialScan>\n"
+            "          <RandomAccess>false</RandomAccess>\n"
+            "          <TemporaryFile>false</TemporaryFile>\n"
+            "          <UseLargePages>false</UseLargePages>\n"
+            "          <WriteBufferContent>\n"
+            "            <Pattern>sequential</Pattern>\n"
+            "          </WriteBufferContent>\n"
+            "          <ParallelAsyncIO>false</ParallelAsyncIO>\n"
+            "          <StrideSize>65536</StrideSize>\n"
+            "          <InterlockedSequential>false</InterlockedSequential>\n"
+            "          <ThreadStride>0</ThreadStride>\n"
+            "          <MaxFileSize>0</MaxFileSize>\n"
+            "          <RequestCount>2</RequestCount>\n"
+            "          <WriteRatio>0</WriteRatio>\n"
+            "          <Throughput>0</Throughput>\n"
+            "          <ThreadsPerFile>1</ThreadsPerFile>\n"
+            "          <IOPriority>3</IOPriority>\n"
+            "          <Weight>1</Weight>\n"
+            "        </Target>\n"
+            "      </Targets>\n"
+            "    </TimeSpan>\n"
+            "  </TimeSpans>\n"
+            "</Profile>\n";
+
+        //VERIFY_ARE_EQUAL(pcszExpectedOutput, s.c_str());
+        VERIFY_ARE_EQUAL(strlen(pcszExpectedOutput), s.length());
+        VERIFY_IS_TRUE(!strcmp(pcszExpectedOutput, s.c_str()));
+    }
+
+    void XmlResultParserUnitTests::Test_ParseTargetProfile()
+    {
+        Target target;
+        string sResults;
+        char pszExpectedOutput[4096];
+        int nWritten;
+
+        const char *pcszOutputTemplate = \
+            "<Target>\n"
+            "  <Path>testfile1.dat</Path>\n"
+            "  <BlockSize>65536</BlockSize>\n"
+            "  <BaseFileOffset>0</BaseFileOffset>\n"
+            "  <SequentialScan>false</SequentialScan>\n"
+            "  <RandomAccess>false</RandomAccess>\n"
+            "  <TemporaryFile>false</TemporaryFile>\n"
+            "  <UseLargePages>false</UseLargePages>\n"
+            "  <DisableOSCache>true</DisableOSCache>\n"
+            "  <WriteThrough>true</WriteThrough>\n"
+            "  <WriteBufferContent>\n"
+            "    <Pattern>sequential</Pattern>\n"
+            "  </WriteBufferContent>\n"
+            "  <ParallelAsyncIO>false</ParallelAsyncIO>\n"
+            "  <StrideSize>65536</StrideSize>\n"
+            "  <InterlockedSequential>false</InterlockedSequential>\n"
+            "  <ThreadStride>0</ThreadStride>\n"
+            "  <MaxFileSize>0</MaxFileSize>\n"
+            "  <RequestCount>2</RequestCount>\n"
+            "  <WriteRatio>0</WriteRatio>\n"
+            "  <Throughput%s>%s</Throughput>\n"       // 2 param
+            "  <ThreadsPerFile>1</ThreadsPerFile>\n"
+            "  <IOPriority>3</IOPriority>\n"
+            "  <Weight>1</Weight>\n"
+            "</Target>\n";
+
+        target.SetPath("testfile1.dat");
+        target.SetCacheMode(TargetCacheMode::DisableOSCache);
+        target.SetWriteThroughMode(WriteThroughMode::On);
+
+        // Base case - no limit
+
+        nWritten = sprintf_s(pszExpectedOutput, sizeof(pszExpectedOutput),
+                             pcszOutputTemplate, "", "0");
+        VERIFY_IS_GREATER_THAN(nWritten, 0);
+        sResults = target.GetXml(0);
+        VERIFY_ARE_EQUAL(sResults, pszExpectedOutput);
+
+        // IOPS - with units
+
+        target.SetThroughputIOPS(1000);
+        nWritten = sprintf_s(pszExpectedOutput, sizeof(pszExpectedOutput),
+                             pcszOutputTemplate, " unit=\"IOPS\"", "1000");
+        VERIFY_IS_GREATER_THAN(nWritten, 0);
+        sResults = target.GetXml(0);
+        VERIFY_ARE_EQUAL(sResults, pszExpectedOutput);
+
+        // BPMS - not specified with units in output
+
+        target.SetThroughput(1000);
+        nWritten = sprintf_s(pszExpectedOutput, sizeof(pszExpectedOutput),
+                             pcszOutputTemplate, "", "1000");
+        VERIFY_IS_GREATER_THAN(nWritten, 0);
+        sResults = target.GetXml(0);
+        VERIFY_ARE_EQUAL(sResults, pszExpectedOutput);
+    }
+}

CristalDiskMark/source/diskspd22/UnitTests/XmlResultParser/XmlResultParser.UnitTests.h

@@ -0,0 +1,47 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+#include "WexTestClass.h"
+
+namespace UnitTests
+{
+    BEGIN_MODULE()
+        MODULE_PROPERTY(L"Feature", L"XmlResultParser")
+    END_MODULE()
+
+    class XmlResultParserUnitTests : public WEX::TestClass<XmlResultParserUnitTests>
+    {
+    public:
+        TEST_CLASS(XmlResultParserUnitTests);
+        TEST_METHOD(Test_ParseResults);
+        TEST_METHOD(Test_ParseProfile);
+        TEST_METHOD(Test_ParseTargetProfile);
+    };
+}

CristalDiskMark/source/diskspd22/UnitTests/XmlResultParser/XmlResultParser.rc

@@ -0,0 +1,17 @@
+#include <windows.h>
+#include "Version.h"
+
+#include <ntverp.h>
+
+#define VER_FILETYPE                VFT_APP
+#define VER_FILESUBTYPE             VFT2_UNKNOWN
+#define VER_FILEDESCRIPTION_STR     "DiskSpd Storage Performance Tool"
+#define VER_INTERNALNAME_STR        "XmlResultParser.UnitTests.dll"
+
+#undef VER_PRODUCTVERSION
+#define VER_PRODUCTVERSION DISKSPD_MAJOR,DISKSPD_MINOR,DISKSPD_BUILD,DISKSPD_QFE
+
+#undef VER_PRODUCTVERSION_STR
+#define VER_PRODUCTVERSION_STR      DISKSPD_NUMERIC_VERSION_STRING
+
+#include "common.ver"

CristalDiskMark/source/diskspd22/UnitTests/XmlResultParser/stdafx.h

@@ -0,0 +1,32 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#pragma once
+
+#include <windows.h>

CristalDiskMark/source/diskspd22/XmlProfileParser/diskspd.xsd

@@ -0,0 +1,400 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xs:schema xmlns:ds="http://microsoft.com/diskspd/DiskSpdConfig.xsd"
+    xmlns:xs="http://www.w3.org/2001/XMLSchema">
+  <xs:element name="Profile">
+    <xs:complexType>
+      <xs:all>
+        <xs:element name="TimeSpans" minOccurs="1" maxOccurs="1">
+          <xs:complexType>
+            <xs:sequence>
+              <xs:element name="TimeSpan" minOccurs="1" maxOccurs="unbounded">
+                <xs:complexType>
+                  <xs:all>
+                    <xs:element name="Targets" minOccurs="1" maxOccurs="1">
+                      <xs:complexType>
+                        <xs:sequence>
+                          <xs:element name="Target" minOccurs="1" maxOccurs="unbounded">
+                            <!-- file / disk -->
+                            <xs:complexType>
+                              <xs:all>
+
+                              <!-- Random Distributions
+
+                                Note use of typed elements to implement basic validation. Exactly one type of distribution
+                                is allowed. Range specification is cumulative per command line model and validated after load.
+
+                                -rdabs10/50000:10/20
+                                <Absolute>
+                                  <Range IO="10">50000</Range>
+                                  <Range IO="10">20</Range>
+                                </Absolute>
+
+                                -rdpct10/50:10/20
+                                <Percent>
+                                  <Range IO="10">50</Range>
+                                  <Range IO="10">20</Range>
+                                </Percent>
+
+                              -->
+
+                                <xs:element name="Distribution" minOccurs="0" maxOccurs="1">
+                                  <xs:complexType>
+                                    <xs:choice>
+                                      <xs:element name="Percent" minOccurs="1" maxOccurs="1">
+                                        <xs:complexType>
+                                          <xs:sequence>
+                                            <xs:element name="Range" minOccurs="1" maxOccurs="unbounded">
+                                              <xs:complexType>
+                                                <xs:simpleContent>
+                                                  <xs:extension base="PercentNZ">
+                                                    <xs:attribute type="Percent" name="IO" use="required"/>
+                                                  </xs:extension>
+                                                </xs:simpleContent>
+                                              </xs:complexType>
+                                            </xs:element>
+                                          </xs:sequence>
+                                        </xs:complexType>
+                                      </xs:element>
+                                      <xs:element name="Absolute" minOccurs="1" maxOccurs="1">
+                                        <xs:complexType>
+                                          <xs:sequence>
+                                            <xs:element name="Range" minOccurs="1" maxOccurs="unbounded">
+                                              <xs:complexType>
+                                                <xs:simpleContent>
+                                                  <xs:extension base="xs:unsignedLong">
+                                                    <xs:attribute type="Percent" name="IO" use="required"/>
+                                                  </xs:extension>
+                                                </xs:simpleContent>
+                                              </xs:complexType>
+                                            </xs:element>
+                                          </xs:sequence>
+                                        </xs:complexType>
+                                      </xs:element>
+                                    </xs:choice>
+                                  </xs:complexType>
+                                </xs:element>
+
+                                <!-- string _sPath -->
+                                <xs:element name="Path" type="xs:string" minOccurs="1" maxOccurs="1"/>
+
+                                <!-- DWORD dwBlockSize -->
+                                <xs:element name="BlockSize" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- UINT64 ullStrideSize -->
+                                <xs:element name="StrideSize" type="xs:unsignedLong" minOccurs="0" maxOccurs="1"/>
+
+                                <xs:element name="InterlockedSequential" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- UINT64 ullBaseFileOffset -->
+                                <xs:element name="BaseFileOffset" type="xs:unsignedLong" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- BOOL fSequentialScan (open file with the FILE_FLAG_SEQUENTIAL_SCAN flag) -->
+                                <xs:element name="SequentialScan" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- BOOL fRandomAccess (open file with the FILE_FLAG_RANDOM_ACCESS flag) -->
+                                <xs:element name="RandomAccess" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- BOOL fTemporaryFile (open file with the FILE_ATTRIBUTE_TEMPORARY flag) -->
+                                <xs:element name="TemporaryFile" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- BOOL fUseLargePages (Use large pages for IO buffers) -->
+                                <xs:element name="UseLargePages" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- DWORD dwRequestCount -->
+                                <!-- -o<count> number of overlapped I/O requests per file per thread
+                                             (1=synchronous I/O, unless more than 1 thread is specified with -F) [default=2]-->
+                                <xs:element name="RequestCount" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- UINT64 ullRandom
+                                   -r<align>[K|M|G|b] random I/O aligned to <align> bytes (doesn't make sense with -s)
+                                   <align> can be stated in bytes/KB/MB/GB/blocks [default access=sequential, default alignment=block size] -->
+                                <xs:element name="Random" type="xs:unsignedLong" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- BOOL DisableAllCache combined DisableOSCache & WriteThrough (compat) -->
+                                <xs:element name="DisableAllCache" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- BOOL DisableLocalCache -->
+                                <xs:element name="DisableLocalCache" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- BOOL DisableOSCache -->
+                                <xs:element name="DisableOSCache" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- BOOL WriteThrough -->
+                                <xs:element name="WriteThrough" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- BOOL MemoryMappedIo -->
+                                <xs:element name="MemoryMappedIo" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- enum FlushType, only has effect with memory mapped writes -->
+                                <xs:element name="FlushType" minOccurs="0" maxOccurs="1">
+                                  <xs:simpleType>
+                                    <xs:restriction base="xs:string">
+                                      <xs:enumeration value="ViewOfFile"/>
+                                      <xs:enumeration value="NonVolatileMemory"/>
+                                      <xs:enumeration value="NonVolatileMemoryNoDrain"/>
+                                    </xs:restriction>
+                                  </xs:simpleType>
+                                </xs:element>
+
+                                <xs:element name="WriteBufferContent" minOccurs="0" maxOccurs="1">
+                                  <xs:complexType>
+                                    <xs:all>
+
+                                      <xs:element name="Pattern" minOccurs="1" maxOccurs="1">
+                                        <xs:simpleType>
+                                          <xs:restriction base="xs:string">
+                                            <xs:enumeration value="sequential"/>
+                                            <xs:enumeration value="zero"/>
+                                            <xs:enumeration value="random"/>
+                                          </xs:restriction>
+                                        </xs:simpleType>
+                                      </xs:element>
+
+                                      <!-- used only with pattern == random -->
+                                      <xs:element name="RandomDataSource" minOccurs="0" maxOccurs="1">
+                                        <xs:complexType>
+                                          <xs:all>
+                                            <xs:element name="SizeInBytes" type="xs:unsignedLong" minOccurs="1" maxOccurs="1"/>
+                                            <xs:element name="FilePath" type="xs:string" minOccurs="0" maxOccurs="1"/>
+                                          </xs:all>
+                                        </xs:complexType>
+                                      </xs:element>
+
+                                    </xs:all>
+                                  </xs:complexType>
+                                </xs:element>
+
+                                <!-- DWORD dwBurstSize (number of IOs in a burst) -->
+                                <xs:element name="BurstSize" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- DWORD dwThinkTime (time to pause before issuing the next burst of IOs) -->
+                                <xs:element name="ThinkTime" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- DWORD dwThroughput (in bytes per millisecond); this can not be specified when using completion routines -->
+                                <xs:element name="Throughput" minOccurs="0" maxOccurs="1">
+                                  <xs:complexType>
+                                    <xs:simpleContent>
+                                      <xs:extension base="xs:unsignedInt">
+                                        <xs:attribute name="unit" default="BPMS">
+                                          <xs:simpleType>
+                                            <xs:restriction base="xs:string">
+                                              <xs:enumeration value="IOPS"/>
+                                              <xs:enumeration value="BPMS"/>
+                                            </xs:restriction>
+                                          </xs:simpleType>
+                                        </xs:attribute>
+                                      </xs:extension>
+                                    </xs:simpleContent>
+                                  </xs:complexType>
+                                </xs:element>
+
+                                <!-- DWORD dwThreadsPerFile -->
+                                <xs:element name="ThreadsPerFile" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- UINT64 ullFileSize -->
+                                <!-- used by the -c (create file) switch-->
+                                <xs:element name="FileSize" type="xs:unsignedLong" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- UINT64 ullMaxFileSize -->
+                                <xs:element name="MaxFileSize" type="xs:unsignedLong" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- UINT32 ulWriteRatio -->
+                                <xs:element name="WriteRatio" type="Percent" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- UINT32 ulRandomRatio -->
+                                <!-- Note: RandomRatio should only ever be between 1 and 99 - 0 is <StrideSize> in isolation, and 100 is <Random> in isolation -->
+                                <xs:element name="RandomRatio" type="PercentNZNM" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- BOOL fParallelAsyncIO
+                                   -p                 start async (overlapped) I/O operations with the same offset (makes sense only with -o2 or greater) -->
+                                <xs:element name="ParallelAsyncIO" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- UINT64 ullThreadStride -->
+                                <xs:element name="ThreadStride" type="xs:unsignedLong" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- 0 - not set, 1 - very low, 2 - low, 3 - normal -->
+                                <xs:element name="IOPriority" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- Weight to apply to this target on the specified threads - only valid when timespan
+                                   specifies ThreadCount and RequestCount, no cmd line equivalent.  The actual weight
+                                   given to this target will depend on the weights that other targets specify.  The
+                                   weight for this target is Weight / sum(Weight for all targets).  If the total weight
+                                   is 100 and the weight for this target is 30, then 30% of the IOs will be to this target. -->
+                                <xs:element name="Weight" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                                <!-- Threads to use - only valid when timespan specifies ThreadCount, no cmd line equivalent.
+                                   If a weight is specified it will override the overall weight for the target.  If no
+                                   ThreadTargets are specified then all threads will be used.  If any ThreadTarget is
+                                   specified then only the specified ThreadTargets will be used.  -->
+                                <xs:element name="ThreadTargets" minOccurs="0" maxOccurs="1">
+                                  <xs:complexType>
+                                    <xs:sequence>
+                                      <xs:element name="ThreadTarget" minOccurs="1" maxOccurs="unbounded">
+                                        <xs:complexType>
+                                          <xs:all>
+                                            <xs:element name="Thread" type="xs:unsignedInt" minOccurs="1" maxOccurs="1"/>
+                                            <xs:element name="Weight" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+                                          </xs:all>
+                                        </xs:complexType>
+                                      </xs:element>
+                                    </xs:sequence>
+                                  </xs:complexType>
+                                </xs:element>
+
+                              </xs:all>
+                            </xs:complexType>
+                          </xs:element>
+                        </xs:sequence>
+
+                      </xs:complexType>
+                    </xs:element>
+
+                    <!-- UINT32 ulDuration -->
+                    <xs:element name="Duration" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                    <!-- UINT32 ulWarmUp -->
+                    <xs:element name="Warmup" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                    <!-- UINT32 ulCoolDown -->
+                    <xs:element name="Cooldown" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                    <!-- UINT32 ulRandSeed
+                       -z                 set random seed [default=0 if parameter not provided, GetTickCount() if value not provided] -->
+                    <xs:element name="RandSeed" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                    <!-- BOOL fRandomWriteData -->
+                    <xs:element name="RandomWriteData" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                    <!-- DWORD dwThreadCount
+                       -F<count>          total number of threads (cannot be used with -t) this cannot be used if per-file thread count is provided -->
+                    <xs:element name="ThreadCount" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                    <!-- DWORD dwRequestCount
+                       -O<count>          total number of requests per thread - for use with -F -->
+                    <xs:element name="RequestCount" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+                    <!-- BOOL fGroupAffinity -->
+                    <xs:element name="GroupAffinity" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                    <!-- UINT32 *puAffinity -->
+                    <xs:element name="Affinity" minOccurs="0" maxOccurs="1">
+                      <xs:complexType>
+                        <xs:sequence>
+                          <xs:choice>
+                            <xs:element name="AffinityAssignment" type="xs:unsignedInt" minOccurs="1" maxOccurs="unbounded"/>
+                            <xs:element name="AffinityGroupAssignment" type="AffinityGroupAssignment" minOccurs="1" maxOccurs="unbounded"/>
+                          </xs:choice>
+                        </xs:sequence>
+                      </xs:complexType>
+                    </xs:element>
+
+                    <!-- BOOL fDisableAffinity
+                       -n                 disable affinity (cannot be used with -a) -->
+                    <xs:element name="DisableAffinity" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                    <!-- BOOL fCompletionRoutines -->
+                    <!-- TODO: this should be decided on a target level -->
+                    <xs:element name="CompletionRoutines" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                    <xs:element name="MeasureLatency" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+                    <xs:element name="CalculateIopsStdDev" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+                    <xs:element name="IoBucketDuration" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+                  </xs:all>
+                </xs:complexType>
+              </xs:element>
+            </xs:sequence>
+          </xs:complexType>
+        </xs:element>
+
+        <!-- BOOL bVerbose -->
+        <xs:element name="Verbose" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+        <xs:element name="VerboseStats" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+
+        <!-- DWORD dwProgress -->
+        <xs:element name="Progress" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+        <!-- Experimental behavior flags -->
+        <xs:element name="ExperimentFlags" type="xs:unsignedInt" minOccurs="0" maxOccurs="1"/>
+
+        <xs:element name="ResultFormat" minOccurs="0" maxOccurs="1">
+          <xs:simpleType>
+            <xs:restriction base="xs:string">
+              <xs:enumeration value="text"/>
+              <xs:enumeration value="xml"/>
+            </xs:restriction>
+          </xs:simpleType>
+        </xs:element>
+
+        <!-- ETWMask -->
+        <xs:element name="ETW" minOccurs="0" maxOccurs="1">
+          <xs:complexType>
+            <xs:all>
+              <!-- BOOL bProcess -->
+              <xs:element name="Process" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+              <!-- BOOL bThread -->
+              <xs:element name="Thread" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+              <!-- BOOL bImageLoad -->
+              <xs:element name="ImageLoad" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+              <!-- BOOL bDiskIO -->
+              <xs:element name="DiskIO" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+              <!-- BOOL bMemoryPageFaults -->
+              <xs:element name="MemoryPageFaults" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+              <!-- BOOL bMemoryHardFaults -->
+              <xs:element name="MemoryHardFaults" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+              <!-- BOOL bNetwork -->
+              <xs:element name="Network" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+              <!-- BOOL bRegistry -->
+              <xs:element name="Registry" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+              <!-- BOOL bUsePagedMemory -->
+              <xs:element name="UsePagedMemory" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+              <!-- BOOL bUsePerfTimer -->
+              <xs:element name="UsePerfTimer" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+              <!-- BOOL bUseSystemTimer -->
+              <xs:element name="UseSystemTimer" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+              <!-- BOOL bUseCyclesCounter -->
+              <xs:element name="UseCyclesCounter" type="xs:boolean" minOccurs="0" maxOccurs="1"/>
+            </xs:all>
+          </xs:complexType>
+        </xs:element>
+
+        <!-- files should be created before the first time span -->
+        <xs:element name="PrecreateFiles" minOccurs="0" maxOccurs="1">
+          <xs:simpleType>
+            <xs:restriction base="xs:string">
+              <!-- if the same path exists in multiple time spans, the max size will be used -->
+              <xs:enumeration value="UseMaxSize"/>
+              <xs:enumeration value="CreateOnlyFilesWithConstantSizes"/>
+              <xs:enumeration value="CreateOnlyFilesWithConstantOrZeroSizes"/>
+            </xs:restriction>
+          </xs:simpleType>
+        </xs:element>
+
+      </xs:all>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:complexType name="AffinityGroupAssignment">
+    <xs:attribute name="Group" type="xs:unsignedInt"/>
+    <xs:attribute name="Processor" type="xs:unsignedInt"/>
+  </xs:complexType>
+
+  <xs:simpleType name="Percent">
+    <xs:restriction base="xs:unsignedInt">
+      <xs:minInclusive value="0"/>
+      <xs:maxInclusive value="100"/>
+    </xs:restriction>
+  </xs:simpleType>
+  <xs:simpleType name="PercentNZ">
+    <xs:restriction base="xs:unsignedInt">
+      <xs:minInclusive value="1"/>
+      <xs:maxInclusive value="100"/>
+    </xs:restriction>
+  </xs:simpleType>
+  <xs:simpleType name="PercentNZNM">
+    <xs:restriction base="xs:unsignedInt">
+      <xs:minInclusive value="1"/>
+      <xs:maxInclusive value="99"/>
+    </xs:restriction>
+  </xs:simpleType>
+</xs:schema>

CristalDiskMark/source/diskspd22/XmlResultParser/XmlResultParser.cpp

@@ -0,0 +1,620 @@
+/*
+
+DISKSPD
+
+Copyright(c) Microsoft Corporation
+All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#include "xmlresultparser.h"
+
+// TODO: refactor to a single function shared with the ResultParser
+char printBuffer[4096] = {};
+
+/// for CrystalDiskMark
+int XmlResultParser::GetTotalScore()
+{
+    return 0;
+}
+
+double XmlResultParser::GetAverageLatency()
+{
+    return 0.0;
+}
+
+void XmlResultParser::_PrintV(const char *format, va_list listArg)
+{
+    _sResult.append(_indent, ' ');
+    vsprintf_s(printBuffer, _countof(printBuffer), format, listArg);
+    _sResult += printBuffer;
+}
+
+void XmlResultParser::_Print(const char *format, ...)
+{
+    assert(nullptr != format);
+    va_list listArg;
+    va_start(listArg, format);
+
+    _PrintV(format, listArg);
+    va_end(listArg);
+}
+
+void XmlResultParser::_PrintInc(const char *format, ...)
+{
+    assert(nullptr != format);
+    va_list listArg;
+    va_start(listArg, format);
+
+    // Print & Increment Indent
+    // e.g., <StartOfSection>
+
+    _PrintV(format, listArg);
+    _indent += 2;
+    va_end(listArg);
+}
+
+void XmlResultParser::_PrintDec(const char *format, ...)
+{
+    assert(nullptr != format);
+    va_list listArg;
+    va_start(listArg, format);
+
+    // Decrement Indent & Print
+    // e.g., </EndOfSection>
+
+    _indent -= 2;
+    _PrintV(format, listArg);
+    va_end(listArg);
+}
+
+void XmlResultParser::_PrintTargetResults(const TargetResults& results)
+{
+    // TODO: results.readBucketizer;
+    // TODO: results.writeBucketizer;
+
+    _Print("<Path>%s</Path>\n", results.sPath.c_str());
+    _Print("<BytesCount>%I64u</BytesCount>\n", results.ullBytesCount);
+    _Print("<FileSize>%I64u</FileSize>\n", results.ullFileSize);
+    _Print("<IOCount>%I64u</IOCount>\n", results.ullIOCount);
+    _Print("<ReadBytes>%I64u</ReadBytes>\n", results.ullReadBytesCount);
+    _Print("<ReadCount>%I64u</ReadCount>\n", results.ullReadIOCount);
+    _Print("<WriteBytes>%I64u</WriteBytes>\n", results.ullWriteBytesCount);
+    _Print("<WriteCount>%I64u</WriteCount>\n", results.ullWriteIOCount);
+
+    if (results.vDistributionRange.size())
+    {
+        _PrintInc("<Distribution>\n");
+        _PrintInc("<Absolute>\n");
+
+        //
+        // Render hole(s) in effective distribution. Keep track of the expected base
+        // of the next range and render a hole (IO = 0) over the gap as needed.
+        //
+
+        UINT64 expectBase = 0;
+        for (auto& r : results.vDistributionRange)
+        {
+            if (r._dst.first != expectBase)
+            {
+                _Print("<Range IO=\"%u\">%I64u</Range>\n", 0, r._dst.first - expectBase);
+            }
+
+            _Print("<Range IO=\"%u\">%I64u</Range>\n", r._span, r._dst.second);
+            expectBase = r._dst.first + r._dst.second;
+        }
+
+        _PrintDec("</Absolute>\n");
+        _PrintDec("</Distribution>\n");
+    }
+}
+
+void XmlResultParser::_PrintTargetLatency(const TargetResults& results)
+{
+    if (results.readLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<AverageReadLatencyMilliseconds>%.3f</AverageReadLatencyMilliseconds>\n", results.readLatencyHistogram.GetAvg() / 1000);
+        _Print("<ReadLatencyStdev>%.3f</ReadLatencyStdev>\n", results.readLatencyHistogram.GetStandardDeviation() / 1000);
+    }
+    if (results.writeLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<AverageWriteLatencyMilliseconds>%.3f</AverageWriteLatencyMilliseconds>\n", results.writeLatencyHistogram.GetAvg() / 1000);
+        _Print("<WriteLatencyStdev>%.3f</WriteLatencyStdev>\n", results.writeLatencyHistogram.GetStandardDeviation() / 1000);
+    }
+    Histogram<float> totalLatencyHistogram;
+    totalLatencyHistogram.Merge(results.readLatencyHistogram);
+    totalLatencyHistogram.Merge(results.writeLatencyHistogram);
+    if (totalLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<AverageLatencyMilliseconds>%.3f</AverageLatencyMilliseconds>\n", totalLatencyHistogram.GetAvg() / 1000);
+        _Print("<LatencyStdev>%.3f</LatencyStdev>\n", totalLatencyHistogram.GetStandardDeviation() / 1000);
+    }
+}
+
+void XmlResultParser::_PrintTargetIops(const IoBucketizer& readBucketizer, const IoBucketizer& writeBucketizer, UINT32 bucketTimeInMs)
+{
+    _PrintInc("<Iops>\n");
+
+    IoBucketizer totalIoBucketizer;
+    totalIoBucketizer.Merge(readBucketizer);
+    totalIoBucketizer.Merge(writeBucketizer);
+
+    if (readBucketizer.GetNumberOfValidBuckets() > 0)
+    {
+        _Print("<ReadIopsStdDev>%.3f</ReadIopsStdDev>\n", readBucketizer.GetStandardDeviationIOPS() / (bucketTimeInMs / 1000.0));
+    }
+    if (writeBucketizer.GetNumberOfValidBuckets() > 0)
+    {
+        _Print("<WriteIopsStdDev>%.3f</WriteIopsStdDev>\n", writeBucketizer.GetStandardDeviationIOPS() / (bucketTimeInMs / 1000.0));
+    }
+    if (totalIoBucketizer.GetNumberOfValidBuckets() > 0)
+    {
+        _Print("<IopsStdDev>%.3f</IopsStdDev>\n", totalIoBucketizer.GetStandardDeviationIOPS() / (bucketTimeInMs / 1000.0));
+    }
+    _PrintIops(readBucketizer, writeBucketizer, bucketTimeInMs);
+    _PrintDec("</Iops>\n");
+}
+
+void XmlResultParser::_PrintETWSessionInfo(struct ETWSessionInfo sessionInfo)
+{
+    _PrintInc("<ETWSessionInfo>\n");
+    _Print("<BufferSizeKB>%lu</BufferSizeKB>\n", sessionInfo.ulBufferSize);
+    _Print("<MinimimBuffers>%lu</MinimimBuffers>\n", sessionInfo.ulMinimumBuffers);
+    _Print("<MaximumBuffers>%lu</MaximumBuffers>\n", sessionInfo.ulMaximumBuffers);
+    _Print("<FreeBuffers>%lu</FreeBuffers>", sessionInfo.ulFreeBuffers);
+    _Print("<BuffersWritten>%lu</BuffersWritten>\n", sessionInfo.ulBuffersWritten);
+    _Print("<FlushTimerSeconds>%lu</FlushTimerSeconds>\n", sessionInfo.ulFlushTimer);
+    _Print("<AgeLimitMinutes>%d</AgeLimitMinutes>\n", sessionInfo.lAgeLimit);
+
+    _Print("<AllocatedBuffers>%lu</AllocatedBuffers>\n", sessionInfo.ulNumberOfBuffers);
+    _Print("<LostEvents>%lu</LostEvents>\n", sessionInfo.ulEventsLost);
+    _Print("<LostLogBuffers>%lu</LostLogBuffers>\n", sessionInfo.ulLogBuffersLost);
+    _Print("<LostRealTimeBuffers>%lu</LostRealTimeBuffers>\n", sessionInfo.ulRealTimeBuffersLost);
+    _PrintDec("</ETWSessionInfo>\n");
+}
+
+void XmlResultParser::_PrintETW(struct ETWMask ETWMask, struct ETWEventCounters EtwEventCounters)
+{
+    _PrintInc("<ETW>\n");
+    if (ETWMask.bDiskIO)
+    {
+        _PrintInc("<DiskIO>\n");
+        _Print("<Read>%I64u</Read>\n", EtwEventCounters.ullIORead);
+        _Print("<Write>%I64u</Write>\n", EtwEventCounters.ullIOWrite);
+        _PrintDec("</DiskIO>\n");
+    }
+    if (ETWMask.bImageLoad)
+    {
+        _Print("<LoadImage>%I64u</LoadImage>\n", EtwEventCounters.ullImageLoad);
+    }
+    if (ETWMask.bMemoryPageFaults)
+    {
+        _PrintInc("<MemoryPageFaults>\n");
+        _Print("<CopyOnWrite>%I64u</CopyOnWrite>\n", EtwEventCounters.ullMMCopyOnWrite);
+        _Print("<DemandZeroFault>%I64u</DemandZeroFault>\n", EtwEventCounters.ullMMDemandZeroFault);
+        _Print("<GuardPageFault>%I64u</GuardPageFault>\n", EtwEventCounters.ullMMGuardPageFault);
+        _Print("<HardPageFault>%I64u</HardPageFault>\n", EtwEventCounters.ullMMHardPageFault);
+        _Print("<TransitionFault>%I64u</TransitionFault>\n", EtwEventCounters.ullMMTransitionFault);
+        _PrintDec("</MemoryPageFaults>\n");
+    }
+    if (ETWMask.bMemoryHardFaults && !ETWMask.bMemoryPageFaults)
+    {
+        _Print("<HardPageFault>%I64u</HardPageFault>\n", EtwEventCounters.ullMMHardPageFault);
+    }
+    if (ETWMask.bNetwork)
+    {
+        _PrintInc("<Network>\n");
+        _Print("<Accept>%I64u</Accept>\n", EtwEventCounters.ullNetAccept);
+        _Print("<Connect>%I64u</Connect>\n", EtwEventCounters.ullNetConnect);
+        _Print("<Disconnect>%I64u</Disconnect>\n", EtwEventCounters.ullNetDisconnect);
+        _Print("<Reconnect>%I64u</Reconnect>\n", EtwEventCounters.ullNetReconnect);
+        _Print("<Retransmit>%I64u</Retransmit>\n", EtwEventCounters.ullNetRetransmit);
+        _Print("<TCPIPSend>%I64u</TCPIPSend>\n", EtwEventCounters.ullNetTcpSend);
+        _Print("<TCPIPReceive>%I64u</TCPIPReceive>\n", EtwEventCounters.ullNetTcpReceive);
+        _Print("<UDPIPSend>%I64u</UDPIPSend>\n", EtwEventCounters.ullNetUdpSend);
+        _Print("<UDPIPReceive>%I64u</UDPIPReceive>\n", EtwEventCounters.ullNetUdpReceive);
+        _PrintDec("</Network>\n");
+    }
+    if (ETWMask.bProcess)
+    {
+        _PrintInc("<Process>\n");
+        _Print("<Start>%I64u</Start>\n", EtwEventCounters.ullProcessStart);
+        _Print("<End>%I64u</End>\n", EtwEventCounters.ullProcessEnd);
+        _PrintDec("</Process>\n");
+    }
+    if (ETWMask.bRegistry)
+    {
+        _PrintInc("<Registry>\n");
+        _Print("<NtCreateKey>%I64u</NtCreateKey>\n", EtwEventCounters.ullRegCreate);
+        _Print("<NtDeleteKey>%I64u</NtDeleteKey>\n", EtwEventCounters.ullRegDelete);
+        _Print("<NtDeleteValueKey>%I64u</NtDeleteValueKey>\n", EtwEventCounters.ullRegDeleteValue);
+        _Print("<NtEnumerateKey>%I64u</NtEnumerateKey>\n", EtwEventCounters.ullRegEnumerateKey);
+        _Print("<NtEnumerateValueKey>%I64u</NtEnumerateValueKey>\n", EtwEventCounters.ullRegEnumerateValueKey);
+        _Print("<NtFlushKey>%I64u</NtFlushKey>\n", EtwEventCounters.ullRegFlush);
+        _Print("<NtOpenKey>%I64u</NtOpenKey>\n", EtwEventCounters.ullRegOpen);
+        _Print("<NtQueryKey>%I64u</NtQueryKey>\n", EtwEventCounters.ullRegQuery);
+        _Print("<NtQueryMultipleValueKey>%I64u</NtQueryMultipleValueKey>\n", EtwEventCounters.ullRegQueryMultipleValue);
+        _Print("<NtQueryValueKey>%I64u</NtQueryValueKey>\n", EtwEventCounters.ullRegQueryValue);
+        _Print("<NtSetInformationKey>%I64u</NtSetInformationKey>\n", EtwEventCounters.ullRegSetInformation);
+        _Print("<NtSetValueKey>%I64u</NtSetValueKey>\n", EtwEventCounters.ullRegSetValue);
+        _PrintDec("</Registry>\n");
+    }
+    if (ETWMask.bThread)
+    {
+        _PrintInc("<Thread>\n");
+        _Print("<Start>%I64u</Start>\n", EtwEventCounters.ullThreadStart);
+        _Print("<End>%I64u</End>\n", EtwEventCounters.ullThreadEnd);
+        _PrintDec("</Thread>\n");
+    }
+    _PrintDec("</ETW>\n");
+}
+
+void XmlResultParser::_PrintCpuUtilization(const Results& results, const SystemInformation& system)
+{
+    const auto& topo = system.processorTopology;
+    size_t procCount = results.vSystemProcessorPerfInfo.size();
+    size_t baseProc = 0;
+    BYTE efficiencyClass = 0;
+    BYTE processorCore = 0;
+
+    _PrintInc("<CpuUtilization>\n");
+
+    double busyTime = 0;
+    double totalIdleTime = 0;
+    double totalUserTime = 0;
+    double totalKrnlTime = 0;
+
+    for (const auto& group : topo._vProcessorGroupInformation) {
+
+        // Sanity assert - results are sized to the sum of active processors
+        assert(baseProc + group._activeProcessorCount <= procCount);
+
+        for (BYTE processor = 0; processor < group._activeProcessorCount; processor++) {
+
+            long long fTime = results.vSystemProcessorPerfInfo[baseProc + processor].KernelTime.QuadPart +
+                              results.vSystemProcessorPerfInfo[baseProc + processor].UserTime.QuadPart;
+
+            double idleTime = 100.0 * results.vSystemProcessorPerfInfo[baseProc + processor].IdleTime.QuadPart / fTime;
+            double krnlTime = 100.0 * results.vSystemProcessorPerfInfo[baseProc + processor].KernelTime.QuadPart / fTime;
+            double userTime = 100.0 * results.vSystemProcessorPerfInfo[baseProc + processor].UserTime.QuadPart / fTime;
+            double usedTime = (krnlTime - idleTime) + userTime;
+
+            _PrintInc("<CPU>\n");
+            _Print("<Socket>%d</Socket>\n", topo.GetSocketOfProcessor(group._groupNumber, processor));
+            _Print("<Node>%d</Node>\n", topo.GetNumaOfProcessor(group._groupNumber, processor));
+            _Print("<Group>%d</Group>\n", group._groupNumber);
+            processorCore = topo.GetCoreOfProcessor(group._groupNumber, processor, efficiencyClass);
+            _Print("<Core>%d</Core>\n", processorCore);
+            _Print("<EfficiencyClass>%d</EfficiencyClass>\n", efficiencyClass);
+            _Print("<Id>%d</Id>\n", processor);
+            _Print("<UsagePercent>%.2f</UsagePercent>\n", usedTime);
+            _Print("<UserPercent>%.2f</UserPercent>\n", userTime);
+            _Print("<KernelPercent>%.2f</KernelPercent>\n", krnlTime - idleTime);
+            _Print("<IdlePercent>%.2f</IdlePercent>\n", idleTime);
+            _PrintDec("</CPU>\n");
+
+            busyTime += usedTime;
+            totalIdleTime += idleTime;
+            totalUserTime += userTime;
+            totalKrnlTime += krnlTime;
+        }
+
+        baseProc += group._activeProcessorCount;
+    }
+
+    assert(baseProc == procCount);
+
+    _PrintInc("<Average>\n");
+    _Print("<UsagePercent>%.2f</UsagePercent>\n", busyTime / procCount);
+    _Print("<UserPercent>%.2f</UserPercent>\n", totalUserTime / procCount);
+    _Print("<KernelPercent>%.2f</KernelPercent>\n", (totalKrnlTime - totalIdleTime) / procCount);
+    _Print("<IdlePercent>%.2f</IdlePercent>\n", totalIdleTime / procCount);
+    _PrintDec("</Average>\n");
+
+    _PrintDec("</CpuUtilization>\n");
+}
+
+// emit the iops time series (this obviates needing perfmon counters, in common cases, and provides file level data)
+void XmlResultParser::_PrintIops(const IoBucketizer& readBucketizer, const IoBucketizer& writeBucketizer, UINT32 bucketTimeInMs)
+{
+    bool done = false;
+    for (size_t i = 0; !done; i++)
+    {
+        done = true;
+
+        double r = 0.0;
+        double r_min = 0.0;
+        double r_max = 0.0;
+        double r_avg = 0.0;
+        double r_stddev = 0.0;
+
+        double w = 0.0;
+        double w_min = 0.0;
+        double w_max = 0.0;
+        double w_avg = 0.0;
+        double w_stddev = 0.0;
+
+        if (readBucketizer.GetNumberOfValidBuckets() > i)
+        {
+            r = readBucketizer.GetIoBucketCount(i) / (bucketTimeInMs / 1000.0);
+            r_min = readBucketizer.GetIoBucketMinDurationUsec(i) / 1000.0;
+            r_max = readBucketizer.GetIoBucketMaxDurationUsec(i) / 1000.0;
+            r_avg = readBucketizer.GetIoBucketAvgDurationUsec(i) / 1000.0;
+            r_stddev = readBucketizer.GetIoBucketDurationStdDevUsec(i) / 1000.0;
+            done = false;
+        }
+        if (writeBucketizer.GetNumberOfValidBuckets() > i)
+        {
+            w = writeBucketizer.GetIoBucketCount(i) / (bucketTimeInMs / 1000.0);
+            w_min = writeBucketizer.GetIoBucketMinDurationUsec(i) / 1000.0;
+            w_max = writeBucketizer.GetIoBucketMaxDurationUsec(i) / 1000.0;
+            w_avg = writeBucketizer.GetIoBucketAvgDurationUsec(i) / 1000.0;
+            w_stddev = writeBucketizer.GetIoBucketDurationStdDevUsec(i) / 1000.0;
+            done = false;
+        }
+        if (!done)
+        {
+            _Print("<Bucket SampleMillisecond=\"%lu\" Read=\"%.0f\" Write=\"%.0f\" Total=\"%.0f\" "
+                   "ReadMinLatencyMilliseconds=\"%.3f\" ReadMaxLatencyMilliseconds=\"%.3f\" ReadAvgLatencyMilliseconds=\"%.3f\" ReadLatencyStdDev=\"%.3f\" "
+                   "WriteMinLatencyMilliseconds=\"%.3f\" WriteMaxLatencyMilliseconds=\"%.3f\" WriteAvgLatencyMilliseconds=\"%.3f\" WriteLatencyStdDev=\"%.3f\"/>\n",
+                   bucketTimeInMs*(i + 1), r, w, r + w,
+                   r_min, r_max, r_avg, r_stddev,
+                   w_min, w_max, w_avg, w_stddev);
+        }
+    }
+}
+
+void XmlResultParser::_PrintOverallIops(const Results& results, UINT32 bucketTimeInMs)
+{
+    IoBucketizer readBucketizer;
+    IoBucketizer writeBucketizer;
+
+    for (const auto& thread : results.vThreadResults)
+    {
+        for (const auto& target : thread.vTargetResults)
+        {
+            readBucketizer.Merge(target.readBucketizer);
+            writeBucketizer.Merge(target.writeBucketizer);
+        }
+    }
+
+    _PrintTargetIops(readBucketizer, writeBucketizer, bucketTimeInMs);
+}
+
+void XmlResultParser::_PrintLatencyPercentiles(const Results& results)
+{
+    Histogram<float> readLatencyHistogram;
+    Histogram<float> writeLatencyHistogram;
+    Histogram<float> totalLatencyHistogram;
+
+    for (const auto& thread : results.vThreadResults)
+    {
+        for (const auto& target : thread.vTargetResults)
+        {
+            readLatencyHistogram.Merge(target.readLatencyHistogram);
+
+            writeLatencyHistogram.Merge(target.writeLatencyHistogram);
+
+            totalLatencyHistogram.Merge(target.writeLatencyHistogram);
+            totalLatencyHistogram.Merge(target.readLatencyHistogram);
+        }
+    }
+
+    _PrintInc("<Latency ReadHistoBuckets=\"%lu\" WriteHistoBuckets=\"%lu\" TotalHistoBuckets=\"%lu\">\n",
+                readLatencyHistogram.GetSampleBuckets(),
+                writeLatencyHistogram.GetSampleBuckets(),
+                totalLatencyHistogram.GetSampleBuckets());
+
+    if (readLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<AverageReadMilliseconds>%.3f</AverageReadMilliseconds>\n", readLatencyHistogram.GetAvg() / 1000);
+        _Print("<ReadLatencyStdev>%.3f</ReadLatencyStdev>\n", readLatencyHistogram.GetStandardDeviation() / 1000);
+    }
+    if (writeLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<AverageWriteMilliseconds>%.3f</AverageWriteMilliseconds>\n", writeLatencyHistogram.GetAvg() / 1000);
+        _Print("<WriteLatencyStdev>%.3f</WriteLatencyStdev>\n", writeLatencyHistogram.GetStandardDeviation() / 1000);
+    }
+    if (totalLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<AverageTotalMilliseconds>%.3f</AverageTotalMilliseconds>\n", totalLatencyHistogram.GetAvg() / 1000);
+        _Print("<LatencyStdev>%.3f</LatencyStdev>\n", totalLatencyHistogram.GetStandardDeviation() / 1000);
+    }
+
+    _PrintInc("<Bucket>\n");
+    _Print("<Percentile>0</Percentile>\n");
+    if (readLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<ReadMilliseconds>%.3f</ReadMilliseconds>\n", readLatencyHistogram.GetMin() / 1000);
+    }
+    if (writeLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<WriteMilliseconds>%.3f</WriteMilliseconds>\n", writeLatencyHistogram.GetMin() / 1000);
+    }
+    if (totalLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<TotalMilliseconds>%.3f</TotalMilliseconds>\n", totalLatencyHistogram.GetMin() / 1000);
+    }
+    _PrintDec("</Bucket>\n");
+
+    //  Construct vector of percentiles and decimal precision to squelch trailing zeroes.  This is more
+    //  detailed than summary text output, and does not contain the decorated names (15th, etc.)
+
+    vector<pair<int, double>> vPercentiles;
+    for (int p = 1; p <= 99; p++)
+    {
+        vPercentiles.push_back(make_pair(0, p));
+    }
+
+    vPercentiles.push_back(make_pair(1, 99.9));
+    vPercentiles.push_back(make_pair(2, 99.99));
+    vPercentiles.push_back(make_pair(3, 99.999));
+    vPercentiles.push_back(make_pair(4, 99.9999));
+    vPercentiles.push_back(make_pair(5, 99.99999));
+    vPercentiles.push_back(make_pair(6, 99.999999));
+    vPercentiles.push_back(make_pair(7, 99.9999999));
+
+    for (auto p : vPercentiles)
+    {
+        _PrintInc("<Bucket>\n");
+        _Print("<Percentile>%.*f</Percentile>\n", p.first, p.second);
+        if (readLatencyHistogram.GetSampleSize() > 0)
+        {
+            _Print("<ReadMilliseconds>%.3f</ReadMilliseconds>\n", readLatencyHistogram.GetPercentile(p.second / 100) / 1000);
+        }
+        if (writeLatencyHistogram.GetSampleSize() > 0)
+        {
+            _Print("<WriteMilliseconds>%.3f</WriteMilliseconds>\n", writeLatencyHistogram.GetPercentile(p.second / 100) / 1000);
+        }
+        if (totalLatencyHistogram.GetSampleSize() > 0)
+        {
+            _Print("<TotalMilliseconds>%.3f</TotalMilliseconds>\n", totalLatencyHistogram.GetPercentile(p.second / 100) / 1000);
+        }
+        _PrintDec("</Bucket>\n");
+    }
+
+    _PrintInc("<Bucket>\n");
+    _Print("<Percentile>100</Percentile>\n");
+    if (readLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<ReadMilliseconds>%.3f</ReadMilliseconds>\n", readLatencyHistogram.GetMax() / 1000);
+    }
+    if (writeLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<WriteMilliseconds>%.3f</WriteMilliseconds>\n", writeLatencyHistogram.GetMax() / 1000);
+    }
+    if (totalLatencyHistogram.GetSampleSize() > 0)
+    {
+        _Print("<TotalMilliseconds>%.3f</TotalMilliseconds>\n", totalLatencyHistogram.GetMax() / 1000);
+    }
+    _PrintDec("</Bucket>\n");
+    _PrintDec("</Latency>\n");
+}
+
+string XmlResultParser::ParseProfile(const Profile& profile)
+{
+    _sResult = profile.GetXml(0);
+    return _sResult;
+}
+
+void XmlResultParser::_PrintWaitStats(const ThreadResults &threadResult)
+{
+    _PrintInc("<WaitStatistics>\n");
+    _Print("<CompletionWait>%llu</CompletionWait>\n", threadResult.WaitStats.Wait);
+    _Print("<ThrottleWait>%llu</ThrottleWait>\n", threadResult.WaitStats.ThrottleWait);
+    _Print("<ThrottleSleep>%llu</ThrottleSleep>\n", threadResult.WaitStats.ThrottleSleep);
+    _Print("<Lookaside>%llu</Lookaside>\n", threadResult.WaitStats.Lookaside);
+    _Print("<LookasideCompletion>%llu %llu %llu %llu %llu %llu %llu %llu</LookasideCompletion>\n",
+        threadResult.WaitStats.LookasideCompletion[0],
+        threadResult.WaitStats.LookasideCompletion[1],
+        threadResult.WaitStats.LookasideCompletion[2],
+        threadResult.WaitStats.LookasideCompletion[3],
+        threadResult.WaitStats.LookasideCompletion[4],
+        threadResult.WaitStats.LookasideCompletion[5],
+        threadResult.WaitStats.LookasideCompletion[6],
+        threadResult.WaitStats.LookasideCompletion[7]);
+    _PrintDec("</WaitStatistics>\n");
+}
+
+string XmlResultParser::ParseResults(const Profile& profile, const SystemInformation& system, vector<Results> vResults)
+{
+    _sResult.clear();
+
+    _PrintInc("<Results>\n");
+
+    _sResult += system.GetXml(_indent);
+    _sResult += profile.GetXml(_indent);
+    for (size_t iResults = 0; iResults < vResults.size(); iResults++)
+    {
+        const Results& results = vResults[iResults];
+        const TimeSpan& timeSpan = profile.GetTimeSpans()[iResults];
+
+        _PrintInc("<TimeSpan>\n");
+
+        double fTime = PerfTimer::PerfTimeToSeconds(results.ullTimeCount); //test duration
+        if (fTime >= 0.0000001)
+        {
+            // There either is a fixed number of threads for all files to share (GetThreadCount() > 0) or a number of threads per file.
+            // In the latter case vThreadResults.size() == number of threads per file * file count
+            size_t ulThreadCnt = (timeSpan.GetThreadCount() > 0) ? timeSpan.GetThreadCount() : results.vThreadResults.size();
+
+            _Print("<TestTimeSeconds>%.2f</TestTimeSeconds>\n", fTime);
+            _Print("<ThreadCount>%u</ThreadCount>\n", ulThreadCnt);
+            _Print("<RequestCount>%u</RequestCount>\n", timeSpan.GetRequestCount());
+            _Print("<ProcCount>%u</ProcCount>\n", system.processorTopology._ulProcessorCount);
+
+            _PrintCpuUtilization(results, system);
+
+            if (timeSpan.GetMeasureLatency())
+            {
+                _PrintLatencyPercentiles(results);
+            }
+
+            if (timeSpan.GetCalculateIopsStdDev())
+            {
+                _PrintOverallIops(results, timeSpan.GetIoBucketDurationInMilliseconds());
+            }
+
+            if (results.fUseETW)
+            {
+                _PrintETW(results.EtwMask, results.EtwEventCounters);
+                _PrintETWSessionInfo(results.EtwSessionInfo);
+            }
+
+            for (size_t iThread = 0; iThread < results.vThreadResults.size(); iThread++)
+            {
+                const ThreadResults& threadResults = results.vThreadResults[iThread];
+                _PrintInc("<Thread>\n");
+                _Print("<Id>%u</Id>\n", iThread);
+                for (const auto& targetResults : threadResults.vTargetResults)
+                {
+                    _PrintInc("<Target>\n");
+                    _PrintTargetResults(targetResults);
+                    if (timeSpan.GetMeasureLatency())
+                    {
+                        _PrintTargetLatency(targetResults);
+                    }
+                    if (timeSpan.GetCalculateIopsStdDev())
+                    {
+                        _PrintTargetIops(targetResults.readBucketizer, targetResults.writeBucketizer, timeSpan.GetIoBucketDurationInMilliseconds());
+                    }
+                    _PrintDec("</Target>\n");
+                }
+                if (profile.GetVerboseStats())
+                {
+                    _PrintWaitStats(threadResults);
+                }
+                _PrintDec("</Thread>\n");
+            }
+        }
+        else
+        {
+            _Print("<Error>The test was interrupted before the measurements began. No results are displayed.</Error>\n");
+        }
+
+        _PrintDec("</TimeSpan>\n");
+    }
+
+    _PrintDec("</Results>");
+    return _sResult;
+}

CristalDiskMark/source/diskspd22/diskspd.wprp

@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="utf-8"?>
+<WindowsPerformanceRecorder Author="Microsoft Corporation" Copyright="Microsoft Corporation. All rights reserved." Version="1.0">
+    <Profiles>
+        <EventCollector Id="EventCollectorDiskSpd" Name="WPR Event Collector">
+            <BufferSize Value="4"/>
+            <Buffers Value="1"/>
+        </EventCollector>
+        <EventCollector Id="EventCollectorDiskSpdIO" Name="WPR Event Collector">
+            <BufferSize Value="1024"/>
+            <Buffers Value="1000"/>
+        </EventCollector>
+        <EventProvider Id="EventProvider.Microsoft-Windows-DiskSpd" Name="CA13DB84-D0A9-5145-FCA4-468DA92FDC2D">
+            <Keywords>
+                <Keyword Value="0x1"/>
+            </Keywords>
+        </EventProvider>
+        <EventProvider Id="EventProvider.Microsoft-Windows-DiskSpd.Verbose" Name="CA13DB84-D0A9-5145-FCA4-468DA92FDC2D"/>
+        <Profile Id="DiskSpd.Light.Memory" Name="DiskSpd" Description="DiskSpd Storage Performance Tool" LoggingMode="Memory" DetailLevel="Light">
+            <Collectors>
+                <EventCollectorId Value="EventCollectorDiskSpd">
+                    <EventProviders>
+                        <EventProviderId Value="EventProvider.Microsoft-Windows-DiskSpd"/>
+                    </EventProviders>
+                </EventCollectorId>
+            </Collectors>
+        </Profile>
+        <Profile Id="DiskSpd.Light.File" Name="DiskSpd" Description="DiskSpd Storage Performance Tool" LoggingMode="File" DetailLevel="Light" Base="DiskSpd.Light.Memory"/>
+        <Profile Id="DiskSpd.Verbose.Memory" Name="DiskSpd" Description="DiskSpd Storage Performance Tool" LoggingMode="Memory" DetailLevel="Verbose">
+            <Collectors>
+                <EventCollectorId Value="EventCollectorDiskSpdIO">
+                    <EventProviders>
+                        <EventProviderId Value="EventProvider.Microsoft-Windows-DiskSpd.Verbose"/>
+                    </EventProviders>
+                </EventCollectorId>
+            </Collectors>
+        </Profile>
+        <Profile Id="DiskSpd.Verbose.File" Name="DiskSpd" Description="DiskSpd Storage Performance Tool" LoggingMode="File" DetailLevel="Verbose" Base="DiskSpd.Verbose.Memory"/>
+    </Profiles>
+</WindowsPerformanceRecorder>