From b4e045ff86a8421d94c1ac78d6d9af286c77db20 Mon Sep 17 00:00:00 2001 From: Yang Pan Date: Fri, 7 Feb 2025 00:11:39 -0800 Subject: [PATCH] Add MetricVerbosity to logging and more workloads --- .../BombardierMetricsParserTests.cs | 32 +++--- .../Compressor7zipMetricsParserTests.cs | 4 +- .../3DMark/ThreeDMarkMetricsParser.cs | 6 +- .../7zip/Compression7zipMetricsParser.cs | 10 +- .../ASPNET/BombardierMetricsParser.cs | 36 +++--- .../CoreMark/CoreMarkMetricsParser.cs | 12 +- .../FIO/FioMetricsParser.cs | 104 +++++++++--------- .../GeekBench/GeekBenchMetricsParser.cs | 2 + .../HPLinpack/HPLinpackMetricsParser.cs | 4 +- .../HammerDB/HammerDBMetricsParser.cs | 12 +- .../Memtier/MemtierMetricsParser.cs | 4 + .../CPS/CPSMetricsParser.cs | 10 +- .../Latte/LatteMetricsParser.cs | 2 + .../NTttcp/NTttcpMetricsParser.cs | 14 +-- .../SockPerf/SockPerfMetricsParser.cs | 14 +-- .../OpenSSL/OpenSslMetricsParser.cs | 8 +- .../Redis/RedisBenchmarkMetricsParser.cs | 2 + .../SPECjbb/SpecJbbMetricsParser.cs | 2 +- .../SPECjvm/SpecJvmMetricsParser.cs | 4 +- .../StressNg/StressNgMetricsParser.cs | 12 +- .../VirtualClientLoggingExtensionsTests.cs | 39 +++++-- ...gExtensionsTests_BackwardsCompatibility.cs | 8 +- .../MetricAggregate.cs | 10 +- .../VirtualClient.Contracts/MetricUnit.cs | 5 + .../VirtualClientLoggingExtensions.cs | 6 + 25 files changed, 209 insertions(+), 153 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierMetricsParserTests.cs index 944e6aafbf..4a27f88c64 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierMetricsParserTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierMetricsParserTests.cs @@ -36,23 +36,23 @@ public void BombardierParserVerifyMetrics() IList metrics = this.testParser.Parse(); Assert.AreEqual(16, metrics.Count); - MetricAssert.Exists(metrics, "Latency Max", 178703, "microsecond"); - MetricAssert.Exists(metrics, "Latency Average", 8270.807963429836, "microsecond"); - MetricAssert.Exists(metrics, "Latency Stddev", 6124.356473307014, "microsecond"); - MetricAssert.Exists(metrics, "Latency P50", 6058, "microsecond"); - MetricAssert.Exists(metrics, "Latency P75", 10913, "microsecond"); - MetricAssert.Exists(metrics, "Latency P90", 17949, "microsecond"); - MetricAssert.Exists(metrics, "Latency P95", 23318, "microsecond"); - MetricAssert.Exists(metrics, "Latency P99", 35856, "microsecond"); + MetricAssert.Exists(metrics, "Latency Max", 178703, MetricUnit.Microseconds); + MetricAssert.Exists(metrics, "Latency Average", 8270.807963429836, MetricUnit.Microseconds); + MetricAssert.Exists(metrics, "Latency Stddev", 6124.356473307014, MetricUnit.Microseconds); + MetricAssert.Exists(metrics, "Latency P50", 6058, MetricUnit.Microseconds); + MetricAssert.Exists(metrics, "Latency P75", 10913, MetricUnit.Microseconds); + MetricAssert.Exists(metrics, "Latency P90", 17949, MetricUnit.Microseconds); + MetricAssert.Exists(metrics, "Latency P95", 23318, MetricUnit.Microseconds); + MetricAssert.Exists(metrics, "Latency P99", 35856, MetricUnit.Microseconds); - MetricAssert.Exists(metrics, "RequestPerSecond Max", 67321.282458945348, "Reqs/sec"); - MetricAssert.Exists(metrics, "RequestPerSecond Average", 31211.609987720527, "Reqs/sec"); - MetricAssert.Exists(metrics, "RequestPerSecond Stddev", 6446.822354105378, "Reqs/sec"); - MetricAssert.Exists(metrics, "RequestPerSecond P50", 31049.462844, "Reqs/sec"); - MetricAssert.Exists(metrics, "RequestPerSecond P75", 35597.436614, "Reqs/sec"); - MetricAssert.Exists(metrics, "RequestPerSecond P90", 39826.205746, "Reqs/sec"); - MetricAssert.Exists(metrics, "RequestPerSecond P95", 41662.542962, "Reqs/sec"); - MetricAssert.Exists(metrics, "RequestPerSecond P99", 49625.656227, "Reqs/sec"); + MetricAssert.Exists(metrics, "RequestPerSecond Max", 67321.282458945348, MetricUnit.RequestsPerSec); + MetricAssert.Exists(metrics, "RequestPerSecond Average", 31211.609987720527, MetricUnit.RequestsPerSec); + MetricAssert.Exists(metrics, "RequestPerSecond Stddev", 6446.822354105378, MetricUnit.RequestsPerSec); + MetricAssert.Exists(metrics, "RequestPerSecond P50", 31049.462844, MetricUnit.RequestsPerSec); + MetricAssert.Exists(metrics, "RequestPerSecond P75", 35597.436614, MetricUnit.RequestsPerSec); + MetricAssert.Exists(metrics, "RequestPerSecond P90", 39826.205746, MetricUnit.RequestsPerSec); + MetricAssert.Exists(metrics, "RequestPerSecond P95", 41662.542962, MetricUnit.RequestsPerSec); + MetricAssert.Exists(metrics, "RequestPerSecond P99", 49625.656227, MetricUnit.RequestsPerSec); } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Compressor7zip/Compressor7zipMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Compressor7zip/Compressor7zipMetricsParserTests.cs index 5a892e6f5a..4cca53eb00 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Compressor7zip/Compressor7zipMetricsParserTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Compressor7zip/Compressor7zipMetricsParserTests.cs @@ -44,8 +44,8 @@ public void Compressor7zipResultsParserParsesTheExpectedSizesAndTimeFromResults( public void Compressor7zipResultsParserCreatesTheExpectedMetricsFromResults() { IList metrics = this.testParser.Parse(); - MetricAssert.Exists(metrics, "Compressed size and Original size ratio", 26.03795395904138); - MetricAssert.Exists(metrics, "CompressionTime", 53.223, "seconds"); + MetricAssert.Exists(metrics, "Compression_Ratio", 26.03795395904138); + MetricAssert.Exists(metrics, "Compression_Time", 53.223, "seconds"); } [Test] diff --git a/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs index 6d5e3039d0..fac39805d9 100644 --- a/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs @@ -41,15 +41,15 @@ public override IList Parse() { if (this.Defintion == "custom_TSGT1.3dmdef") { - metrics.Add(new Metric("timespy.graphics.1 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest1"), "fps", MetricRelativity.HigherIsBetter)); + metrics.Add(new Metric("timespy.graphics.1 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest1"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0)); } else if (this.Defintion == "custom_TSGT2.3dmdef") { - metrics.Add(new Metric("timespy.graphics.2 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest2"), "fps", MetricRelativity.HigherIsBetter)); + metrics.Add(new Metric("timespy.graphics.2 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0)); } else if (this.Defintion == "custom_TSCT.3dmdef") { - metrics.Add(new Metric("timespy.cpu [fps]", this.ParseXMLTag("TimeSpyPerformanceCpuSection2"), "fps", MetricRelativity.HigherIsBetter)); + metrics.Add(new Metric("timespy.cpu [fps]", this.ParseXMLTag("TimeSpyPerformanceCpuSection2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0)); } } catch (Exception exc) diff --git a/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs index 0f7e0350f5..85525a3b56 100644 --- a/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs @@ -61,7 +61,13 @@ public override IList Parse() List metrics = new List(); int rows = this.SizeAndTime.Rows.Count; - metrics.Add(new Metric("Compressed size and Original size ratio", (Convert.ToDouble(this.SizeAndTime.Rows[2].ItemArray[2]) / Convert.ToDouble(this.SizeAndTime.Rows[0].ItemArray[2])) * 100, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric( + "Compression_Ratio", + (Convert.ToDouble(this.SizeAndTime.Rows[2].ItemArray[2]) / Convert.ToDouble(this.SizeAndTime.Rows[0].ItemArray[2])) * 100, + "precentage", + relativity: MetricRelativity.LowerIsBetter, + verbosity: 2)); + double compressionTime = 0; for (int i = rows - 1; i > 2; i--) @@ -69,7 +75,7 @@ public override IList Parse() compressionTime += Convert.ToDouble(this.SizeAndTime.Rows[i].ItemArray[2]); } - metrics.Add(new Metric("CompressionTime", compressionTime, MetricUnit.Seconds, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric("Compression_Time", compressionTime, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 0)); return metrics; } diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs index 58e71fe611..9aa2736eb9 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs @@ -14,8 +14,6 @@ namespace VirtualClient.Actions /// public class BombardierMetricsParser : MetricsParser { - private const string RequestPerSecond = "Reqs/sec"; - private const string Microsecond = "microsecond"; private List metrics; /// @@ -34,23 +32,23 @@ public override IList Parse() this.metrics = new List(); Root root = JsonSerializer.Deserialize(this.PreprocessedText); - this.metrics.Add(new Metric("Latency Max", root.Result.Latency.Max, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency Average", root.Result.Latency.Mean, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency Stddev", root.Result.Latency.Stddev, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency P50", root.Result.Latency.Percentiles.P50, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency P75", root.Result.Latency.Percentiles.P75, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency P90", root.Result.Latency.Percentiles.P90, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency P95", root.Result.Latency.Percentiles.P95, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency P99", root.Result.Latency.Percentiles.P99, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter)); - - this.metrics.Add(new Metric("RequestPerSecond Max", root.Result.Rps.Max, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter)); - this.metrics.Add(new Metric("RequestPerSecond Average", root.Result.Rps.Mean, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter)); - this.metrics.Add(new Metric("RequestPerSecond Stddev", root.Result.Rps.Stddev, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter)); - this.metrics.Add(new Metric("RequestPerSecond P50", root.Result.Rps.Percentiles.P50, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter)); - this.metrics.Add(new Metric("RequestPerSecond P75", root.Result.Rps.Percentiles.P75, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter)); - this.metrics.Add(new Metric("RequestPerSecond P90", root.Result.Rps.Percentiles.P90, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter)); - this.metrics.Add(new Metric("RequestPerSecond P95", root.Result.Rps.Percentiles.P95, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter)); - this.metrics.Add(new Metric("RequestPerSecond P99", root.Result.Rps.Percentiles.P99, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter)); + this.metrics.Add(new Metric("Latency Max", root.Result.Latency.Max, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + this.metrics.Add(new Metric("Latency Average", root.Result.Latency.Mean, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); + this.metrics.Add(new Metric("Latency Stddev", root.Result.Latency.Stddev, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + this.metrics.Add(new Metric("Latency P50", root.Result.Latency.Percentiles.P50, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); + this.metrics.Add(new Metric("Latency P75", root.Result.Latency.Percentiles.P75, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); + this.metrics.Add(new Metric("Latency P90", root.Result.Latency.Percentiles.P90, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); + this.metrics.Add(new Metric("Latency P95", root.Result.Latency.Percentiles.P95, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); + this.metrics.Add(new Metric("Latency P99", root.Result.Latency.Percentiles.P99, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); + + this.metrics.Add(new Metric("RequestPerSecond Max", root.Result.Rps.Max, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); + this.metrics.Add(new Metric("RequestPerSecond Average", root.Result.Rps.Mean, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric("RequestPerSecond Stddev", root.Result.Rps.Stddev, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); + this.metrics.Add(new Metric("RequestPerSecond P50", root.Result.Rps.Percentiles.P50, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter)); + this.metrics.Add(new Metric("RequestPerSecond P75", root.Result.Rps.Percentiles.P75, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter)); + this.metrics.Add(new Metric("RequestPerSecond P90", root.Result.Rps.Percentiles.P90, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter)); + this.metrics.Add(new Metric("RequestPerSecond P95", root.Result.Rps.Percentiles.P95, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter)); + this.metrics.Add(new Metric("RequestPerSecond P99", root.Result.Rps.Percentiles.P99, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter)); return this.metrics; } diff --git a/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs index 299e5186de..6ad353945d 100644 --- a/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs @@ -46,21 +46,23 @@ public override IList Parse() metrics.AddRange(this.CoreMarkResult.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "NA", namePrefix: string.Empty, ignoreFormatError: true)); // CoreMark result doesn't define the unit so needs manually assign units. metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Unit = "bytes"; - metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter; + metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Verbosity = 2; metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Unit = "ticks"; - metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter; + metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Verbosity = 2; metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Unit = "secs"; - metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter; + metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Verbosity = 2; metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Unit = "iterations/sec"; metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter; + metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Verbosity = 0; metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Unit = "iterations"; - metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter; + metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Relativity = MetricRelativity.Undefined; + metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Verbosity = 2; // This line won't be there if it's running single thread. if (metrics.Any(m => m.Name == "Parallel PThreads")) { metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Unit = "threads"; - metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter; + metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Verbosity = 2; } return metrics; diff --git a/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs index cad4d9d73d..0763edd594 100644 --- a/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs @@ -120,30 +120,29 @@ private void AddLatencyHistogramMeasurements(IList metrics, JToken resul { foreach (JToken job in jobs) { - this.AddMeasurement(metrics, job, $"latency_us.2", "h000000_002"); - this.AddMeasurement(metrics, job, $"latency_us.4", "h000000_004"); - this.AddMeasurement(metrics, job, $"latency_us.10", "h000000_010"); - this.AddMeasurement(metrics, job, $"latency_us.20", "h000000_020"); - this.AddMeasurement(metrics, job, $"latency_us.50", "h000000_050"); - this.AddMeasurement(metrics, job, $"latency_us.100", "h000000_100"); - this.AddMeasurement(metrics, job, $"latency_us.250", "h000000_250"); - this.AddMeasurement(metrics, job, $"latency_us.500", "h000000_500"); - this.AddMeasurement(metrics, job, $"latency_us.750", "h000000_750"); - this.AddMeasurement(metrics, job, $"latency_us.1000", "h000001_000"); - - this.AddMeasurement(metrics, job, $"latency_ms.2", "h000002_000"); - this.AddMeasurement(metrics, job, $"latency_ms.4", "h000004_000"); - this.AddMeasurement(metrics, job, $"latency_ms.10", "h000010_000"); - this.AddMeasurement(metrics, job, $"latency_ms.20", "h000020_000"); - this.AddMeasurement(metrics, job, $"latency_ms.50", "h000050_000"); - this.AddMeasurement(metrics, job, $"latency_ms.100", "h000100_000"); - this.AddMeasurement(metrics, job, $"latency_ms.250", "h000250_000"); - this.AddMeasurement(metrics, job, $"latency_ms.500", "h000500_000"); - this.AddMeasurement(metrics, job, $"latency_ms.750", "h000750_000"); - this.AddMeasurement(metrics, job, $"latency_ms.1000", "h001000_000"); - this.AddMeasurement(metrics, job, $"latency_ms.2000", "h002000_000"); - this.AddMeasurement(metrics, job, $"latency_ms.['>=2000']", "hgt002000_000"); - + this.AddMeasurement(metrics, job, $"latency_us.2", "h000000_002", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_us.4", "h000000_004", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_us.10", "h000000_010", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_us.20", "h000000_020", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_us.50", "h000000_050", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_us.100", "h000000_100", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_us.250", "h000000_250", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_us.500", "h000000_500", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_us.750", "h000000_750", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_us.1000", "h000001_000", verbosity: 2); + + this.AddMeasurement(metrics, job, $"latency_ms.2", "h000002_000", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_ms.4", "h000004_000", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_ms.10", "h000010_000", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_ms.20", "h000020_000", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_ms.50", "h000050_000", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_ms.100", "h000100_000", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_ms.250", "h000250_000", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_ms.500", "h000500_000", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_ms.750", "h000750_000", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_ms.1000", "h001000_000", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_ms.2000", "h002000_000", verbosity: 2); + this.AddMeasurement(metrics, job, $"latency_ms.['>=2000']", "hgt002000_000", verbosity: 2); } } } @@ -161,44 +160,43 @@ private void AddReadMeasurements(IList metrics, JToken resultsJson) { foreach (JToken job in jobs) { - this.AddMeasurement(metrics, job, $"read.io_bytes", "read_bytes", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.total_ios", "read_ios", null, MetricRelativity.HigherIsBetter); - - this.AddMeasurement(metrics, job, $"read.short_ios", "read_ios_short", null, MetricRelativity.LowerIsBetter); - this.AddMeasurement(metrics, job, $"read.drop_ios", "read_ios_dropped", null, MetricRelativity.LowerIsBetter); + this.AddMeasurement(metrics, job, $"read.io_bytes", "read_bytes", null, MetricRelativity.HigherIsBetter, verbosity: 2); + this.AddMeasurement(metrics, job, $"read.total_ios", "read_ios", null, MetricRelativity.HigherIsBetter, verbosity: 2); + this.AddMeasurement(metrics, job, $"read.short_ios", "read_ios_short", null, MetricRelativity.LowerIsBetter, verbosity: 2); + this.AddMeasurement(metrics, job, $"read.drop_ios", "read_ios_dropped", null, MetricRelativity.LowerIsBetter, verbosity: 2); this.AddMeasurement(metrics, job, $"read.bw", "read_bandwidth", MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter); this.AddMeasurement(metrics, job, $"read.bw_min", "read_bandwidth_min", MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter); this.AddMeasurement(metrics, job, $"read.bw_max", "read_bandwidth_max", MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.bw_mean", "read_bandwidth_mean", MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.bw_dev", "read_bandwidth_stdev", MetricUnit.KilobytesPerSecond, MetricRelativity.LowerIsBetter); + this.AddMeasurement(metrics, job, $"read.bw_mean", "read_bandwidth_mean", MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0); + this.AddMeasurement(metrics, job, $"read.bw_dev", "read_bandwidth_stdev", MetricUnit.KilobytesPerSecond, MetricRelativity.LowerIsBetter, verbosity: 2); this.AddMeasurement(metrics, job, $"read.iops", "read_iops", null, MetricRelativity.HigherIsBetter); this.AddMeasurement(metrics, job, $"read.iops_min", "read_iops_min", null, MetricRelativity.HigherIsBetter); this.AddMeasurement(metrics, job, $"read.iops_max", "read_iops_max", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.iops_mean", "read_iops_mean", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.iops_stddev", "read_iops_stdev", null, MetricRelativity.LowerIsBetter); + this.AddMeasurement(metrics, job, $"read.iops_mean", "read_iops_mean", null, MetricRelativity.HigherIsBetter, verbosity: 0); + this.AddMeasurement(metrics, job, $"read.iops_stddev", "read_iops_stdev", null, MetricRelativity.LowerIsBetter, verbosity: 2); this.AddMeasurement(metrics, job, $"read.lat_ns.min", "read_latency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"read.lat_ns.max", "read_latency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"read.lat_ns.mean", "read_latency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.lat_ns.stddev", "read_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); + this.AddMeasurement(metrics, job, $"read.lat_ns.stddev", "read_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); this.AddMeasurement(metrics, job, $"read.clat_ns.min", "read_completionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"read.clat_ns.max", "read_completionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"read.clat_ns.mean", "read_completionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.stddev", "read_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); + this.AddMeasurement(metrics, job, $"read.clat_ns.stddev", "read_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['50.000000']", "read_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['50.000000']", "read_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['70.000000']", "read_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['90.000000']", "read_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.000000']", "read_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.000000']", "read_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.990000']", "read_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"read.slat_ns.min", "read_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"read.slat_ns.max", "read_submissionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"read.slat_ns.mean", "read_submissionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.slat_ns.stddev", "read_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); + this.AddMeasurement(metrics, job, $"read.slat_ns.stddev", "read_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); } } @@ -217,49 +215,49 @@ private void AddWriteMeasurements(IList metrics, JToken resultsJson) { foreach (JToken job in jobs) { - this.AddMeasurement(metrics, job, $"write.io_bytes", "write_bytes", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.total_ios", "write_ios", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.short_ios", "write_ios_short", null, MetricRelativity.LowerIsBetter); - this.AddMeasurement(metrics, job, $"write.drop_ios", "write_ios_dropped", null, MetricRelativity.LowerIsBetter); + this.AddMeasurement(metrics, job, $"write.io_bytes", "write_bytes", null, MetricRelativity.HigherIsBetter, verbosity: 2); + this.AddMeasurement(metrics, job, $"write.total_ios", "write_ios", null, MetricRelativity.HigherIsBetter, verbosity: 2); + this.AddMeasurement(metrics, job, $"write.short_ios", "write_ios_short", null, MetricRelativity.LowerIsBetter, verbosity: 2); + this.AddMeasurement(metrics, job, $"write.drop_ios", "write_ios_dropped", null, MetricRelativity.LowerIsBetter, verbosity: 2); this.AddMeasurement(metrics, job, $"write.bw", "write_bandwidth", MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter); this.AddMeasurement(metrics, job, $"write.bw_min", "write_bandwidth_min", MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter); this.AddMeasurement(metrics, job, $"write.bw_max", "write_bandwidth_max", MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.bw_mean", "write_bandwidth_mean", MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.bw_dev", "write_bandwidth_stdev", MetricUnit.KilobytesPerSecond, MetricRelativity.LowerIsBetter); + this.AddMeasurement(metrics, job, $"write.bw_mean", "write_bandwidth_mean", MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0); + this.AddMeasurement(metrics, job, $"write.bw_dev", "write_bandwidth_stdev", MetricUnit.KilobytesPerSecond, MetricRelativity.LowerIsBetter, verbosity: 2); this.AddMeasurement(metrics, job, $"write.iops", "write_iops", null, MetricRelativity.HigherIsBetter); this.AddMeasurement(metrics, job, $"write.iops_min", "write_iops_min", null, MetricRelativity.HigherIsBetter); this.AddMeasurement(metrics, job, $"write.iops_max", "write_iops_max", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.iops_mean", "write_iops_mean", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.iops_stddev", "write_iops_stdev", null, MetricRelativity.LowerIsBetter); + this.AddMeasurement(metrics, job, $"write.iops_mean", "write_iops_mean", null, MetricRelativity.HigherIsBetter, verbosity: 0); + this.AddMeasurement(metrics, job, $"write.iops_stddev", "write_iops_stdev", null, MetricRelativity.LowerIsBetter, verbosity: 2); this.AddMeasurement(metrics, job, $"write.lat_ns.min", "write_latency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"write.lat_ns.max", "write_latency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"write.lat_ns.mean", "write_latency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.lat_ns.stddev", "write_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); + this.AddMeasurement(metrics, job, $"write.lat_ns.stddev", "write_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); this.AddMeasurement(metrics, job, $"write.clat_ns.min", "write_completionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"write.clat_ns.max", "write_completionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"write.clat_ns.mean", "write_completionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.stddev", "write_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); + this.AddMeasurement(metrics, job, $"write.clat_ns.stddev", "write_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['50.000000']", "write_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['50.000000']", "write_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['70.000000']", "write_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['90.000000']", "write_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.000000']", "write_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.000000']", "write_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.990000']", "write_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"write.slat_ns.min", "write_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"write.slat_ns.max", "write_submissionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); this.AddMeasurement(metrics, job, $"write.slat_ns.mean", "write_submissionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.slat_ns.stddev", "write_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); + this.AddMeasurement(metrics, job, $"write.slat_ns.stddev", "write_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); } } } - private void AddMeasurement(IList metrics, JToken job, string path, string metricName, string metricUnit = null, MetricRelativity metricRelativity = MetricRelativity.Undefined, double? conversionFactor = null) + private void AddMeasurement(IList metrics, JToken job, string path, string metricName, string metricUnit = null, MetricRelativity metricRelativity = MetricRelativity.Undefined, double? conversionFactor = null, int verbosity = 1) { var jobOptions = job.SelectToken("['job options']") as JObject; @@ -282,7 +280,7 @@ private void AddMeasurement(IList metrics, JToken job, string path, stri measurementValue = measurementValue * conversionFactor.Value; } - metrics.Add(new Metric(metricName, measurementValue, metricUnit, metricRelativity, metadata: metricMetaData)); + metrics.Add(new Metric(metricName, measurementValue, metricUnit, metricRelativity, verbosity: verbosity, metadata: metricMetaData)); } } } diff --git a/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs index a361d90ceb..740cb33262 100644 --- a/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs @@ -79,6 +79,8 @@ public override IList Parse() metrics.AddRange(this.SingleCoreSummary.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "Score", namePrefix: "SingleCoreSummary-", metricRelativity: MetricRelativity.HigherIsBetter)); metrics.AddRange(this.MultiCoreSummary.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "Score", namePrefix: "MultiCoreSummary-", metricRelativity: MetricRelativity.HigherIsBetter)); + metrics.ForEach(metric => metric.Verbosity = 0); + return metrics; } diff --git a/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs index d500fd7768..f96bac30ab 100644 --- a/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs @@ -64,8 +64,8 @@ public override IList Parse() { $"Q_WR{st[0]}", st[4] }, }; - this.metrics.Add(new Metric($"Time", Convert.ToDouble(st[5]), "secs", metadata: metadata)); - this.metrics.Add(new Metric($"GFlops", Convert.ToDouble(st[6]), "Gflops", metadata: metadata)); + this.metrics.Add(new Metric($"Time", Convert.ToDouble(st[5]), "secs", MetricRelativity.Undefined, metadata: metadata, verbosity: 2)); + this.metrics.Add(new Metric($"GFlops", Convert.ToDouble(st[6]), "Gflops", metadata: metadata, relativity: MetricRelativity.HigherIsBetter, verbosity: 0)); } } diff --git a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs index 004a4a179b..48a17b8eef 100644 --- a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs @@ -56,18 +56,22 @@ public override IList Parse() metrics.Add(new Metric( "Operations/sec", metrics.First(m => m.Name == "Operations/min").Value / 60, - relativity: MetricRelativity.HigherIsBetter)); + MetricUnit.OperationsPerSec, + relativity: MetricRelativity.HigherIsBetter, + verbosity: 0)); metrics.Add(new Metric( "Transactions/sec", metrics.First(m => m.Name == "Transactions/min").Value / 60, - relativity: MetricRelativity.HigherIsBetter)); + MetricUnit.TransactionsPerSec, + relativity: MetricRelativity.HigherIsBetter, + verbosity: 0)); return metrics; } catch (Exception exc) { - throw new WorkloadResultsException("Failed to parse PostgreSQL metrics from results.", exc, ErrorReason.InvalidResults); + throw new WorkloadResultsException("Failed to parse HammerDB metrics from results.", exc, ErrorReason.InvalidResults); } } @@ -94,7 +98,7 @@ private void ThrowIfInvalidOutputFormat() { if (this.Sections.Count <= 0 || !this.Sections.ContainsKey("TEST RESULT")) { - throw new SchemaException("The PostgreSQL output file has incorrect format for parsing"); + throw new SchemaException("The HammerDB output file has incorrect format for parsing"); } } } diff --git a/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs index 1d1bba621a..687c3237ee 100644 --- a/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs @@ -321,6 +321,7 @@ private static IDictionary CreateMetricsBin(string metr metricNamePrefix == null ? "Throughput" : $"{metricNamePrefix}-Throughput", metricUnit: MetricUnit.RequestsPerSec, relativity: MetricRelativity.HigherIsBetter, + verbosity: 0, description: "Total number of requests/operations per second during the period of time.") }, { @@ -351,6 +352,7 @@ private static IDictionary CreateMetricsBin(string metr metricNamePrefix == null ? "Latency-P50" : $"{metricNamePrefix}-Latency-P50", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, + verbosity: 0, description: "The latency for 50% of all requests was at or under this value.") }, { @@ -375,6 +377,7 @@ private static IDictionary CreateMetricsBin(string metr metricNamePrefix == null ? "Latency-P99" : $"{metricNamePrefix}-Latency-P99", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, + verbosity: 0, description: "The latency for 99% of all requests was at or under this value.") }, { @@ -391,6 +394,7 @@ private static IDictionary CreateMetricsBin(string metr metricNamePrefix == null ? "Bandwidth" : $"{metricNamePrefix}-Bandwidth", metricUnit: MetricUnit.KilobytesPerSecond, relativity: MetricRelativity.HigherIsBetter, + verbosity: 0, description: "Total amount of data transferred per second during the period of time.") } }; diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs index 8b4d7d0ccb..1d1489957b 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs @@ -85,7 +85,7 @@ private static void AddStatisticalMetrics(IList metrics, List ti metrics.Add(new Metric("ConnectsPerSec_Min", connectsPerSec.Min())); metrics.Add(new Metric("ConnectsPerSec_Max", connectsPerSec.Max())); metrics.Add(new Metric("ConnectsPerSec_Med", connectsPerSec.Median())); - metrics.Add(new Metric("ConnectsPerSec_Avg", connectsPerSec.Average())); + metrics.Add(new Metric("ConnectsPerSec_Avg", connectsPerSec.Average(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 0)); metrics.Add(new Metric("ConnectsPerSec_P25", connectsPerSec.Percentile(25))); metrics.Add(new Metric("ConnectsPerSec_P50", connectsPerSec.Percentile(50))); metrics.Add(new Metric("ConnectsPerSec_P75", connectsPerSec.Percentile(75))); @@ -96,10 +96,10 @@ private static void AddStatisticalMetrics(IList metrics, List ti metrics.Add(new Metric("ConnectsPerSec_P99_999", Statistics.QuantileCustom(connectsPerSec, 1d - 0.00001d, QuantileDefinition.R3))); double median = Statistics.Median(connectsPerSec); double[] absoluteDeviations = connectsPerSec.Select(x => Math.Abs(x - median)).ToArray(); - metrics.Add(new Metric("ConnectsPerSec_Mad", Statistics.Median(absoluteDeviations))); - metrics.Add(new Metric("ConnectsPerSec_StandardErrorMean", sem)); - metrics.Add(new Metric("ConnectsPerSec_LowerCI", lowerCI)); - metrics.Add(new Metric("ConnectsPerSec_UpperCI", upperCI)); + metrics.Add(new Metric("ConnectsPerSec_Mad", Statistics.Median(absoluteDeviations), MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_StandardErrorMean", sem, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_LowerCI", lowerCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_UpperCI", upperCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); } /// diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs index 8f994e1806..c09ac0905a 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs @@ -144,6 +144,7 @@ private void AddLatencyPercentileMetrics(IList metrics, IEnumerable entry.FrequencyInTotal <= totalObservations * .75) @@ -177,6 +178,7 @@ private void AddLatencyPercentileMetrics(IList metrics, IEnumerable entry.FrequencyInTotal <= totalObservations * .999) diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs index 2a0741387e..b0f0d7c913 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs @@ -51,14 +51,14 @@ public override IList Parse() NTttcpMetric throughputMetric = this.result.Throughput.First(t => t.Units.Equals("mbps")); IList metricList = new List() { - new Metric("TotalBytesMB", this.result.TotalBytesMB.Value, this.result.TotalBytesMB.Units, MetricRelativity.HigherIsBetter), - new Metric("AvgBytesPerCompl", this.result.AverageBytesPerCompletion.Value, this.result.AverageBytesPerCompletion.Units, MetricRelativity.Undefined), - new Metric("AvgFrameSize", this.result.AverageFrameSize.Value, this.result.AverageFrameSize.Units, MetricRelativity.Undefined), - new Metric("ThroughputMbps", throughputMetric.Value, throughputMetric.Units, MetricRelativity.HigherIsBetter), - new Metric("AvgPacketsPerInterrupt", this.result.AveragePacketsPerInterrupt.Value, this.result.AveragePacketsPerInterrupt.Units, MetricRelativity.Undefined), + new Metric("TotalBytesMB", this.result.TotalBytesMB.Value, this.result.TotalBytesMB.Units, MetricRelativity.HigherIsBetter, verbosity: 2), + new Metric("AvgBytesPerCompl", this.result.AverageBytesPerCompletion.Value, this.result.AverageBytesPerCompletion.Units, MetricRelativity.Undefined, verbosity: 2), + new Metric("AvgFrameSize", this.result.AverageFrameSize.Value, this.result.AverageFrameSize.Units, MetricRelativity.Undefined, verbosity: 2), + new Metric("ThroughputMbps", throughputMetric.Value, throughputMetric.Units, MetricRelativity.HigherIsBetter, verbosity: 0), + new Metric("AvgPacketsPerInterrupt", this.result.AveragePacketsPerInterrupt.Value, this.result.AveragePacketsPerInterrupt.Units, MetricRelativity.Undefined, verbosity: 2), new Metric("InterruptsPerSec", this.result.Interrupts.Value, this.result.Interrupts.Units, MetricRelativity.Undefined), - new Metric("PacketsRetransmitted", this.result.PacketsRetransmitted, MetricRelativity.LowerIsBetter), - new Metric("Errors", this.result.Errors, MetricRelativity.LowerIsBetter), + new Metric("PacketsRetransmitted", this.result.PacketsRetransmitted, MetricUnit.Count, MetricRelativity.LowerIsBetter, verbosity: 2), + new Metric("Errors", this.result.Errors, MetricUnit.Count, MetricRelativity.LowerIsBetter), }; if (this.result.TcpAverageRtt != null) diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs index ddf7c8daf8..248f29e5d9 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs @@ -111,20 +111,20 @@ private static void AddStatisticalMetrics(IList metrics, List pa metrics.Add(new Metric("Latency-Max", packetsLatencyValues.Max(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); metrics.Add(new Metric("Latency-Avg", packetsLatencyValues.Average(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); metrics.Add(new Metric("Latency-P25", packetsLatencyValues.Percentile(25), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P50", packetsLatencyValues.Percentile(50), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric("Latency-P50", packetsLatencyValues.Percentile(50), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); metrics.Add(new Metric("Latency-P75", packetsLatencyValues.Percentile(75), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); metrics.Add(new Metric("Latency-P90", packetsLatencyValues.Percentile(90), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P99", packetsLatencyValues.Percentile(99), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric("Latency-P99", packetsLatencyValues.Percentile(99), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); metrics.Add(new Metric("Latency-P99.9", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); metrics.Add(new Metric("Latency-P99.99", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.0001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); metrics.Add(new Metric("Latency-P99.999", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.00001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); double median = Statistics.Median(packetsLatencyValues); double[] absoluteDeviations = packetsLatencyValues.Select(x => Math.Abs(x - median)).ToArray(); - metrics.Add(new Metric("Latency-Mad", Statistics.Median(absoluteDeviations), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-StandardErrorMean", sem, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-Stdev", sd, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-LowerCI", lowerCI)); - metrics.Add(new Metric("Latency-UpperCI", upperCI)); + metrics.Add(new Metric("Latency-Mad", Statistics.Median(absoluteDeviations), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-StandardErrorMean", sem, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-Stdev", sd, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-LowerCI", lowerCI, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-UpperCI", upperCI, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs index fab67ae2f5..c1a348b6a9 100644 --- a/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs @@ -171,7 +171,7 @@ public override IList Parse() // aes-128-cbc -1040172814.93 1589805.17k 1657786.91k 1674053.70k 1677365.52k 1633415.99k if (metricValue >= 0) { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); } } } @@ -187,7 +187,7 @@ public override IList Parse() { if (metricName.Contains("/")) { - metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter)); + metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 0)); } else { @@ -208,7 +208,7 @@ public override IList Parse() { if (metricName.Contains("/")) { - metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter)); + metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 0)); } else { @@ -229,7 +229,7 @@ public override IList Parse() { if (metricName.Contains("/")) { - metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter)); + metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 0)); } else { diff --git a/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs index cc4e08c2af..5c7904bd3e 100644 --- a/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs @@ -108,6 +108,7 @@ private void AddMetricsFromCsv(List metrics) reqPerSec, MetricUnit.RequestsPerSec, relativity: MetricRelativity.HigherIsBetter, + verbosity: 0, description: "Total number of requests/operations per second during the period of time.")); } @@ -158,6 +159,7 @@ private void AddMetricsFromCsv(List metrics) p99Latency, MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, + verbosity: 0, description: "The latency for 99% of all requests was at or under this value.")); } diff --git a/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs index f5be2020cd..9d671963f6 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs @@ -49,7 +49,7 @@ public override IList Parse() } else { - this.metrics.Add(new Metric(name.Trim(), Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter)); + this.metrics.Add(new Metric(name.Trim(), Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); } } diff --git a/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs index f59a59173d..f546bfaffd 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs @@ -46,7 +46,7 @@ public override IList Parse() string[] nameAndValue = Regex.Split(line, columnRegex.ToString(), columnRegex.Options); string metricName = nameAndValue[0].Trim(); double metricValue = Convert.ToDouble(nameAndValue[1].Replace(SpecJvmMetricsParser.operationPerSecond, string.Empty)); - this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter)); + this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); } else { @@ -55,7 +55,7 @@ public override IList Parse() string metricName = nameAndValue[0].Trim(); string[] a = line.Split(" "); double metricValue = Convert.ToDouble(nameAndValue[1]); - this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter)); + this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); } } diff --git a/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs index f684cea643..8f8a008a2f 100644 --- a/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs @@ -42,12 +42,12 @@ public override IList Parse() foreach (StressNgStressorResult stressor in parsedResult.Metrics) { - this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops", stressor.BogoOps, "BogoOps")); - this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-usr-sys-time", stressor.BogoOpsPerSecondUsrSysTime, BogusOperationsPerSecond)); - this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-real-time", stressor.BogoOpsPerSecondRealTime, BogusOperationsPerSecond)); - this.metrics.Add(new Metric($"{stressor.Stressor}-wall-clock-time", stressor.WallClockTime, "second")); - this.metrics.Add(new Metric($"{stressor.Stressor}-user-time", stressor.UserTime, "second")); - this.metrics.Add(new Metric($"{stressor.Stressor}-system-time", stressor.SystemTime, "second")); + this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops", stressor.BogoOps, "BogoOps", MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-usr-sys-time", stressor.BogoOpsPerSecondUsrSysTime, BogusOperationsPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-real-time", stressor.BogoOpsPerSecondRealTime, BogusOperationsPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric($"{stressor.Stressor}-wall-clock-time", stressor.WallClockTime, "second", MetricRelativity.LowerIsBetter, verbosity: 2)); + this.metrics.Add(new Metric($"{stressor.Stressor}-user-time", stressor.UserTime, "second", MetricRelativity.LowerIsBetter, verbosity: 2)); + this.metrics.Add(new Metric($"{stressor.Stressor}-system-time", stressor.SystemTime, "second", MetricRelativity.LowerIsBetter, verbosity: 2)); } return this.metrics; diff --git a/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientLoggingExtensionsTests.cs b/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientLoggingExtensionsTests.cs index b29b653ffa..d75192e7f1 100644 --- a/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientLoggingExtensionsTests.cs +++ b/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientLoggingExtensionsTests.cs @@ -604,7 +604,7 @@ public void LogMetricsExtensionLogsTheExpectedEvents_Scenario1() this.mockLogger.Verify(logger => logger.Log( LogLevel.Information, It.Is(eventId => eventId.Id == (int)LogType.Metrics && eventId.Name.EndsWith("ScenarioResult")), - It.Is(context => context.Properties.Count == 15 + It.Is(context => context.Properties.Count == 16 && context.ActivityId == this.mockEventContext.ActivityId && context.ParentActivityId == this.mockEventContext.ParentActivityId && context.Properties.ContainsKey("scenarioName") @@ -630,6 +630,7 @@ public void LogMetricsExtensionLogsTheExpectedEvents_Scenario1() && context.Properties["metricCategorization"].ToString() == expectedMetricCategorization && context.Properties["metricDescription"].ToString() == string.Empty && context.Properties["metricRelativity"].ToString() == MetricRelativity.Undefined.ToString() + && context.Properties["metricVerbosity"].ToString() == "1" && context.Properties["toolName"].ToString() == expectedToolName && context.Properties["toolVersion"].ToString() == string.Empty && context.Properties["toolResults"].ToString() == string.Empty @@ -649,6 +650,7 @@ public void LogMetricsExtensionLogsTheExpectedEvents_Scenario2() string expectedToolResults = "Tool A: metric1=value 1 | metric 2=value 2"; double expectedMetricValue = 123.456; string expectedUnits = "seconds"; + int expectedVerbosity = 1; string expectedMetricCategorization = "instanceA"; string expectedDescription = "Metric description"; MetricRelativity expectedRelativity = MetricRelativity.LowerIsBetter; @@ -674,6 +676,7 @@ public void LogMetricsExtensionLogsTheExpectedEvents_Scenario2() expectedTags, this.mockEventContext, expectedRelativity, + expectedVerbosity, expectedDescription, expectedToolResults, expectedToolVersion, @@ -682,7 +685,7 @@ public void LogMetricsExtensionLogsTheExpectedEvents_Scenario2() this.mockLogger.Verify(logger => logger.Log( LogLevel.Information, It.Is(eventId => eventId.Id == (int)LogType.Metrics && eventId.Name.EndsWith("ScenarioResult")), - It.Is(context => context.Properties.Count == 15 + It.Is(context => context.Properties.Count == 16 && context.ActivityId == this.mockEventContext.ActivityId && context.ParentActivityId == this.mockEventContext.ParentActivityId && context.Properties.ContainsKey("scenarioName") @@ -708,6 +711,7 @@ public void LogMetricsExtensionLogsTheExpectedEvents_Scenario2() && context.Properties["metricCategorization"].ToString() == expectedMetricCategorization && context.Properties["metricDescription"].ToString() == expectedDescription && context.Properties["metricRelativity"].ToString() == expectedRelativity.ToString() + && context.Properties["metricVerbosity"].ToString() == "1" && context.Properties["toolName"].ToString() == expectedToolName && context.Properties["tags"].ToString() == string.Join(",", expectedTags) && context.Properties["metadata_metrics"] == expectedMetadata as Object @@ -1751,7 +1755,7 @@ public void LogFailedMetricExtensionLogsTheExpectedInformation() this.mockLogger.Verify(logger => logger.Log( LogLevel.Information, It.Is(eventId => eventId.Id == (int)LogType.Metrics && eventId.Name.EndsWith("ScenarioResult")), - It.Is(context => context.Properties.Count == 15 + It.Is(context => context.Properties.Count == 16 && context.ActivityId == this.mockEventContext.ActivityId && context.ParentActivityId == this.mockEventContext.ParentActivityId && context.Properties.ContainsKey("scenarioName") @@ -1777,6 +1781,7 @@ public void LogFailedMetricExtensionLogsTheExpectedInformation() && context.Properties["metricCategorization"].ToString() == expectedMetricCategorization && context.Properties["metricDescription"].ToString() == "Indicates the component or toolset execution failed for the scenario defined." && context.Properties["metricRelativity"].ToString() == MetricRelativity.LowerIsBetter.ToString() + && context.Properties["metricVerbosity"].ToString() == "0" && context.Properties["toolName"].ToString() == expectedToolName && context.Properties["toolVersion"].ToString() == expectedToolVersion && context.Properties["tags"].ToString() == string.Join(",", expectedTags) @@ -1799,7 +1804,7 @@ public void LogFailedMetricExtensionLogsTheExpectedInformation_WhenMinimumInfoIs this.mockLogger.Verify(logger => logger.Log( LogLevel.Information, It.Is(eventId => eventId.Id == (int)LogType.Metrics && eventId.Name.EndsWith("ScenarioResult")), - It.Is(context => context.Properties.Count == 15 + It.Is(context => context.Properties.Count == 16 && context.ActivityId == this.mockEventContext.ActivityId && context.ParentActivityId == this.mockEventContext.ParentActivityId && context.Properties.ContainsKey("scenarioName") @@ -1825,6 +1830,7 @@ public void LogFailedMetricExtensionLogsTheExpectedInformation_WhenMinimumInfoIs && context.Properties["metricCategorization"].ToString() == string.Empty && context.Properties["metricDescription"].ToString() == "Indicates the component or toolset execution failed for the scenario defined." && context.Properties["metricRelativity"].ToString() == MetricRelativity.LowerIsBetter.ToString() + && context.Properties["metricVerbosity"].ToString() == "0" && context.Properties["toolName"].ToString() == expectedToolName && context.Properties["toolVersion"].ToString() == string.Empty && context.Properties["tags"].ToString() == string.Empty @@ -1859,7 +1865,7 @@ public void LogSuccessMetricExtensionLogsTheExpectedInformation() this.mockLogger.Verify(logger => logger.Log( LogLevel.Information, It.Is(eventId => eventId.Id == (int)LogType.Metrics && eventId.Name.EndsWith("ScenarioResult")), - It.Is(context => context.Properties.Count == 15 + It.Is(context => context.Properties.Count == 16 && context.ActivityId == this.mockEventContext.ActivityId && context.ParentActivityId == this.mockEventContext.ParentActivityId && context.Properties.ContainsKey("scenarioName") @@ -1885,6 +1891,7 @@ public void LogSuccessMetricExtensionLogsTheExpectedInformation() && context.Properties["metricCategorization"].ToString() == expectedMetricCategorization && context.Properties["metricDescription"].ToString() == "Indicates the component or toolset execution succeeded for the scenario defined." && context.Properties["metricRelativity"].ToString() == MetricRelativity.HigherIsBetter.ToString() + && context.Properties["metricVerbosity"].ToString() == "2" && context.Properties["toolName"].ToString() == expectedToolName && context.Properties["toolVersion"].ToString() == expectedToolVersion && context.Properties["tags"].ToString() == string.Join(",", expectedTags) @@ -1907,7 +1914,7 @@ public void LogSuccessMetricExtensionLogsTheExpectedInformation_WhenMinimumInfoI this.mockLogger.Verify(logger => logger.Log( LogLevel.Information, It.Is(eventId => eventId.Id == (int)LogType.Metrics && eventId.Name.EndsWith("ScenarioResult")), - It.Is(context => context.Properties.Count == 15 + It.Is(context => context.Properties.Count == 16 && context.ActivityId == this.mockEventContext.ActivityId && context.ParentActivityId == this.mockEventContext.ParentActivityId && context.Properties.ContainsKey("scenarioName") @@ -1933,6 +1940,7 @@ public void LogSuccessMetricExtensionLogsTheExpectedInformation_WhenMinimumInfoI && context.Properties["metricCategorization"].ToString() == string.Empty && context.Properties["metricDescription"].ToString() == "Indicates the component or toolset execution succeeded for the scenario defined." && context.Properties["metricRelativity"].ToString() == MetricRelativity.HigherIsBetter.ToString() + && context.Properties["metricVerbosity"].ToString() == "2" && context.Properties["toolName"].ToString() == expectedToolName && context.Properties["toolVersion"].ToString() == string.Empty && context.Properties["tags"].ToString() == string.Empty @@ -1958,7 +1966,7 @@ public void LogFailedMetricExtensionOnComponentLogsTheExpectedInformation_scenar Assert.IsInstanceOf(loggedMetric.Item3); EventContext context = loggedMetric.Item3 as EventContext; - Assert.IsTrue(context.Properties.Count == 15 + Assert.IsTrue(context.Properties.Count == 16 && context.Properties.ContainsKey("scenarioName") && context.Properties.ContainsKey("scenarioStartTime") && context.Properties.ContainsKey("scenarioEndTime") @@ -1980,6 +1988,7 @@ public void LogFailedMetricExtensionOnComponentLogsTheExpectedInformation_scenar && context.Properties["metricCategorization"].ToString() == string.Empty && context.Properties["metricDescription"].ToString() == "Indicates the component or toolset execution failed for the scenario defined." && context.Properties["metricRelativity"].ToString() == MetricRelativity.LowerIsBetter.ToString() + && context.Properties["metricVerbosity"].ToString() == "0" && context.Properties["toolName"].ToString() == component.TypeName && context.Properties["toolVersion"].ToString() == string.Empty && context.Properties["tags"].ToString() == string.Empty @@ -2003,7 +2012,7 @@ public void LogFailedMetricExtensionOnComponentLogsTheExpectedInformation_scenar Assert.IsInstanceOf(loggedMetric.Item3); EventContext context = loggedMetric.Item3 as EventContext; - Assert.IsTrue(context.Properties.Count == 15 + Assert.IsTrue(context.Properties.Count == 16 && context.Properties.ContainsKey("scenarioName") && context.Properties.ContainsKey("scenarioStartTime") && context.Properties.ContainsKey("scenarioEndTime") @@ -2025,6 +2034,7 @@ public void LogFailedMetricExtensionOnComponentLogsTheExpectedInformation_scenar && context.Properties["metricCategorization"].ToString() == string.Empty && context.Properties["metricDescription"].ToString() == "Indicates the component or toolset execution failed for the scenario defined." && context.Properties["metricRelativity"].ToString() == MetricRelativity.LowerIsBetter.ToString() + && context.Properties["metricVerbosity"].ToString() == "0" && context.Properties["toolName"].ToString() == component.TypeName && context.Properties["toolVersion"].ToString() == string.Empty && context.Properties["tags"].ToString() == string.Empty @@ -2054,7 +2064,7 @@ public void LogFailedMetricExtensionOnComponentLogsTheExpectedInformation_scenar Assert.IsInstanceOf(loggedMetric.Item3); EventContext context = loggedMetric.Item3 as EventContext; - Assert.IsTrue(context.Properties.Count == 15 + Assert.IsTrue(context.Properties.Count == 16 && context.Properties.ContainsKey("scenarioName") && context.Properties.ContainsKey("scenarioStartTime") && context.Properties.ContainsKey("scenarioEndTime") @@ -2076,6 +2086,7 @@ public void LogFailedMetricExtensionOnComponentLogsTheExpectedInformation_scenar && context.Properties["metricCategorization"].ToString() == expectedMetricCategorization && context.Properties["metricDescription"].ToString() == "Indicates the component or toolset execution failed for the scenario defined." && context.Properties["metricRelativity"].ToString() == MetricRelativity.LowerIsBetter.ToString() + && context.Properties["metricVerbosity"].ToString() == "0" && context.Properties["toolName"].ToString() == expectedToolName && context.Properties["toolVersion"].ToString() == expectedToolVersion && context.Properties["tags"].ToString() == string.Empty @@ -2099,7 +2110,7 @@ public void LogSuccessMetricExtensionOnComponentLogsTheExpectedInformation_scena Assert.IsInstanceOf(loggedMetric.Item3); EventContext context = loggedMetric.Item3 as EventContext; - Assert.IsTrue(context.Properties.Count == 15 + Assert.IsTrue(context.Properties.Count == 16 && context.Properties.ContainsKey("scenarioName") && context.Properties.ContainsKey("scenarioStartTime") && context.Properties.ContainsKey("scenarioEndTime") @@ -2121,6 +2132,7 @@ public void LogSuccessMetricExtensionOnComponentLogsTheExpectedInformation_scena && context.Properties["metricCategorization"].ToString() == string.Empty && context.Properties["metricDescription"].ToString() == "Indicates the component or toolset execution succeeded for the scenario defined." && context.Properties["metricRelativity"].ToString() == MetricRelativity.HigherIsBetter.ToString() + && context.Properties["metricVerbosity"].ToString() == "2" && context.Properties["toolName"].ToString() == component.TypeName && context.Properties["toolVersion"].ToString() == string.Empty && context.Properties["tags"].ToString() == string.Empty @@ -2144,7 +2156,7 @@ public void LogSuccessMetricExtensionOnComponentLogsTheExpectedInformation_scena Assert.IsInstanceOf(loggedMetric.Item3); EventContext context = loggedMetric.Item3 as EventContext; - Assert.IsTrue(context.Properties.Count == 15 + Assert.IsTrue(context.Properties.Count == 16 && context.Properties.ContainsKey("scenarioName") && context.Properties.ContainsKey("scenarioStartTime") && context.Properties.ContainsKey("scenarioEndTime") @@ -2166,6 +2178,7 @@ public void LogSuccessMetricExtensionOnComponentLogsTheExpectedInformation_scena && context.Properties["metricCategorization"].ToString() == string.Empty && context.Properties["metricDescription"].ToString() == "Indicates the component or toolset execution succeeded for the scenario defined." && context.Properties["metricRelativity"].ToString() == MetricRelativity.HigherIsBetter.ToString() + && context.Properties["metricVerbosity"].ToString() == "2" && context.Properties["toolName"].ToString() == component.TypeName && context.Properties["toolVersion"].ToString() == string.Empty && context.Properties["tags"].ToString() == string.Empty @@ -2195,7 +2208,7 @@ public void LogSuccessMetricExtensionOnComponentLogsTheExpectedInformation_scena Assert.IsInstanceOf(loggedMetric.Item3); EventContext context = loggedMetric.Item3 as EventContext; - Assert.IsTrue(context.Properties.Count == 15 + Assert.IsTrue(context.Properties.Count == 16 && context.Properties.ContainsKey("scenarioName") && context.Properties.ContainsKey("scenarioStartTime") && context.Properties.ContainsKey("scenarioEndTime") @@ -2205,6 +2218,7 @@ public void LogSuccessMetricExtensionOnComponentLogsTheExpectedInformation_scena && context.Properties.ContainsKey("metricCategorization") && context.Properties.ContainsKey("metricDescription") && context.Properties.ContainsKey("metricRelativity") + && context.Properties.ContainsKey("metricVerbosity") && context.Properties.ContainsKey("toolName") && context.Properties.ContainsKey("toolVersion") && context.Properties.ContainsKey("toolResults") @@ -2217,6 +2231,7 @@ public void LogSuccessMetricExtensionOnComponentLogsTheExpectedInformation_scena && context.Properties["metricCategorization"].ToString() == expectedMetricCategorization && context.Properties["metricDescription"].ToString() == "Indicates the component or toolset execution succeeded for the scenario defined." && context.Properties["metricRelativity"].ToString() == MetricRelativity.HigherIsBetter.ToString() + && context.Properties["metricVerbosity"].ToString() == "2" && context.Properties["toolName"].ToString() == expectedToolName && context.Properties["toolVersion"].ToString() == expectedToolVersion && context.Properties["tags"].ToString() == string.Empty diff --git a/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientLoggingExtensionsTests_BackwardsCompatibility.cs b/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientLoggingExtensionsTests_BackwardsCompatibility.cs index a9eac98c8b..5b5fe9b033 100644 --- a/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientLoggingExtensionsTests_BackwardsCompatibility.cs +++ b/src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientLoggingExtensionsTests_BackwardsCompatibility.cs @@ -119,6 +119,7 @@ public void LogMetricsExtensionLogsTheExpectedEventsWhenSupportForOriginalSchema string expectedToolResults = "Tool A: metric1=value 1 | metric 2=value 2"; double expectedMetricValue = 123.456; string expectedUnits = "seconds"; + int expectedVerbosity = 1; string expectedMetricCategorization = "instanceA"; string expectedDescription = "Metric description"; MetricRelativity expectedRelativity = MetricRelativity.LowerIsBetter; @@ -144,6 +145,7 @@ public void LogMetricsExtensionLogsTheExpectedEventsWhenSupportForOriginalSchema expectedTags, this.mockEventContext, expectedRelativity, + expectedVerbosity, expectedDescription, expectedToolResults, expectedToolVersion, @@ -154,7 +156,7 @@ public void LogMetricsExtensionLogsTheExpectedEventsWhenSupportForOriginalSchema this.mockLogger.Verify(logger => logger.Log( LogLevel.Information, It.Is(eventId => eventId.Id == (int)LogType.Metrics && eventId.Name.EndsWith("ScenarioResult")), - It.Is(context => context.Properties.Count == 22 + It.Is(context => context.Properties.Count == 23 && context.ActivityId == this.mockEventContext.ActivityId && context.ParentActivityId == this.mockEventContext.ParentActivityId && context.Properties.ContainsKey("testName") @@ -170,6 +172,7 @@ public void LogMetricsExtensionLogsTheExpectedEventsWhenSupportForOriginalSchema && context.Properties["testArguments"].ToString() == expectedTestArguments && context.Properties["metricName"].ToString() == expectedMetricName && context.Properties["testResult"].ToString() == expectedMetricValue.ToString() + && context.Properties["metricVerbosity"].ToString() == "1" && context.Properties["testedInstance"].ToString() == expectedMetricCategorization), null, null)); @@ -178,7 +181,7 @@ public void LogMetricsExtensionLogsTheExpectedEventsWhenSupportForOriginalSchema this.mockLogger.Verify(logger => logger.Log( LogLevel.Information, It.Is(eventId => eventId.Id == (int)LogType.Metrics && eventId.Name.EndsWith("ScenarioResult")), - It.Is(context => context.Properties.Count == 22 + It.Is(context => context.Properties.Count == 23 && context.ActivityId == this.mockEventContext.ActivityId && context.ParentActivityId == this.mockEventContext.ParentActivityId && context.Properties.ContainsKey("scenarioName") @@ -204,6 +207,7 @@ public void LogMetricsExtensionLogsTheExpectedEventsWhenSupportForOriginalSchema && context.Properties["metricCategorization"].ToString() == expectedMetricCategorization && context.Properties["metricDescription"].ToString() == expectedDescription && context.Properties["metricRelativity"].ToString() == expectedRelativity.ToString() + && context.Properties["metricVerbosity"].ToString() == "1" && context.Properties["toolName"].ToString() == expectedToolName && context.Properties["tags"].ToString() == string.Join(",", expectedTags) && context.Properties["metadata_metrics"] == expectedMetadata as Object diff --git a/src/VirtualClient/VirtualClient.Contracts/MetricAggregate.cs b/src/VirtualClient/VirtualClient.Contracts/MetricAggregate.cs index 788f71558e..9e0c543c0b 100644 --- a/src/VirtualClient/VirtualClient.Contracts/MetricAggregate.cs +++ b/src/VirtualClient/VirtualClient.Contracts/MetricAggregate.cs @@ -63,12 +63,14 @@ public MetricAggregate(string metricName, string metricUnit, MetricAggregateType /// The unit of measurement (e.g. KB/sec). /// The relativity of the metric in comparison with other samples of itself (e.g. higher/lower is better). /// The type of aggregation to apply to the metric values/samples. + /// Verbosity/Importance of metric /// A description of the metric. - public MetricAggregate(string metricName, string metricUnit, MetricRelativity relativity, MetricAggregateType aggregateType = MetricAggregateType.Average, string description = null) + public MetricAggregate(string metricName, string metricUnit, MetricRelativity relativity, MetricAggregateType aggregateType = MetricAggregateType.Average, int verbosity = 1, string description = null) : this(metricName, aggregateType, description) { this.Unit = metricUnit; this.Relativity = relativity; + this.Verbosity = verbosity; } /// @@ -97,6 +99,12 @@ public MetricAggregate(string metricName, string metricUnit, MetricRelativity re /// public MetricRelativity Relativity { get; set; } + /// + /// Metric verbosity to descript importance of metric. Default to 1, which means standard. + /// Verbosity 0: Critical. Verbosity 1: Standard. Verbosity 2: Informational. + /// + public int Verbosity { get; set; } + /// /// Creates a metric from the aggregate of values/samples. /// diff --git a/src/VirtualClient/VirtualClient.Contracts/MetricUnit.cs b/src/VirtualClient/VirtualClient.Contracts/MetricUnit.cs index ad4ef73afb..7e0acaa80c 100644 --- a/src/VirtualClient/VirtualClient.Contracts/MetricUnit.cs +++ b/src/VirtualClient/VirtualClient.Contracts/MetricUnit.cs @@ -44,6 +44,11 @@ public class MetricUnit /// public const string Bytes = "bytes"; + /// + /// Count + /// + public const string Count = "Count"; + /// /// Kilobytes per second /// diff --git a/src/VirtualClient/VirtualClient.Contracts/VirtualClientLoggingExtensions.cs b/src/VirtualClient/VirtualClient.Contracts/VirtualClientLoggingExtensions.cs index 9f82cd0928..982b51201c 100644 --- a/src/VirtualClient/VirtualClient.Contracts/VirtualClientLoggingExtensions.cs +++ b/src/VirtualClient/VirtualClient.Contracts/VirtualClientLoggingExtensions.cs @@ -435,6 +435,7 @@ public static void LogFailedMetric( tags, eventContext, MetricRelativity.LowerIsBetter, + verbosity: 0, "Indicates the component or toolset execution failed for the scenario defined.", toolVersion: toolVersion); } @@ -709,6 +710,7 @@ public static void LogMetrics( tags, eventContext, metric.Relativity, + metric.Verbosity, metric.Description, toolResults, toolVersion, @@ -733,6 +735,7 @@ public static void LogMetrics( /// Tags associated with the test. /// Provided correlation identifiers and context properties for the event. /// The relationship of the metric value to an outcome (e.g. higher/lower is better). + /// Metric verbosity. Default to 1. /// A description of the metric. /// The raw results produced by the workload/monitor etc. from which the metrics were parsed. /// The version of the tool/toolset. @@ -752,6 +755,7 @@ public static void LogMetrics( IEnumerable tags, EventContext eventContext, MetricRelativity relativity = MetricRelativity.Undefined, + int verbosity = 1, string description = null, string toolResults = null, string toolVersion = null, @@ -776,6 +780,7 @@ public static void LogMetrics( { "metricCategorization", metricCategorization ?? string.Empty }, { "metricDescription", description ?? string.Empty }, { "metricRelativity", relativity.ToString() }, + { "metricVerbosity", verbosity.ToString() }, { "toolName", toolName }, { "toolVersion", toolVersion ?? string.Empty }, { "toolResults", toolResults ?? string.Empty }, @@ -951,6 +956,7 @@ public static void LogSuccessMetric( tags, eventContext, MetricRelativity.HigherIsBetter, + verbosity: 2, "Indicates the component or toolset execution succeeded for the scenario defined.", toolVersion: toolVersion); }