Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add MetricVerbosity to logging and more workloads #433

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add MetricVerbosity to logging and more workloads #433

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 16 additions & 16 deletions ...VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/BombardierMetricsParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,23 +36,23 @@ public void BombardierParserVerifyMetrics()
IList<Metric> metrics = this.testParser.Parse();

Assert.AreEqual(16, metrics.Count);
MetricAssert.Exists(metrics, "Latency Max", 178703, "microsecond");
MetricAssert.Exists(metrics, "Latency Average", 8270.807963429836, "microsecond");
MetricAssert.Exists(metrics, "Latency Stddev", 6124.356473307014, "microsecond");
MetricAssert.Exists(metrics, "Latency P50", 6058, "microsecond");
MetricAssert.Exists(metrics, "Latency P75", 10913, "microsecond");
MetricAssert.Exists(metrics, "Latency P90", 17949, "microsecond");
MetricAssert.Exists(metrics, "Latency P95", 23318, "microsecond");
MetricAssert.Exists(metrics, "Latency P99", 35856, "microsecond");
MetricAssert.Exists(metrics, "Latency Max", 178703, MetricUnit.Microseconds);
MetricAssert.Exists(metrics, "Latency Average", 8270.807963429836, MetricUnit.Microseconds);
MetricAssert.Exists(metrics, "Latency Stddev", 6124.356473307014, MetricUnit.Microseconds);
MetricAssert.Exists(metrics, "Latency P50", 6058, MetricUnit.Microseconds);
MetricAssert.Exists(metrics, "Latency P75", 10913, MetricUnit.Microseconds);
MetricAssert.Exists(metrics, "Latency P90", 17949, MetricUnit.Microseconds);
MetricAssert.Exists(metrics, "Latency P95", 23318, MetricUnit.Microseconds);
MetricAssert.Exists(metrics, "Latency P99", 35856, MetricUnit.Microseconds);

MetricAssert.Exists(metrics, "RequestPerSecond Max", 67321.282458945348, "Reqs/sec");
MetricAssert.Exists(metrics, "RequestPerSecond Average", 31211.609987720527, "Reqs/sec");
MetricAssert.Exists(metrics, "RequestPerSecond Stddev", 6446.822354105378, "Reqs/sec");
MetricAssert.Exists(metrics, "RequestPerSecond P50", 31049.462844, "Reqs/sec");
MetricAssert.Exists(metrics, "RequestPerSecond P75", 35597.436614, "Reqs/sec");
MetricAssert.Exists(metrics, "RequestPerSecond P90", 39826.205746, "Reqs/sec");
MetricAssert.Exists(metrics, "RequestPerSecond P95", 41662.542962, "Reqs/sec");
MetricAssert.Exists(metrics, "RequestPerSecond P99", 49625.656227, "Reqs/sec");
MetricAssert.Exists(metrics, "RequestPerSecond Max", 67321.282458945348, MetricUnit.RequestsPerSec);
MetricAssert.Exists(metrics, "RequestPerSecond Average", 31211.609987720527, MetricUnit.RequestsPerSec);
MetricAssert.Exists(metrics, "RequestPerSecond Stddev", 6446.822354105378, MetricUnit.RequestsPerSec);
MetricAssert.Exists(metrics, "RequestPerSecond P50", 31049.462844, MetricUnit.RequestsPerSec);
MetricAssert.Exists(metrics, "RequestPerSecond P75", 35597.436614, MetricUnit.RequestsPerSec);
MetricAssert.Exists(metrics, "RequestPerSecond P90", 39826.205746, MetricUnit.RequestsPerSec);
MetricAssert.Exists(metrics, "RequestPerSecond P95", 41662.542962, MetricUnit.RequestsPerSec);
MetricAssert.Exists(metrics, "RequestPerSecond P99", 49625.656227, MetricUnit.RequestsPerSec);
}
}
}
4 changes: 2 additions & 2 deletions ...Client/VirtualClient.Actions.UnitTests/Compressor7zip/Compressor7zipMetricsParserTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ public void Compressor7zipResultsParserParsesTheExpectedSizesAndTimeFromResults(
public void Compressor7zipResultsParserCreatesTheExpectedMetricsFromResults()
{
IList<Metric> metrics = this.testParser.Parse();
MetricAssert.Exists(metrics, "Compressed size and Original size ratio", 26.03795395904138);
MetricAssert.Exists(metrics, "CompressionTime", 53.223, "seconds");
MetricAssert.Exists(metrics, "Compression_Ratio", 26.03795395904138);
MetricAssert.Exists(metrics, "Compression_Time", 53.223, "seconds");
}

[Test]
Expand Down
6 changes: 3 additions & 3 deletions src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,15 @@ public override IList<Metric> Parse()
{
if (this.Defintion == "custom_TSGT1.3dmdef")
{
metrics.Add(new Metric("timespy.graphics.1 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest1"), "fps", MetricRelativity.HigherIsBetter));
metrics.Add(new Metric("timespy.graphics.1 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest1"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0));
}
else if (this.Defintion == "custom_TSGT2.3dmdef")
{
metrics.Add(new Metric("timespy.graphics.2 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest2"), "fps", MetricRelativity.HigherIsBetter));
metrics.Add(new Metric("timespy.graphics.2 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0));
}
else if (this.Defintion == "custom_TSCT.3dmdef")
{
metrics.Add(new Metric("timespy.cpu [fps]", this.ParseXMLTag("TimeSpyPerformanceCpuSection2"), "fps", MetricRelativity.HigherIsBetter));
metrics.Add(new Metric("timespy.cpu [fps]", this.ParseXMLTag("TimeSpyPerformanceCpuSection2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0));
}
}
catch (Exception exc)
Expand Down
10 changes: 8 additions & 2 deletions src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,15 +61,21 @@ public override IList<Metric> Parse()
List<Metric> metrics = new List<Metric>();
int rows = this.SizeAndTime.Rows.Count;

metrics.Add(new Metric("Compressed size and Original size ratio", (Convert.ToDouble(this.SizeAndTime.Rows[2].ItemArray[2]) / Convert.ToDouble(this.SizeAndTime.Rows[0].ItemArray[2])) * 100, MetricRelativity.LowerIsBetter));
metrics.Add(new Metric(
"Compression_Ratio",
(Convert.ToDouble(this.SizeAndTime.Rows[2].ItemArray[2]) / Convert.ToDouble(this.SizeAndTime.Rows[0].ItemArray[2])) * 100,
"precentage",
relativity: MetricRelativity.LowerIsBetter,
verbosity: 2));

double compressionTime = 0;

for (int i = rows - 1; i > 2; i--)
{
compressionTime += Convert.ToDouble(this.SizeAndTime.Rows[i].ItemArray[2]);
}

metrics.Add(new Metric("CompressionTime", compressionTime, MetricUnit.Seconds, MetricRelativity.LowerIsBetter));
metrics.Add(new Metric("Compression_Time", compressionTime, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 0));

return metrics;
}
Expand Down
36 changes: 17 additions & 19 deletions src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ namespace VirtualClient.Actions
/// </summary>
public class BombardierMetricsParser : MetricsParser
{
private const string RequestPerSecond = "Reqs/sec";
private const string Microsecond = "microsecond";
private List<Metric> metrics;

/// <summary>
Expand All @@ -34,23 +32,23 @@ public override IList<Metric> Parse()
this.metrics = new List<Metric>();

Root root = JsonSerializer.Deserialize<Root>(this.PreprocessedText);
this.metrics.Add(new Metric("Latency Max", root.Result.Latency.Max, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter));
this.metrics.Add(new Metric("Latency Average", root.Result.Latency.Mean, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter));
this.metrics.Add(new Metric("Latency Stddev", root.Result.Latency.Stddev, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter));
this.metrics.Add(new Metric("Latency P50", root.Result.Latency.Percentiles.P50, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter));
this.metrics.Add(new Metric("Latency P75", root.Result.Latency.Percentiles.P75, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter));
this.metrics.Add(new Metric("Latency P90", root.Result.Latency.Percentiles.P90, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter));
this.metrics.Add(new Metric("Latency P95", root.Result.Latency.Percentiles.P95, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter));
this.metrics.Add(new Metric("Latency P99", root.Result.Latency.Percentiles.P99, BombardierMetricsParser.Microsecond, MetricRelativity.LowerIsBetter));

this.metrics.Add(new Metric("RequestPerSecond Max", root.Result.Rps.Max, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("RequestPerSecond Average", root.Result.Rps.Mean, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("RequestPerSecond Stddev", root.Result.Rps.Stddev, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("RequestPerSecond P50", root.Result.Rps.Percentiles.P50, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("RequestPerSecond P75", root.Result.Rps.Percentiles.P75, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("RequestPerSecond P90", root.Result.Rps.Percentiles.P90, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("RequestPerSecond P95", root.Result.Rps.Percentiles.P95, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("RequestPerSecond P99", root.Result.Rps.Percentiles.P99, BombardierMetricsParser.RequestPerSecond, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("Latency Max", root.Result.Latency.Max, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2));
this.metrics.Add(new Metric("Latency Average", root.Result.Latency.Mean, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter));
this.metrics.Add(new Metric("Latency Stddev", root.Result.Latency.Stddev, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2));
this.metrics.Add(new Metric("Latency P50", root.Result.Latency.Percentiles.P50, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0));
this.metrics.Add(new Metric("Latency P75", root.Result.Latency.Percentiles.P75, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter));
this.metrics.Add(new Metric("Latency P90", root.Result.Latency.Percentiles.P90, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter));
this.metrics.Add(new Metric("Latency P95", root.Result.Latency.Percentiles.P95, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter));
this.metrics.Add(new Metric("Latency P99", root.Result.Latency.Percentiles.P99, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0));

this.metrics.Add(new Metric("RequestPerSecond Max", root.Result.Rps.Max, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2));
this.metrics.Add(new Metric("RequestPerSecond Average", root.Result.Rps.Mean, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 0));
this.metrics.Add(new Metric("RequestPerSecond Stddev", root.Result.Rps.Stddev, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2));
this.metrics.Add(new Metric("RequestPerSecond P50", root.Result.Rps.Percentiles.P50, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("RequestPerSecond P75", root.Result.Rps.Percentiles.P75, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("RequestPerSecond P90", root.Result.Rps.Percentiles.P90, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("RequestPerSecond P95", root.Result.Rps.Percentiles.P95, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter));
this.metrics.Add(new Metric("RequestPerSecond P99", root.Result.Rps.Percentiles.P99, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter));

return this.metrics;
}
Expand Down
12 changes: 7 additions & 5 deletions src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,21 +46,23 @@ public override IList<Metric> Parse()
metrics.AddRange(this.CoreMarkResult.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "NA", namePrefix: string.Empty, ignoreFormatError: true));
// CoreMark result doesn't define the unit so needs manually assign units.
metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Unit = "bytes";
metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter;
metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Verbosity = 2;
metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Unit = "ticks";
metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter;
metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Verbosity = 2;
metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Unit = "secs";
metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter;
metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Verbosity = 2;
metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Unit = "iterations/sec";
metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter;
metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Verbosity = 0;
metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Unit = "iterations";
metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter;
metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Relativity = MetricRelativity.Undefined;
metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Verbosity = 2;

// This line won't be there if it's running single thread.
if (metrics.Any(m => m.Name == "Parallel PThreads"))
{
metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Unit = "threads";
metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter;
metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Verbosity = 2;
}

return metrics;
Expand Down
Loading
Loading