From 14ba967a43a448229cebd24f34f5c019cec49a83 Mon Sep 17 00:00:00 2001 From: Saul Rennison Date: Thu, 5 May 2022 11:37:14 +0100 Subject: [PATCH 1/5] Add System.Net.NameResolution metrics --- .../IntegrationTests/IntegrationTestBase.cs | 27 ++++++------ .../IntegrationTests/NameResolutionTests.cs | 38 +++++++++++++++++ .../DotNetRuntimeStatsBuilder.cs | 17 +++++++- .../Parsers/NameResolutionEventParser.cs | 28 +++++++++++++ .../Producers/NameResolutionMetricProducer.cs | 42 +++++++++++++++++++ tools/DocsGenerator/Program.cs | 5 ++- 6 files changed, 140 insertions(+), 17 deletions(-) create mode 100644 src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/NameResolutionTests.cs create mode 100644 src/prometheus-net.DotNetRuntime/EventListening/Parsers/NameResolutionEventParser.cs create mode 100644 src/prometheus-net.DotNetRuntime/Metrics/Producers/NameResolutionMetricProducer.cs diff --git a/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/IntegrationTestBase.cs b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/IntegrationTestBase.cs index 4fd8fe5..5697377 100644 --- a/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/IntegrationTestBase.cs +++ b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/IntegrationTestBase.cs @@ -24,21 +24,22 @@ public void SetUp() MetricProducer = (TMetricProducer)_collector.ServiceProvider.GetServices().Single(x => x is TMetricProducer); // wait for event listener thread to spin up - var waitingFor = Stopwatch.StartNew(); var waitFor = TimeSpan.FromSeconds(10); - - while (!_collector.EventListeners.All(x => x.StartedReceivingEvents)) + + Console.Write("Waiting for event listeners to be active.. "); + if (!SpinWait.SpinUntil(() => + _collector.EventListeners.All(x => x.StartedReceivingEvents), + waitFor)) { - Thread.Sleep(10); - Console.Write("Waiting for event listeners to be active.. "); - - if (waitingFor.Elapsed > waitFor) - { - Assert.Fail($"Waited {waitFor} and still not all event listeners were ready! Event listeners not ready: {string.Join(", ", _collector.EventListeners.Where(x => !x.StartedReceivingEvents))}"); - return; - } + var notReadySources = + _collector.EventListeners.Where(x => !x.StartedReceivingEvents) + .Select(x => x.EventListener.EventSourceName) + .ToList(); + + if (notReadySources.Any()) + Assert.Fail($"Waited {waitFor} and still not all event listeners were ready! Event listeners not ready: {string.Join(", ", notReadySources)}"); } - + Console.WriteLine("All event listeners should be active now."); } @@ -50,4 +51,4 @@ public void TearDown() protected abstract DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure); } -} \ No newline at end of file +} diff --git a/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/NameResolutionTests.cs b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/NameResolutionTests.cs new file mode 100644 index 0000000..d14aaa6 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime.Tests/IntegrationTests/NameResolutionTests.cs @@ -0,0 +1,38 @@ +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using NUnit.Framework; +using Prometheus.DotNetRuntime.Metrics.Producers; + +namespace Prometheus.DotNetRuntime.Tests.IntegrationTests +{ + public class NameResolutionTests : IntegrationTestBase + { + [Test] + public async Task Given_a_DNS_lookup_metrics_should_increase() + { + // arrange + var initialLookups = MetricProducer.DnsLookups.Value; + var initialLookupDuration = MetricProducer.DnsLookupDuration.Sum; + + // act + var lookups = Enumerable.Range(1, 10) + .Select(n => Dns.GetHostEntryAsync("localhost")) + .ToArray(); + + await Task.WhenAll(lookups); + + // assert + Assert.That(() => MetricProducer.DnsLookups.Value, Is.GreaterThanOrEqualTo(initialLookups + 10).After(10_000, 100)); + Assert.That(MetricProducer.DnsLookupDuration.Sum, Is.GreaterThan(initialLookupDuration)); + } + + protected override DotNetRuntimeStatsBuilder.Builder ConfigureBuilder(DotNetRuntimeStatsBuilder.Builder toConfigure) + { + // Do a DNS lookup to force creation of the NameResolution event source. + Dns.GetHostEntry("localhost"); + + return toConfigure.WithNameResolution(); + } + } +} diff --git a/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs b/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs index f2fa3c9..0855645 100644 --- a/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs +++ b/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs @@ -31,7 +31,8 @@ public static Builder Default() .WithGcStats() .WithJitStats() .WithSocketStats() - .WithExceptionStats(); + .WithExceptionStats() + .WithNameResolution(); } /// @@ -192,6 +193,18 @@ public Builder WithSocketStats() return this; } + /// + /// Include metrics around DNS lookup requests and average duration. + /// + /// + public Builder WithNameResolution() + { + ListenerRegistrations.AddOrReplace(ListenerRegistration.Create(CaptureLevel.Counters, sp => new NameResolutionEventParser())); + _services.TryAddSingletonEnumerable(); + + return this; + } + /// /// Include metrics that measure the number of exceptions thrown. /// @@ -299,4 +312,4 @@ internal static void RegisterDefaultConsumers(IServiceCollection services) } } } -} \ No newline at end of file +} diff --git a/src/prometheus-net.DotNetRuntime/EventListening/Parsers/NameResolutionEventParser.cs b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/NameResolutionEventParser.cs new file mode 100644 index 0000000..4d0d3ac --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/EventListening/Parsers/NameResolutionEventParser.cs @@ -0,0 +1,28 @@ +using System; +using System.Diagnostics.Tracing; + +namespace Prometheus.DotNetRuntime.EventListening.Parsers +{ + public class NameResolutionEventParser : EventCounterParserBase, NameResolutionEventParser.Events.CountersV5_0 + { + +#pragma warning disable CS0067 + [CounterName("dns-lookups-requested")] + public event Action DnsLookupsRequested; + + [CounterName("dns-lookups-duration")] + public event Action DnsLookupsDuration; +#pragma warning restore CS0067 + + public override string EventSourceName => "System.Net.NameResolution"; + + public static class Events + { + public interface CountersV5_0 : ICounterEvents + { + event Action DnsLookupsRequested; + event Action DnsLookupsDuration; + } + } + } +} diff --git a/src/prometheus-net.DotNetRuntime/Metrics/Producers/NameResolutionMetricProducer.cs b/src/prometheus-net.DotNetRuntime/Metrics/Producers/NameResolutionMetricProducer.cs new file mode 100644 index 0000000..1a7ba04 --- /dev/null +++ b/src/prometheus-net.DotNetRuntime/Metrics/Producers/NameResolutionMetricProducer.cs @@ -0,0 +1,42 @@ +using Prometheus.DotNetRuntime.EventListening.Parsers; + +namespace Prometheus.DotNetRuntime.Metrics.Producers +{ + public class NameResolutionMetricProducer : IMetricProducer + { + private readonly Consumes _nameResolutionCounter; + + public NameResolutionMetricProducer(Consumes nameResolutionCounter) + { + _nameResolutionCounter = nameResolutionCounter; + } + + public void RegisterMetrics(MetricFactory metrics) + { + if (!_nameResolutionCounter.Enabled) + return; + + DnsLookups = metrics.CreateCounter("dotnet_dns_lookups_total", "The number of DNS lookups requested since the process started"); + var lastLookups = 0.0; + _nameResolutionCounter.Events.DnsLookupsRequested += e => + { + DnsLookups.Inc(e.Mean - lastLookups); + lastLookups = e.Mean; + }; + + DnsLookupDuration = metrics.CreateHistogram("dotnet_dns_lookup_duration_avg_seconds", "The average time taken for a DNS lookup"); + _nameResolutionCounter.Events.DnsLookupsDuration += e => + { + // Convert milliseconds to seconds + DnsLookupDuration.Observe(e.Mean / 1000.0); + }; + } + + internal Histogram DnsLookupDuration { get; private set; } + internal Counter DnsLookups { get; private set; } + + public void UpdateMetrics() + { + } + } +} diff --git a/tools/DocsGenerator/Program.cs b/tools/DocsGenerator/Program.cs index 2c458fb..e176348 100644 --- a/tools/DocsGenerator/Program.cs +++ b/tools/DocsGenerator/Program.cs @@ -36,7 +36,8 @@ static async Task Main(string[] args) SourceAndConfig.CreateFrom(b => b.WithJitStats(CaptureLevel.Counters, SampleEvery.OneEvent)), SourceAndConfig.CreateFrom(b => b.WithJitStats(CaptureLevel.Verbose, SampleEvery.OneEvent)), SourceAndConfig.CreateFrom(b => b.WithExceptionStats(CaptureLevel.Errors)), - SourceAndConfig.CreateFrom(b => b.WithSocketStats()) + SourceAndConfig.CreateFrom(b => b.WithSocketStats()), + SourceAndConfig.CreateFrom(b => b.WithNameResolution()) }; var assemblyDocs = typeof(DotNetRuntimeStatsBuilder).Assembly.LoadXmlDocumentation(); @@ -216,4 +217,4 @@ public record GroupedMetrics(ImmutableList CommonMetrics, ImmutableDi public IEnumerable<(MethodInfo method, Source[] sources)> MethodsToSources => SourceToMetrics.Keys.GroupBy(x => x.Method, (k, v) => (method: k, sources: v.OrderBy(x => x.Level).ToArray())); } } -} \ No newline at end of file +} From 536a849231389facac8077b8ef1aa929f4184c1e Mon Sep 17 00:00:00 2001 From: Saul Rennison Date: Thu, 5 May 2022 12:56:35 +0100 Subject: [PATCH 2/5] Update exposed metrics markdown files --- docs/metrics-exposed-3.1.md | 94 +++++++++++++------------ docs/metrics-exposed-5.0.md | 133 +++++++++++++++++++----------------- 2 files changed, 122 insertions(+), 105 deletions(-) diff --git a/docs/metrics-exposed-3.1.md b/docs/metrics-exposed-3.1.md index 293eb3d..5dc1011 100644 --- a/docs/metrics-exposed-3.1.md +++ b/docs/metrics-exposed-3.1.md @@ -11,20 +11,41 @@ Metrics that are included by default, regardless of what stats collectors are en | `dotnet_build_info` | `Gauge` | Build information about prometheus\-net.DotNetRuntime and the environment | `version`, `target_framework`, `runtime_version`, `os_version`, `process_architecture`, `gc_mode` | | `process_cpu_count` | `Gauge` | The number of processor cores available to this process. | | -## `.WithJitStats()` +## `.WithContentionStats()` -Include metrics summarizing the volume of methods being compiled - by the Just\-In\-Time compiler. +Include metrics around volume of locks contended. -### `CaptureLevel.Verbose` +### `CaptureLevel.Counters` + +| Name | Type | Description | Labels | +| ------------------------- | --------- | ----------------------------- | ------ | +| `dotnet_contention_total` | `Counter` | The number of locks contended | | + +### `CaptureLevel.Informational` Includes metrics generated by `CaptureLevel.Counters` plus: -| Name | Type | Description | Labels | -| --------------------------------- | --------- | ------------------------------------------------------------------------------------------------- | --------- | -| `dotnet_jit_cpu_ratio` | `Gauge` | The amount of total CPU time consumed spent JIT'ing | | -| `dotnet_jit_method_total` | `Counter` | Total number of methods compiled by the JIT compiler, broken down by compilation for dynamic code | `dynamic` | -| `dotnet_jit_method_seconds_total` | `Counter` | Total number of seconds spent in the JIT compiler, broken down by compilation for dynamic code | `dynamic` | +| Name | Type | Description | Labels | +| --------------------------------- | --------- | ----------------------------------------------- | ------ | +| `dotnet_contention_seconds_total` | `Counter` | The total amount of time spent contending locks | | + +## `.WithExceptionStats()` + +Include metrics that measure the number of exceptions thrown. + +### `CaptureLevel.Counters` + +| Name | Type | Description | Labels | +| ------------------------- | --------- | -------------------------- | ------ | +| `dotnet_exceptions_total` | `Counter` | Count of exceptions thrown | | + +### `CaptureLevel.Errors` + +Includes metrics generated by `CaptureLevel.Counters` plus: + +| Name | Type | Description | Labels | +| ------------------------- | --------- | ----------------------------------------------- | ------ | +| `dotnet_exceptions_total` | `Counter` | Count of exceptions thrown, broken down by type | `type` | ## `.WithGcStats()` @@ -35,11 +56,11 @@ Include metrics recording the frequency and duration of garbage collections\/ pa | Name | Type | Description | Labels | | ---------------------------------------- | --------- | ---------------------------------------------------------------------------------------------- | --------------- | +| `dotnet_gc_allocated_bytes_total` | `Counter` | The total number of bytes allocated on the managed heap | | | `dotnet_gc_collection_count_total` | `Counter` | Counts the number of garbage collections that have occurred, broken down by generation number. | `gc_generation` | | `dotnet_gc_heap_size_bytes` | `Gauge` | The current size of all heaps (only updated after a garbage collection) | `gc_generation` | -| `dotnet_gc_pause_ratio` | `Gauge` | The percentage of time the process spent paused for garbage collection | | | `dotnet_gc_memory_total_available_bytes` | `Gauge` | The upper limit on the amount of physical memory .NET can allocate to | | -| `dotnet_gc_allocated_bytes_total` | `Counter` | The total number of bytes allocated on the managed heap | | +| `dotnet_gc_pause_ratio` | `Gauge` | The percentage of time the process spent paused for garbage collection | | ### `CaptureLevel.Informational` @@ -47,11 +68,11 @@ Includes metrics generated by `CaptureLevel.Counters` plus: | Name | Type | Description | Labels | | ------------------------------------- | ----------- | -------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | -| `dotnet_gc_finalization_queue_length` | `Gauge` | The number of objects waiting to be finalized | | | `dotnet_gc_collection_count_total` | `Counter` | Counts the number of garbage collections that have occurred, broken down by generation number and the reason for the collection. | `gc_generation`, `gc_reason` | -| `dotnet_gc_pause_seconds` | `Histogram` | The amount of time execution was paused for garbage collection | | -| `dotnet_gc_cpu_ratio` | `Gauge` | The percentage of process CPU time spent running garbage collections | | | `dotnet_gc_collection_seconds` | `Histogram` | The amount of time spent running garbage collections | `gc_generation`, `gc_type` | +| `dotnet_gc_cpu_ratio` | `Gauge` | The percentage of process CPU time spent running garbage collections | | +| `dotnet_gc_finalization_queue_length` | `Gauge` | The number of objects waiting to be finalized | | +| `dotnet_gc_pause_seconds` | `Histogram` | The amount of time execution was paused for garbage collection | | | `dotnet_gc_pinned_objects` | `Gauge` | The number of pinned objects | | ### `CaptureLevel.Verbose` @@ -62,41 +83,26 @@ Includes metrics generated by `CaptureLevel.Counters`, `CaptureLevel.Information | --------------------------------- | --------- | ------------------------------------------------------- | --------- | | `dotnet_gc_allocated_bytes_total` | `Counter` | The total number of bytes allocated on the managed heap | `gc_heap` | -## `.WithExceptionStats()` - -Include metrics that measure the number of exceptions thrown. - -### `CaptureLevel.Counters` - -| Name | Type | Description | Labels | -| ------------------------- | --------- | -------------------------- | ------ | -| `dotnet_exceptions_total` | `Counter` | Count of exceptions thrown | | - -### `CaptureLevel.Errors` - -Includes metrics generated by `CaptureLevel.Counters` plus: - -| Name | Type | Description | Labels | -| ------------------------- | --------- | ----------------------------------------------- | ------ | -| `dotnet_exceptions_total` | `Counter` | Count of exceptions thrown, broken down by type | `type` | +## `.WithJitStats()` -## `.WithContentionStats()` +Include metrics summarizing the volume of methods being compiled + by the Just\-In\-Time compiler. -Include metrics around volume of locks contended. +### `CaptureLevel.Verbose` -### `CaptureLevel.Counters` +| Name | Type | Description | Labels | +| --------------------------------- | --------- | ------------------------------------------------------------------------------------------------- | --------- | +| `dotnet_jit_cpu_ratio` | `Gauge` | The amount of total CPU time consumed spent JIT'ing | | +| `dotnet_jit_method_seconds_total` | `Counter` | Total number of seconds spent in the JIT compiler, broken down by compilation for dynamic code | `dynamic` | +| `dotnet_jit_method_total` | `Counter` | Total number of methods compiled by the JIT compiler, broken down by compilation for dynamic code | `dynamic` | -| Name | Type | Description | Labels | -| ------------------------- | --------- | ----------------------------- | ------ | -| `dotnet_contention_total` | `Counter` | The number of locks contended | | +## `.WithNameResolution()` -### `CaptureLevel.Informational` +This method does not export any metrics on this version of the framework. -Includes metrics generated by `CaptureLevel.Counters` plus: +## `.WithSocketStats()` -| Name | Type | Description | Labels | -| --------------------------------- | --------- | ----------------------------------------------- | ------ | -| `dotnet_contention_seconds_total` | `Counter` | The total amount of time spent contending locks | | +This method does not export any metrics on this version of the framework. ## `.WithThreadPoolStats()` @@ -107,10 +113,10 @@ Include metrics around the size of the worker and IO thread pools and reasons | Name | Type | Description | Labels | | ------------------------------------ | ----------- | ----------------------------------------------------------------------------------------------------------------------------- | ------ | -| `dotnet_threadpool_timer_count` | `Gauge` | The number of timers active | | +| `dotnet_threadpool_num_threads` | `Gauge` | The number of active threads in the thread pool | | | `dotnet_threadpool_queue_length` | `Histogram` | Measures the queue length of the thread pool. Values greater than 0 indicate a backlog of work for the threadpool to process. | | | `dotnet_threadpool_throughput_total` | `Counter` | The total number of work items that have finished execution in the thread pool | | -| `dotnet_threadpool_num_threads` | `Gauge` | The number of active threads in the thread pool | | +| `dotnet_threadpool_timer_count` | `Gauge` | The number of timers active | | ### `CaptureLevel.Informational` diff --git a/docs/metrics-exposed-5.0.md b/docs/metrics-exposed-5.0.md index 818ef41..565b3a5 100644 --- a/docs/metrics-exposed-5.0.md +++ b/docs/metrics-exposed-5.0.md @@ -8,10 +8,46 @@ Metrics that are included by default, regardless of what stats collectors are en | Name | Type | Description | Labels | | ------------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | -| `dotnet_internal_recycle_count` | `Counter` | prometheus\-net.DotNetRuntime internal metric. Counts the number of times the underlying event listeners have been recycled | | | `dotnet_build_info` | `Gauge` | Build information about prometheus\-net.DotNetRuntime and the environment | `version`, `target_framework`, `runtime_version`, `os_version`, `process_architecture`, `gc_mode` | +| `dotnet_internal_recycle_count` | `Counter` | prometheus\-net.DotNetRuntime internal metric. Counts the number of times the underlying event listeners have been recycled | | | `process_cpu_count` | `Gauge` | The number of processor cores available to this process. | | +## `.WithContentionStats()` + +Include metrics around volume of locks contended. + +### `CaptureLevel.Counters` + +| Name | Type | Description | Labels | +| ------------------------- | --------- | ----------------------------- | ------ | +| `dotnet_contention_total` | `Counter` | The number of locks contended | | + +### `CaptureLevel.Informational` + +Includes metrics generated by `CaptureLevel.Counters` plus: + +| Name | Type | Description | Labels | +| --------------------------------- | --------- | ----------------------------------------------- | ------ | +| `dotnet_contention_seconds_total` | `Counter` | The total amount of time spent contending locks | | + +## `.WithExceptionStats()` + +Include metrics that measure the number of exceptions thrown. + +### `CaptureLevel.Counters` + +| Name | Type | Description | Labels | +| ------------------------- | --------- | -------------------------- | ------ | +| `dotnet_exceptions_total` | `Counter` | Count of exceptions thrown | | + +### `CaptureLevel.Errors` + +Includes metrics generated by `CaptureLevel.Counters` plus: + +| Name | Type | Description | Labels | +| ------------------------- | --------- | ----------------------------------------------- | ------ | +| `dotnet_exceptions_total` | `Counter` | Count of exceptions thrown, broken down by type | `type` | + ## `.WithGcStats()` Include metrics recording the frequency and duration of garbage collections\/ pauses, heap sizes and @@ -21,9 +57,9 @@ Include metrics recording the frequency and duration of garbage collections\/ pa | Name | Type | Description | Labels | | ---------------------------------------- | --------- | ---------------------------------------------------------------------------------------------- | --------------- | +| `dotnet_gc_allocated_bytes_total` | `Counter` | The total number of bytes allocated on the managed heap | | | `dotnet_gc_collection_count_total` | `Counter` | Counts the number of garbage collections that have occurred, broken down by generation number. | `gc_generation` | | `dotnet_gc_heap_size_bytes` | `Gauge` | The current size of all heaps (only updated after a garbage collection) | `gc_generation` | -| `dotnet_gc_allocated_bytes_total` | `Counter` | The total number of bytes allocated on the managed heap | | | `dotnet_gc_memory_total_available_bytes` | `Gauge` | The upper limit on the amount of physical memory .NET can allocate to | | | `dotnet_gc_pause_ratio` | `Gauge` | The percentage of time the process spent paused for garbage collection | | @@ -33,11 +69,11 @@ Includes metrics generated by `CaptureLevel.Counters` plus: | Name | Type | Description | Labels | | ------------------------------------- | ----------- | -------------------------------------------------------------------------------------------------------------------------------- | ---------------------------- | +| `dotnet_gc_collection_count_total` | `Counter` | Counts the number of garbage collections that have occurred, broken down by generation number and the reason for the collection. | `gc_generation`, `gc_reason` | +| `dotnet_gc_collection_seconds` | `Histogram` | The amount of time spent running garbage collections | `gc_generation`, `gc_type` | | `dotnet_gc_cpu_ratio` | `Gauge` | The percentage of process CPU time spent running garbage collections | | | `dotnet_gc_finalization_queue_length` | `Gauge` | The number of objects waiting to be finalized | | -| `dotnet_gc_collection_seconds` | `Histogram` | The amount of time spent running garbage collections | `gc_generation`, `gc_type` | | `dotnet_gc_pause_seconds` | `Histogram` | The amount of time execution was paused for garbage collection | | -| `dotnet_gc_collection_count_total` | `Counter` | Counts the number of garbage collections that have occurred, broken down by generation number and the reason for the collection. | `gc_generation`, `gc_reason` | | `dotnet_gc_pinned_objects` | `Gauge` | The number of pinned objects | | ### `CaptureLevel.Verbose` @@ -48,47 +84,6 @@ Includes metrics generated by `CaptureLevel.Counters`, `CaptureLevel.Information | --------------------------------- | --------- | ------------------------------------------------------- | --------- | | `dotnet_gc_allocated_bytes_total` | `Counter` | The total number of bytes allocated on the managed heap | `gc_heap` | -## `.WithContentionStats()` - -Include metrics around volume of locks contended. - -### `CaptureLevel.Counters` - -| Name | Type | Description | Labels | -| ------------------------- | --------- | ----------------------------- | ------ | -| `dotnet_contention_total` | `Counter` | The number of locks contended | | - -### `CaptureLevel.Informational` - -Includes metrics generated by `CaptureLevel.Counters` plus: - -| Name | Type | Description | Labels | -| --------------------------------- | --------- | ----------------------------------------------- | ------ | -| `dotnet_contention_seconds_total` | `Counter` | The total amount of time spent contending locks | | - -## `.WithThreadPoolStats()` - -Include metrics around the size of the worker and IO thread pools and reasons - for worker thread pool changes. - -### `CaptureLevel.Counters` - -| Name | Type | Description | Labels | -| ------------------------------------ | ----------- | ----------------------------------------------------------------------------------------------------------------------------- | ------ | -| `dotnet_threadpool_queue_length` | `Histogram` | Measures the queue length of the thread pool. Values greater than 0 indicate a backlog of work for the threadpool to process. | | -| `dotnet_threadpool_num_threads` | `Gauge` | The number of active threads in the thread pool | | -| `dotnet_threadpool_throughput_total` | `Counter` | The total number of work items that have finished execution in the thread pool | | -| `dotnet_threadpool_timer_count` | `Gauge` | The number of timers active | | - -### `CaptureLevel.Informational` - -Includes metrics generated by `CaptureLevel.Counters` plus: - -| Name | Type | Description | Labels | -| ------------------------------------- | --------- | ------------------------------------------------------------------------------------------------- | ------------------- | -| `dotnet_threadpool_io_num_threads` | `Gauge` | The number of active threads in the IO thread pool | | -| `dotnet_threadpool_adjustments_total` | `Counter` | The total number of changes made to the size of the thread pool, labeled by the reason for change | `adjustment_reason` | - ## `.WithJitStats()` Include metrics summarizing the volume of methods being compiled @@ -107,27 +102,20 @@ Includes metrics generated by `CaptureLevel.Counters` plus: | Name | Type | Description | Labels | | --------------------------------- | --------- | ------------------------------------------------------------------------------------------------- | --------- | +| `dotnet_jit_cpu_ratio` | `Gauge` | The amount of total CPU time consumed spent JIT'ing | | | `dotnet_jit_method_seconds_total` | `Counter` | Total number of seconds spent in the JIT compiler, broken down by compilation for dynamic code | `dynamic` | | `dotnet_jit_method_total` | `Counter` | Total number of methods compiled by the JIT compiler, broken down by compilation for dynamic code | `dynamic` | -| `dotnet_jit_cpu_ratio` | `Gauge` | The amount of total CPU time consumed spent JIT'ing | | -## `.WithExceptionStats()` +## `.WithNameResolution()` -Include metrics that measure the number of exceptions thrown. +Include metrics around DNS lookup requests and average duration. ### `CaptureLevel.Counters` -| Name | Type | Description | Labels | -| ------------------------- | --------- | -------------------------- | ------ | -| `dotnet_exceptions_total` | `Counter` | Count of exceptions thrown | | - -### `CaptureLevel.Errors` - -Includes metrics generated by `CaptureLevel.Counters` plus: - -| Name | Type | Description | Labels | -| ------------------------- | --------- | ----------------------------------------------- | ------ | -| `dotnet_exceptions_total` | `Counter` | Count of exceptions thrown, broken down by type | `type` | +| Name | Type | Description | Labels | +| ---------------------------------------- | ----------- | ------------------------------------------------------------- | ------ | +| `dotnet_dns_lookup_duration_avg_seconds` | `Histogram` | The average time taken for a DNS lookup | | +| `dotnet_dns_lookups_total` | `Counter` | The number of DNS lookups requested since the process started | | ## `.WithSocketStats()` @@ -137,7 +125,30 @@ Include metrics around established TCP connections and the volume of bytes sent\ | Name | Type | Description | Labels | | ------------------------------------------------------- | --------- | -------------------------------------------------------- | ------ | -| `dotnet_sockets_connections_established_incoming_total` | `Counter` | The total number of incoming established TCP connections | | -| `dotnet_sockets_connections_established_outgoing_total` | `Counter` | The total number of outgoing established TCP connections | | | `dotnet_sockets_bytes_received_total` | `Counter` | The total number of bytes received over the network | | | `dotnet_sockets_bytes_sent_total` | `Counter` | The total number of bytes sent over the network | | +| `dotnet_sockets_connections_established_incoming_total` | `Counter` | The total number of incoming established TCP connections | | +| `dotnet_sockets_connections_established_outgoing_total` | `Counter` | The total number of outgoing established TCP connections | | + +## `.WithThreadPoolStats()` + +Include metrics around the size of the worker and IO thread pools and reasons + for worker thread pool changes. + +### `CaptureLevel.Counters` + +| Name | Type | Description | Labels | +| ------------------------------------ | ----------- | ----------------------------------------------------------------------------------------------------------------------------- | ------ | +| `dotnet_threadpool_num_threads` | `Gauge` | The number of active threads in the thread pool | | +| `dotnet_threadpool_queue_length` | `Histogram` | Measures the queue length of the thread pool. Values greater than 0 indicate a backlog of work for the threadpool to process. | | +| `dotnet_threadpool_throughput_total` | `Counter` | The total number of work items that have finished execution in the thread pool | | +| `dotnet_threadpool_timer_count` | `Gauge` | The number of timers active | | + +### `CaptureLevel.Informational` + +Includes metrics generated by `CaptureLevel.Counters` plus: + +| Name | Type | Description | Labels | +| ------------------------------------- | --------- | ------------------------------------------------------------------------------------------------- | ------------------- | +| `dotnet_threadpool_adjustments_total` | `Counter` | The total number of changes made to the size of the thread pool, labeled by the reason for change | `adjustment_reason` | +| `dotnet_threadpool_io_num_threads` | `Gauge` | The number of active threads in the IO thread pool | | From 2592da5102fa0ff8e2b4321b9d4627c83168db11 Mon Sep 17 00:00:00 2001 From: Saul Rennison Date: Thu, 5 May 2022 12:56:51 +0100 Subject: [PATCH 3/5] Add HistogramBuckets options for `WithNameResolution` --- .../DotNetRuntimeStatsBuilder.cs | 9 ++++++++- .../Producers/NameResolutionMetricProducer.cs | 17 +++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs b/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs index 0855645..59c46ab 100644 --- a/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs +++ b/src/prometheus-net.DotNetRuntime/DotNetRuntimeStatsBuilder.cs @@ -196,12 +196,19 @@ public Builder WithSocketStats() /// /// Include metrics around DNS lookup requests and average duration. /// + /// Buckets for the DNS lookup duration /// - public Builder WithNameResolution() + public Builder WithNameResolution(double[] histogramBuckets = null) { ListenerRegistrations.AddOrReplace(ListenerRegistration.Create(CaptureLevel.Counters, sp => new NameResolutionEventParser())); _services.TryAddSingletonEnumerable(); + var opts = new NameResolutionMetricProducer.Options(); + if (histogramBuckets != null) + opts.HistogramBuckets = histogramBuckets; + + _services.AddSingleton(opts); + return this; } diff --git a/src/prometheus-net.DotNetRuntime/Metrics/Producers/NameResolutionMetricProducer.cs b/src/prometheus-net.DotNetRuntime/Metrics/Producers/NameResolutionMetricProducer.cs index 1a7ba04..b1a44fc 100644 --- a/src/prometheus-net.DotNetRuntime/Metrics/Producers/NameResolutionMetricProducer.cs +++ b/src/prometheus-net.DotNetRuntime/Metrics/Producers/NameResolutionMetricProducer.cs @@ -1,14 +1,17 @@ using Prometheus.DotNetRuntime.EventListening.Parsers; +using Prometheus.DotNetRuntime.Metrics.Producers.Util; namespace Prometheus.DotNetRuntime.Metrics.Producers { public class NameResolutionMetricProducer : IMetricProducer { + private readonly Options _options; private readonly Consumes _nameResolutionCounter; - public NameResolutionMetricProducer(Consumes nameResolutionCounter) + public NameResolutionMetricProducer(Options options, Consumes nameResolutionCounter) { _nameResolutionCounter = nameResolutionCounter; + _options = options; } public void RegisterMetrics(MetricFactory metrics) @@ -24,7 +27,12 @@ public void RegisterMetrics(MetricFactory metrics) lastLookups = e.Mean; }; - DnsLookupDuration = metrics.CreateHistogram("dotnet_dns_lookup_duration_avg_seconds", "The average time taken for a DNS lookup"); + DnsLookupDuration = metrics.CreateHistogram("dotnet_dns_lookup_duration_avg_seconds", + "The average time taken for a DNS lookup", + new HistogramConfiguration + { + Buckets = _options.HistogramBuckets + }); _nameResolutionCounter.Events.DnsLookupsDuration += e => { // Convert milliseconds to seconds @@ -38,5 +46,10 @@ public void RegisterMetrics(MetricFactory metrics) public void UpdateMetrics() { } + + public class Options + { + public double[] HistogramBuckets { get; set; } = Constants.DefaultHistogramBuckets; + } } } From 8b752095098b23d6e3ae46f9fdbc847cf3d1d273 Mon Sep 17 00:00:00 2001 From: Saul Rennison Date: Thu, 5 May 2022 12:57:04 +0100 Subject: [PATCH 4/5] Fixed DocsGenerator --- tools/DocsGenerator/Program.cs | 59 +++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/tools/DocsGenerator/Program.cs b/tools/DocsGenerator/Program.cs index e176348..6683747 100644 --- a/tools/DocsGenerator/Program.cs +++ b/tools/DocsGenerator/Program.cs @@ -4,6 +4,8 @@ using System.Collections.Immutable; using System.Linq; using System.Linq.Expressions; +using System.Net; +using System.Net.Http; using System.Reflection; using System.Text.RegularExpressions; using System.Threading; @@ -19,16 +21,25 @@ namespace DocsGenerator { class Program { + static void DoHttpRequest() + { + // Do a HTTP request to trigger DNS requests + using var client = new HttpClient(); + client.GetAsync("https://httpstat.us/200").Wait(); + } + static async Task Main(string[] args) { + DoHttpRequest(); + // TODO output different path depending on runtime version var sources = new [] { SourceAndConfig.CreateFrom(b => b.WithThreadPoolStats(CaptureLevel.Counters, new ThreadPoolMetricsProducer.Options())), SourceAndConfig.CreateFrom(b => b.WithThreadPoolStats(CaptureLevel.Informational, new ThreadPoolMetricsProducer.Options())), - SourceAndConfig.CreateFrom(b => b.WithGcStats(CaptureLevel.Counters, new double[0])), - SourceAndConfig.CreateFrom(b => b.WithGcStats(CaptureLevel.Informational, new double[0])), - SourceAndConfig.CreateFrom(b => b.WithGcStats(CaptureLevel.Verbose, new double[0])), + SourceAndConfig.CreateFrom(b => b.WithGcStats(CaptureLevel.Counters, null)), + SourceAndConfig.CreateFrom(b => b.WithGcStats(CaptureLevel.Informational, null)), + SourceAndConfig.CreateFrom(b => b.WithGcStats(CaptureLevel.Verbose, null)), SourceAndConfig.CreateFrom(b => b.WithContentionStats(CaptureLevel.Counters, SampleEvery.OneEvent)), SourceAndConfig.CreateFrom(b => b.WithContentionStats(CaptureLevel.Informational, SampleEvery.OneEvent)), SourceAndConfig.CreateFrom(b => b.WithExceptionStats(CaptureLevel.Counters)), @@ -37,7 +48,7 @@ static async Task Main(string[] args) SourceAndConfig.CreateFrom(b => b.WithJitStats(CaptureLevel.Verbose, SampleEvery.OneEvent)), SourceAndConfig.CreateFrom(b => b.WithExceptionStats(CaptureLevel.Errors)), SourceAndConfig.CreateFrom(b => b.WithSocketStats()), - SourceAndConfig.CreateFrom(b => b.WithNameResolution()) + SourceAndConfig.CreateFrom(b => b.WithNameResolution(null)) }; var assemblyDocs = typeof(DotNetRuntimeStatsBuilder).Assembly.LoadXmlDocumentation(); @@ -66,23 +77,30 @@ MdSpan[] GetCells(Collector m) root.Add(new MdParagraph("Metrics that are included by default, regardless of what stats collectors are enabled.")); root.Add(new MdTable(headerRow: new MdTableRow("Name", "Type", "Description", "Labels"), - allMetrics.CommonMetrics.Select(x => new MdTableRow(GetCells(x))).ToArray())); + allMetrics.CommonMetrics.OrderBy(x => x.Name).Select(x => new MdTableRow(GetCells(x))).ToArray())); - foreach (var methodAndSources in allMetrics.MethodsToSources) + foreach (var methodAndSources in allMetrics.MethodsToSources.OrderBy(x => x.method.Name)) { root.Add(new MdHeading(new MdCodeSpan($".{methodAndSources.method.Name}()"), 2)); + + var nonEmptySources = methodAndSources.sources + .Where(s => allMetrics.SourceToMetrics[s].Count > 0) + .ToList(); + + if (nonEmptySources.Count == 0) + { + root.Add(new MdParagraph("This method does not export any metrics on this version of the framework.")); + continue; + } + root.Add(new MdParagraph(assemblyDocs.GetDocumentation(methodAndSources.method).Summary)); - for (var i = 0; i < methodAndSources.sources.Length; i++) + for (var i = 0; i < nonEmptySources.Count; i++) { - var s = methodAndSources.sources[i]; - if (allMetrics.SourceToMetrics[s].Count == 0) - continue; - + var s = nonEmptySources[i]; root.Add(new MdHeading(new MdCodeSpan($"{nameof(CaptureLevel)}." + s.Level), 3)); - var previousLevels = methodAndSources.sources.Take(i).ToArray(); - + var previousLevels = nonEmptySources.Take(i).ToArray(); if (previousLevels.Length > 0) { root.Add(new MdParagraph( @@ -91,7 +109,7 @@ MdSpan[] GetCells(Collector m) } root.Add(new MdTable(headerRow: new MdTableRow("Name", "Type", "Description", "Labels"), - allMetrics.SourceToMetrics[s].Select(x => new MdTableRow(GetCells(x.Collector))).ToArray())); + allMetrics.SourceToMetrics[s].OrderBy(x => x.Collector.Name).Select(x => new MdTableRow(GetCells(x.Collector))).ToArray())); } } @@ -147,17 +165,20 @@ private static GroupedMetrics GetAllMetrics(SourceAndConfig[] sources) private static IEnumerable GetExposedMetric(SourceAndConfig source) { - Console.WriteLine($"Getting metrics for {source}.."); - + Console.WriteLine($"Getting metrics for {source.Source}.."); + // Start collector var registry = new CollectorRegistry(); - using var statsCollector = source.ApplyConfig(DotNetRuntimeStatsBuilder.Customize()).StartCollecting(registry); - // Wait for metrics to be available (hacky!) - Thread.Sleep(1500); + using var statsCollector = source.ApplyConfig(DotNetRuntimeStatsBuilder.Customize()).StartCollecting(registry) as DotNetRuntimeStatsCollector; // Pull registered collectors var collectors = registry.TryGetFieldValue("_collectors", Flags.InstancePrivate) as ConcurrentDictionary; + // Wait for all listeners to be ready + SpinWait.SpinUntil( + () => statsCollector.EventListeners.All(x => x.StartedReceivingEvents), + TimeSpan.FromSeconds(5)); + return collectors.Values.Select(c => new ExposedMetric(c, source.Source)); } From 0d486def67ba1e708b9215132479ed87cecf0844 Mon Sep 17 00:00:00 2001 From: Saul Rennison Date: Thu, 5 May 2022 13:59:53 +0100 Subject: [PATCH 5/5] Inline DoHttpRequest --- tools/DocsGenerator/Program.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tools/DocsGenerator/Program.cs b/tools/DocsGenerator/Program.cs index 6683747..8dd009f 100644 --- a/tools/DocsGenerator/Program.cs +++ b/tools/DocsGenerator/Program.cs @@ -21,16 +21,11 @@ namespace DocsGenerator { class Program { - static void DoHttpRequest() + static async Task Main(string[] args) { - // Do a HTTP request to trigger DNS requests + // Do a HTTP request to trigger DNS request + open socket using var client = new HttpClient(); client.GetAsync("https://httpstat.us/200").Wait(); - } - - static async Task Main(string[] args) - { - DoHttpRequest(); // TODO output different path depending on runtime version var sources = new []