Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add logging buffering #5635

Draft
wants to merge 29 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
988c709
Buffering
evgenyfedorov2 Nov 13, 2024
2f1a335
Major update
evgenyfedorov2 Dec 7, 2024
2d2412e
Remove Json exception converter
evgenyfedorov2 Dec 12, 2024
f7eaab1
Fix namespaces
evgenyfedorov2 Dec 16, 2024
1f464df
Merge branch 'main' into evgenyfedorov2/log_buffering
evgenyfedorov2 Dec 16, 2024
a371d9c
Fix build
evgenyfedorov2 Dec 16, 2024
d7661a6
Slight design changes with interfaces as per PR comments
evgenyfedorov2 Jan 8, 2025
9d13ab0
Drop json serialization
evgenyfedorov2 Jan 9, 2025
fe00658
Add log record size estimation and limit buffer size in bytes
evgenyfedorov2 Jan 13, 2025
5fc421c
Add filtering by attributes
evgenyfedorov2 Jan 15, 2025
70cfc7c
Use attributes directly instead of Func delegate
evgenyfedorov2 Jan 21, 2025
e96277f
Add http buffer holder
evgenyfedorov2 Jan 21, 2025
a79fcbf
Make ILoggingBuffer and DeserializedLogRecord types internal
evgenyfedorov2 Jan 22, 2025
8a91c15
Move shared files to Shared project and add more tests
evgenyfedorov2 Jan 22, 2025
4f524eb
Add custom equality comparer
evgenyfedorov2 Jan 22, 2025
b2b6e56
Address API Review feedback
evgenyfedorov2 Feb 14, 2025
f3a6b85
Merge branch 'main' into evgenyfedorov2/log_buffering
evgenyfedorov2 Feb 14, 2025
1370225
merge
evgenyfedorov2 Feb 28, 2025
393ce26
API review feedback
evgenyfedorov2 Feb 28, 2025
20b5a4c
Remove extra lines
evgenyfedorov2 Feb 28, 2025
f5ce71e
Make tests culture agnostic
Mar 11, 2025
ce2f7fb
Global log buffering - options validation, rule selector optimization
Mar 11, 2025
6ffbacd
Rename to PerIncomingRequest
evgenyfedorov2 Mar 13, 2025
1370ccb
Minor updates
Mar 14, 2025
56257ca
More renames
Mar 14, 2025
73a8678
Rename shared folder LoggingBuffering to LogBuffering
Mar 14, 2025
8a42385
Remove per request options refresh because buffer are scoped anyway
Mar 14, 2025
7ff16fc
Remove unnecessary casting
Mar 14, 2025
35223f3
Add DebuggerDisplay to SerializedLogRecord.cs
Mar 15, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions eng/MSBuild/Shared.props
Original file line number Diff line number Diff line change
Expand Up @@ -46,4 +46,8 @@
<ItemGroup Condition="'$(InjectStringSplitExtensions)' == 'true'">
<Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\StringSplit\*.cs" LinkBase="Shared\StringSplit" />
</ItemGroup>

<ItemGroup Condition="'$(InjectSharedLoggingBuffering)' == 'true'">
<Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\LoggingBuffering\*.cs" LinkBase="Shared\LoggingBuffering" />
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Threading;
using Microsoft.Extensions.Diagnostics.Buffering;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Microsoft.Shared.Diagnostics;
using static Microsoft.Extensions.Logging.ExtendedLogger;

namespace Microsoft.AspNetCore.Diagnostics.Buffering;

internal sealed class HttpRequestBuffer : ILoggingBuffer
{
private readonly IOptionsMonitor<HttpRequestBufferOptions> _options;
private readonly IOptionsMonitor<GlobalBufferOptions> _globalOptions;
private readonly ConcurrentQueue<SerializedLogRecord> _buffer;
private readonly TimeProvider _timeProvider = TimeProvider.System;
private readonly IBufferedLogger _bufferedLogger;

private DateTimeOffset _lastFlushTimestamp;
private int _bufferSize;

public HttpRequestBuffer(IBufferedLogger bufferedLogger,
IOptionsMonitor<HttpRequestBufferOptions> options,
IOptionsMonitor<GlobalBufferOptions> globalOptions)
{
_options = options;
_globalOptions = globalOptions;
_bufferedLogger = bufferedLogger;
_buffer = new ConcurrentQueue<SerializedLogRecord>();
}

public bool TryEnqueue<TState>(
LogLevel logLevel,
string category,
EventId eventId,
TState state,
Exception? exception,
Func<TState, Exception?, string> formatter)
{
SerializedLogRecord serializedLogRecord = default;
if (state is ModernTagJoiner modernTagJoiner)
{
if (!IsEnabled(category, logLevel, eventId, modernTagJoiner))
{
return false;
}

serializedLogRecord = new SerializedLogRecord(logLevel, eventId, _timeProvider.GetUtcNow(), modernTagJoiner, exception,
((Func<ModernTagJoiner, Exception?, string>)(object)formatter)(modernTagJoiner, exception));
}
else if (state is LegacyTagJoiner legacyTagJoiner)
{
if (!IsEnabled(category, logLevel, eventId, legacyTagJoiner))
{
return false;
}

serializedLogRecord = new SerializedLogRecord(logLevel, eventId, _timeProvider.GetUtcNow(), legacyTagJoiner, exception,
((Func<LegacyTagJoiner, Exception?, string>)(object)formatter)(legacyTagJoiner, exception));
}
else
{
Throw.ArgumentException(nameof(state), $"Unsupported type of the log state object detected: {typeof(TState)}");
}

if (serializedLogRecord.SizeInBytes > _globalOptions.CurrentValue.MaxLogRecordSizeInBytes)
{
return false;
}

_buffer.Enqueue(serializedLogRecord);
_ = Interlocked.Add(ref _bufferSize, serializedLogRecord.SizeInBytes);

Trim();

return true;
}

public void Flush()
{
_lastFlushTimestamp = _timeProvider.GetUtcNow();

SerializedLogRecord[] bufferedRecords = _buffer.ToArray();

_buffer.Clear();

var deserializedLogRecords = new List<DeserializedLogRecord>(bufferedRecords.Length);
foreach (var bufferedRecord in bufferedRecords)
{
deserializedLogRecords.Add(
new DeserializedLogRecord(
bufferedRecord.Timestamp,
bufferedRecord.LogLevel,
bufferedRecord.EventId,
bufferedRecord.Exception,
bufferedRecord.FormattedMessage,
bufferedRecord.Attributes));
}

_bufferedLogger.LogRecords(deserializedLogRecords);
}

public bool IsEnabled(string category, LogLevel logLevel, EventId eventId, IReadOnlyList<KeyValuePair<string, object?>> attributes)
{
if (_timeProvider.GetUtcNow() < _lastFlushTimestamp + _globalOptions.CurrentValue.SuspendAfterFlushDuration)
{
return false;
}

BufferFilterRuleSelector.Select(_options.CurrentValue.Rules, category, logLevel, eventId, attributes, out BufferFilterRule? rule);

return rule is not null;
}

private void Trim()
{
while (_bufferSize > _options.CurrentValue.PerRequestBufferSizeInBytes && _buffer.TryDequeue(out var item))
{
_ = Interlocked.Add(ref _bufferSize, -item.SizeInBytes);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Collections.Generic;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Options;

namespace Microsoft.AspNetCore.Diagnostics.Buffering;

internal sealed class HttpRequestBufferConfigureOptions : IConfigureOptions<HttpRequestBufferOptions>
{
private const string BufferingKey = "Buffering";
private readonly IConfiguration _configuration;

public HttpRequestBufferConfigureOptions(IConfiguration configuration)
{
_configuration = configuration;
}

public void Configure(HttpRequestBufferOptions options)
{
if (_configuration == null)
{
return;
}

var section = _configuration.GetSection(BufferingKey);
if (!section.Exists())
{
return;
}

var parsedOptions = section.Get<HttpRequestBufferOptions>();
if (parsedOptions is null)
{
return;
}

options.Rules.AddRange(parsedOptions.Rules);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Concurrent;
using Microsoft.Extensions.Diagnostics.Buffering;

namespace Microsoft.AspNetCore.Diagnostics.Buffering;

internal sealed class HttpRequestBufferHolder
{
private readonly ConcurrentDictionary<string, ILoggingBuffer> _buffers = new();

public ILoggingBuffer GetOrAdd(string category, Func<string, ILoggingBuffer> valueFactory) =>
_buffers.GetOrAdd(category, valueFactory);

public void Flush()
{
foreach (var buffer in _buffers.Values)
{
buffer.Flush();
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Diagnostics.CodeAnalysis;
using Microsoft.AspNetCore.Diagnostics.Buffering;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Diagnostics.Buffering;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.Shared.DiagnosticIds;
using Microsoft.Shared.Diagnostics;

namespace Microsoft.Extensions.Logging;

/// <summary>
/// Lets you register log buffers in a dependency injection container.
/// </summary>
[Experimental(diagnosticId: DiagnosticIds.Experiments.Telemetry, UrlFormat = DiagnosticIds.UrlFormat)]
public static class HttpRequestBufferLoggerBuilderExtensions
{
/// <summary>
/// Adds HTTP request-aware buffer to the logging infrastructure. Matched logs will be buffered in
/// a buffer specific to each HTTP request and can optionally be flushed and emitted during the request lifetime./>.
/// </summary>
/// <param name="builder">The <see cref="ILoggingBuilder" />.</param>
/// <param name="configuration">The <see cref="IConfiguration" /> to add.</param>
/// <returns>The value of <paramref name="builder"/>.</returns>
/// <exception cref="ArgumentNullException"><paramref name="builder"/> is <see langword="null"/>.</exception>
public static ILoggingBuilder AddHttpRequestBuffering(this ILoggingBuilder builder, IConfiguration configuration)
{
_ = Throw.IfNull(builder);
_ = Throw.IfNull(configuration);

return builder
.AddHttpRequestBufferConfiguration(configuration)
.AddHttpRequestBufferManager()
.AddGlobalBuffer(configuration);
}

/// <summary>
/// Adds HTTP request-aware buffering to the logging infrastructure. Matched logs will be buffered in
/// a buffer specific to each HTTP request and can optionally be flushed and emitted during the request lifetime./>.
/// </summary>
/// <param name="builder">The <see cref="ILoggingBuilder" />.</param>
/// <param name="level">The log level (and below) to apply the buffer to.</param>
/// <param name="configure">The buffer configuration options.</param>
/// <returns>The value of <paramref name="builder"/>.</returns>
/// <exception cref="ArgumentNullException"><paramref name="builder"/> is <see langword="null"/>.</exception>
public static ILoggingBuilder AddHttpRequestBuffering(this ILoggingBuilder builder, LogLevel? level = null, Action<HttpRequestBufferOptions>? configure = null)
{
_ = Throw.IfNull(builder);

_ = builder.Services
.Configure<HttpRequestBufferOptions>(options => options.Rules.Add(new BufferFilterRule(null, level, null, null)))
.Configure(configure ?? new Action<HttpRequestBufferOptions>(_ => { }));

return builder
.AddHttpRequestBufferManager()
.AddGlobalBuffer(level);
}

/// <summary>
/// Adds HTTP request buffer provider to the logging infrastructure.
/// </summary>
/// <param name="builder">The <see cref="ILoggingBuilder" />.</param>
/// <returns>The <see cref="ILoggingBuilder"/> so that additional calls can be chained.</returns>
/// <exception cref="ArgumentNullException"><paramref name="builder"/> is <see langword="null"/>.</exception>
internal static ILoggingBuilder AddHttpRequestBufferManager(this ILoggingBuilder builder)
{
_ = Throw.IfNull(builder);

builder.Services.TryAddScoped<HttpRequestBufferHolder>();
builder.Services.TryAddSingleton<IHttpContextAccessor, HttpContextAccessor>();
builder.Services.TryAddSingleton<HttpRequestBufferManager>();
builder.Services.TryAddSingleton<IBufferManager>(static sp => sp.GetRequiredService<HttpRequestBufferManager>());
builder.Services.TryAddSingleton<IHttpRequestBufferManager>(static sp => sp.GetRequiredService<HttpRequestBufferManager>());

return builder;
}

/// <summary>
/// Configures <see cref="HttpRequestBufferOptions" /> from an instance of <see cref="IConfiguration" />.
/// </summary>
/// <param name="builder">The <see cref="ILoggingBuilder" />.</param>
/// <param name="configuration">The <see cref="IConfiguration" /> to add.</param>
/// <returns>The value of <paramref name="builder"/>.</returns>
/// <exception cref="ArgumentNullException"><paramref name="builder"/> is <see langword="null"/>.</exception>
internal static ILoggingBuilder AddHttpRequestBufferConfiguration(this ILoggingBuilder builder, IConfiguration configuration)
{
_ = Throw.IfNull(builder);

_ = builder.Services.AddSingleton<IConfigureOptions<HttpRequestBufferOptions>>(new HttpRequestBufferConfigureOptions(configuration));

return builder;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.Buffering;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;

namespace Microsoft.AspNetCore.Diagnostics.Buffering;

internal sealed class HttpRequestBufferManager : IHttpRequestBufferManager
{
private readonly IGlobalBufferManager _globalBufferManager;
private readonly IHttpContextAccessor _httpContextAccessor;
private readonly IOptionsMonitor<HttpRequestBufferOptions> _requestOptions;
private readonly IOptionsMonitor<GlobalBufferOptions> _globalOptions;

public HttpRequestBufferManager(
IGlobalBufferManager globalBufferManager,
IHttpContextAccessor httpContextAccessor,
IOptionsMonitor<HttpRequestBufferOptions> requestOptions,
IOptionsMonitor<GlobalBufferOptions> globalOptions)
{
_globalBufferManager = globalBufferManager;
_httpContextAccessor = httpContextAccessor;
_requestOptions = requestOptions;
_globalOptions = globalOptions;
}

public void FlushNonRequestLogs() => _globalBufferManager.Flush();

public void FlushCurrentRequestLogs()
{
_httpContextAccessor.HttpContext?.RequestServices.GetService<HttpRequestBufferHolder>()?.Flush();
}

public bool TryEnqueue<TState>(
IBufferedLogger bufferedLogger,
LogLevel logLevel,
string category,
EventId eventId,
TState state,
Exception? exception,
Func<TState, Exception?, string> formatter)
{
HttpContext? httpContext = _httpContextAccessor.HttpContext;
if (httpContext is null)
{
return _globalBufferManager.TryEnqueue(bufferedLogger, logLevel, category, eventId, state, exception, formatter);
}

HttpRequestBufferHolder? bufferHolder = httpContext.RequestServices.GetService<HttpRequestBufferHolder>();
ILoggingBuffer? buffer = bufferHolder?.GetOrAdd(category, _ => new HttpRequestBuffer(bufferedLogger, _requestOptions, _globalOptions)!);

if (buffer is null)
{
return _globalBufferManager.TryEnqueue(bufferedLogger, logLevel, category, eventId, state, exception, formatter);
}

return buffer.TryEnqueue(logLevel, category, eventId, state, exception, formatter);
}
}
Loading
Loading