Support Request Context in Ollama and Anthropic generators (#844)

Allow to override the model name during a request, using request context
arguments.

The feature is available only for Anthropic and Ollama. Updated example
212 showing how to override Ollama settings at runtime.

The feature is not supported when using OpenAI/Azure OpenAI, because the
underlying connectors hard code the model name in client instances and
would require a considerable amount of refactoring, plus memory
overhead, to support this feature. See
microsoft/semantic-kernel#9337

Author: dluc

examples/210-KM-without-builder/Program.cs

@@ -6,6 +6,7 @@
 using Microsoft.KernelMemory.AI.AzureOpenAI;
 using Microsoft.KernelMemory.AI.OpenAI;
 using Microsoft.KernelMemory.Configuration;
+using Microsoft.KernelMemory.Context;
 using Microsoft.KernelMemory.DataFormats;
 using Microsoft.KernelMemory.DataFormats.AzureAIDocIntel;
 using Microsoft.KernelMemory.DataFormats.Image;
@@ -73,6 +74,7 @@ public static async Task Main()
         LoggerFactory? loggerFactory = null; // Alternative: app.Services.GetService<ILoggerFactory>();
 
         // Generic dependencies
+        var requestContextProvider = new RequestContextProvider();
         var mimeTypeDetection = new MimeTypesDetection();
         var promptProvider = new EmbeddedPromptProvider();
 
@@ -121,7 +123,7 @@ public static async Task Main()
 
         // Create memory instance
         var searchClient = new SearchClient(memoryDb, textGenerator, searchClientConfig, promptProvider, contentModeration, loggerFactory);
-        var memory = new MemoryServerless(orchestrator, searchClient, kernelMemoryConfig);
+        var memory = new MemoryServerless(orchestrator, searchClient, requestContextProvider, kernelMemoryConfig);
 
         // End-to-end test
         await memory.ImportTextAsync("I'm waiting for Godot", documentId: "tg01");

examples/212-dotnet-ollama/Program.cs

@@ -3,6 +3,7 @@
 using Microsoft.KernelMemory;
 using Microsoft.KernelMemory.AI.Ollama;
 using Microsoft.KernelMemory.AI.OpenAI;
+using Microsoft.KernelMemory.Context;
 using Microsoft.KernelMemory.Diagnostics;
 
 /* This example shows how to use KM with Ollama
@@ -49,19 +50,46 @@ public static async Task Main()
 
         // Generate an answer - This uses OpenAI for embeddings and finding relevant data, and LM Studio to generate an answer
         var answer = await memory.AskAsync("What's the current date (don't check for validity)?");
+        Console.WriteLine("-------------------");
         Console.WriteLine(answer.Question);
         Console.WriteLine(answer.Result);
+        Console.WriteLine("-------------------");
 
         /*
 
         -- Output using phi3:medium-128k:
 
         What's the current date (don't check for validity)?
+
         The given fact states that "Today is October 32nd, 2476." However, it appears to be an incorrect statement as
         there are never more than 31 days in any month. If we consider this date without checking its validity and accept
         the stated day of October as being 32, then the current date would be "October 32nd, 2476." However, it is important
         to note that this date does not align with our calendar system.
 
         */
+
+        // How to override config with Request Context
+        var context = new RequestContext();
+        context.SetArg("custom_text_generation_model_name", "llama2:70b");
+        // context.SetArg("custom_embedding_generation_model_name", "...");
+
+        answer = await memory.AskAsync("What's the current date (don't check for validity)?", context: context);
+        Console.WriteLine("-------------------");
+        Console.WriteLine(answer.Question);
+        Console.WriteLine(answer.Result);
+        Console.WriteLine("-------------------");
+
+        /*
+
+        -- Output using llama2:70b:
+
+        What's the current date (don't check for validity)?
+
+        The provided facts state that "Today is October 32nd, 2476." However, considering the Gregorian calendar system
+        commonly used today, this information appears to be incorrect as there are no such dates. This could
+        potentially refer to a different calendar or timekeeping system in use in your fictional world, but based on our
+        current understanding of calendars and dates, an "October 32nd" does not exist. Therefore, the answer is
+        'INFO NOT FOUND'.
+        */
     }
 }

extensions/Anthropic/AnthropicTextGeneration.cs

@@ -8,6 +8,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.KernelMemory.AI.Anthropic.Client;
 using Microsoft.KernelMemory.AI.OpenAI;
+using Microsoft.KernelMemory.Context;
 using Microsoft.KernelMemory.Diagnostics;
 
 namespace Microsoft.KernelMemory.AI.Anthropic;
@@ -23,6 +24,7 @@ public sealed class AnthropicTextGeneration : ITextGenerator, IDisposable
 
     private readonly RawAnthropicClient _client;
     private readonly ITextTokenizer _textTokenizer;
+    private readonly IContextProvider _contextProvider;
     private readonly HttpClient _httpClient;
     private readonly ILogger<AnthropicTextGeneration> _log;
     private readonly string _modelName;
@@ -34,11 +36,13 @@ public sealed class AnthropicTextGeneration : ITextGenerator, IDisposable
     /// <param name="config">Client configuration, including credentials and model details</param>
     /// <param name="textTokenizer">Tokenizer used to count tokens</param>
     /// <param name="httpClientFactory">Optional factory used to inject a pre-configured HTTP client for requests to Anthropic API</param>
+    /// <param name="contextProvider">Request context provider with runtime configuration overrides</param>
     /// <param name="loggerFactory">Optional factory used to inject configured loggers</param>
     public AnthropicTextGeneration(
         AnthropicConfig config,
         ITextTokenizer? textTokenizer = null,
         IHttpClientFactory? httpClientFactory = null,
+        IContextProvider? contextProvider = null,
         ILoggerFactory? loggerFactory = null)
     {
         this._modelName = config.TextModelName;
@@ -48,6 +52,7 @@ public AnthropicTextGeneration(
         this.MaxTokenTotal = config.MaxTokenOut;
 
         this._log = (loggerFactory ?? DefaultLogger.Factory).CreateLogger<AnthropicTextGeneration>();
+        this._contextProvider = contextProvider ?? new RequestContextProvider();
 
         if (httpClientFactory == null)
         {
@@ -96,9 +101,11 @@ public async IAsyncEnumerable<string> GenerateTextAsync(
         TextGenerationOptions options,
         [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
-        this._log.LogTrace("Sending text generation request, model '{0}'", this._modelName);
+        string modelName = this._contextProvider.GetContext().GetCustomTextGenerationModelNameOrDefault(this._modelName);
 
-        CallClaudeStreamingParams parameters = new(this._modelName, prompt)
+        this._log.LogTrace("Sending text generation request, model '{0}'", modelName);
+
+        CallClaudeStreamingParams parameters = new(modelName, prompt)
         {
             System = this._defaultSystemPrompt,
             Temperature = options.Temperature,

extensions/AzureOpenAI/AzureOpenAITextEmbeddingGenerator.cs

@@ -17,6 +17,12 @@
 
 namespace Microsoft.KernelMemory.AI.AzureOpenAI;
 
+/// <summary>
+/// Azure OpenAI connector
+///
+/// Note: does not support model name override via request context
+///       see https://github.com/microsoft/semantic-kernel/issues/9337
+/// </summary>
 [Experimental("KMEXP01")]
 public sealed class AzureOpenAITextEmbeddingGenerator : ITextEmbeddingGenerator, ITextEmbeddingBatchGenerator
 {

extensions/AzureOpenAI/AzureOpenAITextGenerator.cs

@@ -15,6 +15,12 @@
 
 namespace Microsoft.KernelMemory.AI.AzureOpenAI;
 
+/// <summary>
+/// Azure OpenAI connector
+///
+/// Note: does not support model name override via request context
+///       see https://github.com/microsoft/semantic-kernel/issues/9337
+/// </summary>
 [Experimental("KMEXP01")]
 public sealed class AzureOpenAITextGenerator : ITextGenerator
 {

extensions/ONNX/Onnx/OnnxTextGenerator.cs

@@ -19,6 +19,8 @@ namespace Microsoft.KernelMemory.AI.Onnx;
 /// <summary>
 /// Text generator based on ONNX models, via OnnxRuntimeGenAi
 /// See https://github.com/microsoft/onnxruntime-genai
+///
+/// Note: does not support model name override via request context
 /// </summary>
 [Experimental("KMEXP01")]
 public sealed class OnnxTextGenerator : ITextGenerator, IDisposable

extensions/Ollama/Ollama/DependencyInjection.cs

@@ -5,6 +5,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.KernelMemory.AI;
 using Microsoft.KernelMemory.AI.Ollama;
+using Microsoft.KernelMemory.Context;
 using OllamaSharp;
 
 #pragma warning disable IDE0130 // reduce number of "using" statements
@@ -72,6 +73,7 @@ public static IServiceCollection AddOllamaTextGeneration(
                     new OllamaApiClient(new Uri(endpoint), modelName),
                     new OllamaModelConfig { ModelName = modelName },
                     textTokenizer,
+                    serviceProvider.GetService<IContextProvider>(),
                     serviceProvider.GetService<ILoggerFactory>()));
     }
 
@@ -86,6 +88,7 @@ public static IServiceCollection AddOllamaTextGeneration(
                     new OllamaApiClient(new Uri(config.Endpoint), config.TextModel.ModelName),
                     config.TextModel,
                     textTokenizer,
+                    serviceProvider.GetService<IContextProvider>(),
                     serviceProvider.GetService<ILoggerFactory>()));
     }
 
@@ -101,6 +104,7 @@ public static IServiceCollection AddOllamaTextEmbeddingGeneration(
                     new OllamaApiClient(new Uri(endpoint), modelName),
                     new OllamaModelConfig { ModelName = modelName },
                     textTokenizer,
+                    serviceProvider.GetService<IContextProvider>(),
                     serviceProvider.GetService<ILoggerFactory>()));
     }
 
@@ -115,6 +119,7 @@ public static IServiceCollection AddOllamaTextEmbeddingGeneration(
                     new OllamaApiClient(new Uri(config.Endpoint), config.EmbeddingModel.ModelName),
                     config.EmbeddingModel,
                     textTokenizer,
+                    serviceProvider.GetService<IContextProvider>(),
                     serviceProvider.GetService<ILoggerFactory>()));
     }
 }

extensions/Ollama/Ollama/OllamaTextEmbeddingGenerator.cs

@@ -8,6 +8,7 @@
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Microsoft.KernelMemory.AI.OpenAI;
+using Microsoft.KernelMemory.Context;
 using Microsoft.KernelMemory.Diagnostics;
 using OllamaSharp;
 using OllamaSharp.Models;
@@ -20,8 +21,9 @@ public class OllamaTextEmbeddingGenerator : ITextEmbeddingGenerator, ITextEmbedd
 
     private readonly IOllamaApiClient _client;
     private readonly OllamaModelConfig _modelConfig;
-    private readonly ILogger<OllamaTextEmbeddingGenerator> _log;
     private readonly ITextTokenizer _textTokenizer;
+    private readonly IContextProvider _contextProvider;
+    private readonly ILogger<OllamaTextEmbeddingGenerator> _log;
 
     public int MaxTokens { get; }
 
@@ -31,6 +33,7 @@ public OllamaTextEmbeddingGenerator(
         IOllamaApiClient ollamaClient,
         OllamaModelConfig modelConfig,
         ITextTokenizer? textTokenizer = null,
+        IContextProvider? contextProvider = null,
         ILoggerFactory? loggerFactory = null)
     {
         this._client = ollamaClient;
@@ -47,18 +50,21 @@ public OllamaTextEmbeddingGenerator(
         }
 
         this._textTokenizer = textTokenizer;
+        this._contextProvider = contextProvider ?? new RequestContextProvider();
 
         this.MaxTokens = modelConfig.MaxTokenTotal ?? MaxTokensIfUndefined;
     }
 
     public OllamaTextEmbeddingGenerator(
         OllamaConfig config,
         ITextTokenizer? textTokenizer = null,
+        IContextProvider? contextProvider = null,
         ILoggerFactory? loggerFactory = null)
         : this(
             new OllamaApiClient(new Uri(config.Endpoint), config.EmbeddingModel.ModelName),
             config.EmbeddingModel,
             textTokenizer,
+            contextProvider,
             loggerFactory)
     {
     }
@@ -67,11 +73,13 @@ public OllamaTextEmbeddingGenerator(
         HttpClient httpClient,
         OllamaConfig config,
         ITextTokenizer? textTokenizer = null,
+        IContextProvider? contextProvider = null,
         ILoggerFactory? loggerFactory = null)
         : this(
             new OllamaApiClient(httpClient, config.EmbeddingModel.ModelName),
             config.EmbeddingModel,
             textTokenizer,
+            contextProvider,
             loggerFactory)
     {
     }
@@ -104,11 +112,13 @@ public async Task<Embedding[]> GenerateEmbeddingBatchAsync(
         CancellationToken cancellationToken = default)
     {
         var list = textList.ToList();
-        this._log.LogTrace("Generating embeddings batch, size {0} texts", list.Count);
+
+        string modelName = this._contextProvider.GetContext().GetCustomEmbeddingGenerationModelNameOrDefault(this._client.SelectedModel);
+        this._log.LogTrace("Generating embeddings batch, size {0} texts, with model {1}", list.Count, modelName);
 
         var request = new EmbedRequest
         {
-            Model = this._client.SelectedModel,
+            Model = modelName,
             Input = list,
             Options = new RequestOptions
             {

extensions/Ollama/Ollama/OllamaTextGenerator.cs

@@ -7,6 +7,7 @@
 using System.Threading;
 using Microsoft.Extensions.Logging;
 using Microsoft.KernelMemory.AI.OpenAI;
+using Microsoft.KernelMemory.Context;
 using Microsoft.KernelMemory.Diagnostics;
 using OllamaSharp;
 using OllamaSharp.Models;
@@ -19,15 +20,17 @@ public class OllamaTextGenerator : ITextGenerator
 
     private readonly IOllamaApiClient _client;
     private readonly OllamaModelConfig _modelConfig;
-    private readonly ILogger<OllamaTextGenerator> _log;
     private readonly ITextTokenizer _textTokenizer;
+    private readonly IContextProvider _contextProvider;
+    private readonly ILogger<OllamaTextGenerator> _log;
 
     public int MaxTokenTotal { get; }
 
     public OllamaTextGenerator(
         IOllamaApiClient ollamaClient,
         OllamaModelConfig modelConfig,
         ITextTokenizer? textTokenizer = null,
+        IContextProvider? contextProvider = null,
         ILoggerFactory? loggerFactory = null)
     {
         this._client = ollamaClient;
@@ -43,18 +46,21 @@ public OllamaTextGenerator(
         }
 
         this._textTokenizer = textTokenizer;
+        this._contextProvider = contextProvider ?? new RequestContextProvider();
 
         this.MaxTokenTotal = modelConfig.MaxTokenTotal ?? MaxTokensIfUndefined;
     }
 
     public OllamaTextGenerator(
         OllamaConfig config,
         ITextTokenizer? textTokenizer = null,
+        IContextProvider? contextProvider = null,
         ILoggerFactory? loggerFactory = null)
         : this(
             new OllamaApiClient(new Uri(config.Endpoint), config.TextModel.ModelName),
             config.TextModel,
             textTokenizer,
+            contextProvider,
             loggerFactory)
     {
     }
@@ -63,11 +69,13 @@ public OllamaTextGenerator(
         HttpClient httpClient,
         OllamaConfig config,
         ITextTokenizer? textTokenizer = null,
+        IContextProvider? contextProvider = null,
         ILoggerFactory? loggerFactory = null)
         : this(
             new OllamaApiClient(httpClient, config.TextModel.ModelName),
             config.TextModel,
             textTokenizer,
+            contextProvider,
             loggerFactory)
     {
     }
@@ -87,9 +95,12 @@ public async IAsyncEnumerable<string> GenerateTextAsync(
         TextGenerationOptions options,
         [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
+        string modelName = this._contextProvider.GetContext().GetCustomTextGenerationModelNameOrDefault(this._client.SelectedModel);
+        this._log.LogTrace("Generating text with model {0}", modelName);
+
         var request = new GenerateRequest
         {
-            Model = this._client.SelectedModel,
+            Model = modelName,
             Prompt = prompt,
             Stream = true,
             Options = new RequestOptions

extensions/OpenAI/OpenAI/OpenAITextEmbeddingGenerator.cs

@@ -18,6 +18,9 @@ namespace Microsoft.KernelMemory.AI.OpenAI;
 /// <summary>
 /// Text embedding generator. The class can be used with any service
 /// supporting OpenAI HTTP schema.
+///
+/// Note: does not support model name override via request context
+///       see https://github.com/microsoft/semantic-kernel/issues/9337
 /// </summary>
 [Experimental("KMEXP01")]
 public sealed class OpenAITextEmbeddingGenerator : ITextEmbeddingGenerator, ITextEmbeddingBatchGenerator

extensions/OpenAI/OpenAI/OpenAITextGenerator.cs

@@ -17,6 +17,9 @@ namespace Microsoft.KernelMemory.AI.OpenAI;
 /// <summary>
 /// Text generator, supporting OpenAI text and chat completion. The class can be used with any service
 /// supporting OpenAI HTTP schema, such as LM Studio HTTP API.
+///
+/// Note: does not support model name override via request context
+///       see https://github.com/microsoft/semantic-kernel/issues/9337
 /// </summary>
 [Experimental("KMEXP01")]
 public sealed class OpenAITextGenerator : ITextGenerator