Skip to content

Commit 4721625

Browse files
danield137dmytrostrukshawncal
authored
.Net: Adding Kusto as an external memory (#2257)
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> ### Description Adding Kusto as an external memory. ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄 --------- Co-authored-by: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Co-authored-by: Shawn Callegari <36091529+shawncal@users.noreply.github.com>
1 parent bb163b6 commit 4721625

File tree

14 files changed

+1194
-0
lines changed

14 files changed

+1194
-0
lines changed

‎dotnet/Directory.Packages.props‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
<PackageVersion Include="Azure.Identity" Version="1.9.0" />
1010
<PackageVersion Include="Azure.Search.Documents" Version="11.5.0-beta.3" />
1111
<PackageVersion Include="Microsoft.ApplicationInsights.WorkerService" Version="2.21.0" />
12+
<PackageVersion Include="Microsoft.Azure.Kusto.Data" Version="11.3.2" />
1213
<PackageVersion Include="Microsoft.Bcl.HashCode" Version="[1.1.0, )" />
1314
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="6.0.0" />
1415
<PackageVersion Include="Microsoft.Extensions.Http" Version="6.0.0" />

‎dotnet/SK-dotnet.sln‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Planning.StepwisePlanner",
150150
EndProject
151151
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApplicationInsightsExample", "samples\ApplicationInsightsExample\ApplicationInsightsExample.csproj", "{C754950A-E16C-4F96-9CC7-9328E361B5AF}"
152152
EndProject
153+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.Kusto", "src\Connectors\Connectors.Memory.Kusto\Connectors.Memory.Kusto.csproj", "{E07608CC-D710-4655-BB9E-D22CF3CDD193}"
154+
EndProject
153155
Global
154156
GlobalSection(SolutionConfigurationPlatforms) = preSolution
155157
Debug|Any CPU = Debug|Any CPU
@@ -361,6 +363,12 @@ Global
361363
{C754950A-E16C-4F96-9CC7-9328E361B5AF}.Publish|Any CPU.ActiveCfg = Release|Any CPU
362364
{C754950A-E16C-4F96-9CC7-9328E361B5AF}.Release|Any CPU.ActiveCfg = Release|Any CPU
363365
{C754950A-E16C-4F96-9CC7-9328E361B5AF}.Release|Any CPU.Build.0 = Release|Any CPU
366+
{E07608CC-D710-4655-BB9E-D22CF3CDD193}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
367+
{E07608CC-D710-4655-BB9E-D22CF3CDD193}.Debug|Any CPU.Build.0 = Debug|Any CPU
368+
{E07608CC-D710-4655-BB9E-D22CF3CDD193}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
369+
{E07608CC-D710-4655-BB9E-D22CF3CDD193}.Publish|Any CPU.Build.0 = Debug|Any CPU
370+
{E07608CC-D710-4655-BB9E-D22CF3CDD193}.Release|Any CPU.ActiveCfg = Release|Any CPU
371+
{E07608CC-D710-4655-BB9E-D22CF3CDD193}.Release|Any CPU.Build.0 = Release|Any CPU
364372
EndGlobalSection
365373
GlobalSection(SolutionProperties) = preSolution
366374
HideSolutionNode = FALSE
@@ -413,6 +421,7 @@ Global
413421
{677F1381-7830-4115-9C1A-58B282629DC6} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
414422
{4762BCAF-E1C5-4714-B88D-E50FA333C50E} = {078F96B4-09E1-4E0E-B214-F71A4F4BF633}
415423
{C754950A-E16C-4F96-9CC7-9328E361B5AF} = {FA3720F1-C99A-49B2-9577-A940257098BF}
424+
{E07608CC-D710-4655-BB9E-D22CF3CDD193} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
416425
EndGlobalSection
417426
GlobalSection(ExtensibilityGlobals) = postSolution
418427
SolutionGuid = {FBDC56A3-86AD-4323-AA0F-201E59123B83}

‎dotnet/samples/KernelSyntaxExamples/Example53_Kusto.cs‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System;
4+
using System.Threading.Tasks;
5+
using Microsoft.SemanticKernel;
6+
using Microsoft.SemanticKernel.Connectors.Memory.Kusto;
7+
using Microsoft.SemanticKernel.Memory;
8+
using RepoUtils;
9+
10+
// ReSharper disable once InconsistentNaming
11+
public static class Example53_Kusto
12+
{
13+
private const string MemoryCollectionName = "kusto_test";
14+
15+
public static async Task RunAsync()
16+
{
17+
var connectionString = new Kusto.Data.KustoConnectionStringBuilder(TestConfiguration.Kusto.ConnectionString).WithAadUserPromptAuthentication();
18+
using KustoMemoryStore memoryStore = new(connectionString, "MyDatabase");
19+
20+
IKernel kernel = Kernel.Builder
21+
.WithLogger(ConsoleLogger.Logger)
22+
.WithOpenAITextCompletionService(
23+
modelId: TestConfiguration.OpenAI.ModelId,
24+
apiKey: TestConfiguration.OpenAI.ApiKey)
25+
.WithOpenAITextEmbeddingGenerationService(
26+
modelId: TestConfiguration.OpenAI.EmbeddingModelId,
27+
apiKey: TestConfiguration.OpenAI.ApiKey)
28+
.WithMemoryStorage(memoryStore)
29+
.Build();
30+
31+
Console.WriteLine("== Printing Collections in DB ==");
32+
var collections = memoryStore.GetCollectionsAsync();
33+
await foreach (var collection in collections)
34+
{
35+
Console.WriteLine(collection);
36+
}
37+
38+
Console.WriteLine("== Adding Memories ==");
39+
40+
var key1 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat1", text: "british short hair");
41+
var key2 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat2", text: "orange tabby");
42+
var key3 = await kernel.Memory.SaveInformationAsync(MemoryCollectionName, id: "cat3", text: "norwegian forest cat");
43+
44+
Console.WriteLine("== Printing Collections in DB ==");
45+
collections = memoryStore.GetCollectionsAsync();
46+
await foreach (var collection in collections)
47+
{
48+
Console.WriteLine(collection);
49+
}
50+
51+
Console.WriteLine("== Retrieving Memories Through the Kernel ==");
52+
MemoryQueryResult? lookup = await kernel.Memory.GetAsync(MemoryCollectionName, "cat1");
53+
Console.WriteLine(lookup != null ? lookup.Metadata.Text : "ERROR: memory not found");
54+
55+
Console.WriteLine("== Retrieving Memories Directly From the Store ==");
56+
var memory1 = await memoryStore.GetAsync(MemoryCollectionName, key1);
57+
var memory2 = await memoryStore.GetAsync(MemoryCollectionName, key2);
58+
var memory3 = await memoryStore.GetAsync(MemoryCollectionName, key3);
59+
Console.WriteLine(memory1 != null ? memory1.Metadata.Text : "ERROR: memory not found");
60+
Console.WriteLine(memory2 != null ? memory2.Metadata.Text : "ERROR: memory not found");
61+
Console.WriteLine(memory3 != null ? memory3.Metadata.Text : "ERROR: memory not found");
62+
63+
Console.WriteLine("== Similarity Searching Memories: My favorite color is orange ==");
64+
var searchResults = kernel.Memory.SearchAsync(MemoryCollectionName, "My favorite color is orange", limit: 3, minRelevanceScore: 0.8);
65+
66+
await foreach (var item in searchResults)
67+
{
68+
Console.WriteLine(item.Metadata.Text + " : " + item.Relevance);
69+
}
70+
71+
Console.WriteLine("== Removing Collection {0} ==", MemoryCollectionName);
72+
await memoryStore.DeleteCollectionAsync(MemoryCollectionName);
73+
74+
Console.WriteLine("== Printing Collections in DB ==");
75+
await foreach (var collection in collections)
76+
{
77+
Console.WriteLine(collection);
78+
}
79+
}
80+
}

‎dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
<ProjectReference Include="..\..\src\Connectors\Connectors.AI.HuggingFace\Connectors.AI.HuggingFace.csproj" />
3434
<ProjectReference Include="..\..\src\Connectors\Connectors.Memory.AzureCognitiveSearch\Connectors.Memory.AzureCognitiveSearch.csproj" />
3535
<ProjectReference Include="..\..\src\Connectors\Connectors.Memory.Chroma\Connectors.Memory.Chroma.csproj" />
36+
<ProjectReference Include="..\..\src\Connectors\Connectors.Memory.Kusto\Connectors.Memory.Kusto.csproj" />
3637
<ProjectReference Include="..\..\src\Connectors\Connectors.Memory.Postgres\Connectors.Memory.Postgres.csproj" />
3738
<ProjectReference Include="..\..\src\Connectors\Connectors.Memory.Weaviate\Connectors.Memory.Weaviate.csproj" />
3839
<ProjectReference Include="..\..\src\Connectors\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj" />

‎dotnet/samples/KernelSyntaxExamples/Program.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ public static async Task Main()
7272
await Example50_Chroma.RunAsync().SafeWaitAsync(cancelToken);
7373
await Example51_StepwisePlanner.RunAsync().SafeWaitAsync(cancelToken);
7474
await Example52_ApimAuth.RunAsync().SafeWaitAsync(cancelToken);
75+
await Example53_Kusto.RunAsync().SafeWaitAsync(cancelToken);
7576
}
7677

7778
private static void LoadUserSecrets()

‎dotnet/samples/KernelSyntaxExamples/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ dotnet user-secrets set "Apim:SubscriptionKey" "..."
6969
7070
dotnet user-secrets set "Postgres:ConnectionString" "..."
7171
dotnet user-secrets set "Redis:Configuration" "..."
72+
dotnet user-secrets set "Kusto:ConnectionString" "..."
7273
```
7374

7475
To set your secrets with environment variables, use these names:

‎dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public static void Initialize(IConfigurationRoot configRoot)
3636
public static RedisConfig Redis => LoadSection<RedisConfig>();
3737
public static JiraConfig Jira => LoadSection<JiraConfig>();
3838
public static ChromaConfig Chroma => LoadSection<ChromaConfig>();
39+
public static KustoConfig Kusto => LoadSection<KustoConfig>();
3940

4041
private static T LoadSection<T>([CallerMemberName] string? caller = null)
4142
{
@@ -154,5 +155,10 @@ public class ChromaConfig
154155
{
155156
public string Endpoint { get; set; }
156157
}
158+
159+
public class KustoConfig
160+
{
161+
public string ConnectionString { get; set; }
162+
}
157163
#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor.
158164
}

‎dotnet/src/Connectors/Connectors.Memory.Kusto/Connectors.Memory.Kusto.csproj‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<!-- THIS PROPERTY GROUP MUST COME FIRST -->
4+
<AssemblyName>Microsoft.SemanticKernel.Connectors.Memory.Kusto</AssemblyName>
5+
<RootNamespace>Microsoft.SemanticKernel.Connectors.Memory.Kusto</RootNamespace>
6+
<TargetFramework>netstandard2.0</TargetFramework>
7+
8+
<!--NU5104: A stable release of a package should not have a prerelease dependency.-->
9+
<NoWarn>NU5104</NoWarn>
10+
</PropertyGroup>
11+
12+
<!-- IMPORT NUGET PACKAGE SHARED PROPERTIES -->
13+
<Import Project="$(RepoRoot)/dotnet/nuget/nuget-package.props" />
14+
<Import Project="$(RepoRoot)/dotnet/src/InternalUtilities/src/InternalUtilities.props" />
15+
16+
<PropertyGroup>
17+
<!-- NuGet Package Settings -->
18+
<PackageId>Microsoft.SemanticKernel.Connectors.Memory.Kusto</PackageId>
19+
<Title>Semantic Kernel - Azure Data Explorer (Kusto) Semantic Memory</Title>
20+
<Description>Azure Data Explorer (Kusto) Semantic Memory connector for Semantic Kernel</Description>
21+
</PropertyGroup>
22+
23+
<ItemGroup>
24+
<PackageReference Include="Microsoft.Azure.Kusto.Data" />
25+
</ItemGroup>
26+
27+
<ItemGroup>
28+
<ProjectReference Include="..\..\SemanticKernel\SemanticKernel.csproj" />
29+
</ItemGroup>
30+
31+
</Project>

‎dotnet/src/Connectors/Connectors.Memory.Kusto/KustoMemoryRecord.cs‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System;
4+
using Kusto.Cloud.Platform.Utils;
5+
using Microsoft.SemanticKernel.AI.Embeddings;
6+
using Microsoft.SemanticKernel.Memory;
7+
8+
namespace Microsoft.SemanticKernel.Connectors.Memory.Kusto;
9+
10+
/// <summary>
11+
/// Kusto memory record entity.
12+
/// </summary>
13+
public sealed class KustoMemoryRecord
14+
{
15+
/// <summary>
16+
/// Entity key.
17+
/// </summary>
18+
public string Key { get; set; }
19+
20+
/// <summary>
21+
/// Metadata associated with memory entity.
22+
/// </summary>
23+
public MemoryRecordMetadata Metadata { get; set; }
24+
25+
/// <summary>
26+
/// Source content embedding.
27+
/// </summary>
28+
public Embedding<float> Embedding { get; set; }
29+
30+
/// <summary>
31+
/// Optional timestamp.
32+
/// </summary>
33+
public DateTimeOffset? Timestamp { get; set; }
34+
35+
/// <summary>
36+
/// Initializes a new instance of the <see cref="KustoMemoryRecord"/> class.
37+
/// </summary>
38+
/// <param name="record">Instance of <see cref="MemoryRecord"/>.</param>
39+
public KustoMemoryRecord(MemoryRecord record) : this(record.Key, record.Metadata, record.Embedding, record.Timestamp) { }
40+
41+
/// <summary>
42+
/// Initializes a new instance of the <see cref="KustoMemoryRecord"/> class.
43+
/// </summary>
44+
/// <param name="key">Entity key.</param>
45+
/// <param name="metadata">Metadata associated with memory entity.</param>
46+
/// <param name="embedding">Source content embedding.</param>
47+
/// <param name="timestamp">Optional timestamp.</param>
48+
public KustoMemoryRecord(string key, MemoryRecordMetadata metadata, Embedding<float> embedding, DateTimeOffset? timestamp = null)
49+
{
50+
this.Key = key;
51+
this.Metadata = metadata;
52+
this.Embedding = embedding;
53+
this.Timestamp = timestamp;
54+
}
55+
56+
/// <summary>
57+
/// Initializes a new instance of the <see cref="KustoMemoryRecord"/> class.
58+
/// </summary>
59+
/// <param name="key">Entity key.</param>
60+
/// <param name="metadata">Serialized metadata associated with memory entity.</param>
61+
/// <param name="embedding">Source content embedding.</param>
62+
/// <param name="timestamp">Optional timestamp.</param>
63+
public KustoMemoryRecord(string key, string metadata, string? embedding, string? timestamp = null)
64+
{
65+
this.Key = key;
66+
this.Metadata = KustoSerializer.DeserializeMetadata(metadata);
67+
this.Embedding = KustoSerializer.DeserializeEmbedding(embedding);
68+
this.Timestamp = KustoSerializer.DeserializeDateTimeOffset(timestamp);
69+
}
70+
71+
/// <summary>
72+
/// Returns instance of mapped <see cref="MemoryRecord"/>.
73+
/// </summary>
74+
public MemoryRecord ToMemoryRecord()
75+
{
76+
return new MemoryRecord(this.Metadata, this.Embedding, this.Key, this.Timestamp);
77+
}
78+
79+
/// <summary>
80+
/// Writes properties of <see cref="KustoMemoryRecord"/> instance to stream using <see cref="CsvWriter"/>.
81+
/// </summary>
82+
/// <param name="streamWriter">Instance of <see cref="CsvWriter"/> to write properties to stream.</param>
83+
public void WriteToCsvStream(CsvWriter streamWriter)
84+
{
85+
var jsonifiedMetadata = KustoSerializer.SerializeMetadata(this.Metadata);
86+
var jsonifiedEmbedding = KustoSerializer.SerializeEmbedding(this.Embedding);
87+
var isoFormattedDate = KustoSerializer.SerializeDateTimeOffset(this.Timestamp);
88+
89+
streamWriter.WriteField(this.Key);
90+
streamWriter.WriteField(jsonifiedMetadata);
91+
streamWriter.WriteField(jsonifiedEmbedding);
92+
streamWriter.WriteField(isoFormattedDate);
93+
streamWriter.CompleteRecord();
94+
}
95+
}

0 commit comments

Comments
 (0)