Lots of groundwork and the app runs but not currently working to generate embeddings.
This commit is contained in:
27
VectorSearchApp/Configuration/AppConfiguration.cs
Normal file
27
VectorSearchApp/Configuration/AppConfiguration.cs
Normal file
@@ -0,0 +1,27 @@
|
||||
namespace VectorSearchApp.Configuration;
|
||||
|
||||
public class AppConfiguration
|
||||
{
|
||||
public QdrantConfiguration Qdrant { get; set; } = new();
|
||||
public EmbeddingConfiguration Embedding { get; set; } = new();
|
||||
public AppSettings App { get; set; } = new();
|
||||
}
|
||||
|
||||
public class QdrantConfiguration
|
||||
{
|
||||
public string Host { get; set; } = "localhost";
|
||||
public int GrpcPort { get; set; } = 6334;
|
||||
public int HttpPort { get; set; } = 6333;
|
||||
public string CollectionName { get; set; } = "addresses";
|
||||
}
|
||||
|
||||
public class EmbeddingConfiguration
|
||||
{
|
||||
public string ModelName { get; set; } = "sentence-transformers/all-MiniLM-L6-v2";
|
||||
public int Dimension { get; set; } = 384;
|
||||
}
|
||||
|
||||
public class AppSettings
|
||||
{
|
||||
public int BatchSize { get; set; } = 10;
|
||||
}
|
||||
8
VectorSearchApp/Models/Address.cs
Normal file
8
VectorSearchApp/Models/Address.cs
Normal file
@@ -0,0 +1,8 @@
|
||||
namespace VectorSearchApp.Models;
|
||||
|
||||
public class Address
|
||||
{
|
||||
public Guid Id { get; set; } = Guid.NewGuid();
|
||||
public string FullAddress { get; set; } = string.Empty;
|
||||
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
|
||||
}
|
||||
8
VectorSearchApp/Models/AddressEmbedding.cs
Normal file
8
VectorSearchApp/Models/AddressEmbedding.cs
Normal file
@@ -0,0 +1,8 @@
|
||||
namespace VectorSearchApp.Models;
|
||||
|
||||
public class AddressEmbedding
|
||||
{
|
||||
public Guid Id { get; set; }
|
||||
public string FullAddress { get; set; } = string.Empty;
|
||||
public float[] Vector { get; set; } = Array.Empty<float>();
|
||||
}
|
||||
173
VectorSearchApp/Program.cs
Normal file
173
VectorSearchApp/Program.cs
Normal file
@@ -0,0 +1,173 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using VectorSearchApp.Configuration;
|
||||
using VectorSearchApp.Models;
|
||||
using VectorSearchApp.Services;
|
||||
|
||||
Console.WriteLine("=== Vector Search Address Application ===");
|
||||
Console.WriteLine("Using sentence-transformers/all-MiniLM-L6-v2 model and Qdrant vector database");
|
||||
Console.WriteLine();
|
||||
|
||||
// Load configuration
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.SetBasePath(AppContext.BaseDirectory)
|
||||
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
|
||||
.Build();
|
||||
|
||||
var appConfig = new AppConfiguration();
|
||||
configuration.GetSection("Qdrant").Bind(appConfig.Qdrant);
|
||||
configuration.GetSection("Embedding").Bind(appConfig.Embedding);
|
||||
configuration.GetSection("App").Bind(appConfig.App);
|
||||
|
||||
// Initialize services
|
||||
Console.WriteLine("Initializing services...");
|
||||
var embeddingService = new EmbeddingService(appConfig.Embedding);
|
||||
IQdrantService? qdrantService = null;
|
||||
|
||||
try
|
||||
{
|
||||
qdrantService = new QdrantService(appConfig.Qdrant, appConfig.Embedding.Dimension);
|
||||
Console.WriteLine("Initializing Qdrant collection...");
|
||||
await qdrantService.InitializeCollectionAsync();
|
||||
Console.WriteLine($"Collection '{appConfig.Qdrant.CollectionName}' is ready.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Warning: Could not connect to Qdrant at {appConfig.Qdrant.Host}:{appConfig.Qdrant.GrpcPort}");
|
||||
Console.WriteLine($"Error: {ex.Message}");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("Please ensure Qdrant is running. You can start it with:");
|
||||
Console.WriteLine(" cd VectorSearchApp && docker-compose up -d");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("The application will continue, but address storage/search will not be available.");
|
||||
Console.WriteLine();
|
||||
}
|
||||
|
||||
Console.WriteLine("Type 'exit' to quit at any time.");
|
||||
Console.WriteLine();
|
||||
|
||||
while (true)
|
||||
{
|
||||
Console.WriteLine("Options:");
|
||||
Console.WriteLine(" 1. Add a new address");
|
||||
Console.WriteLine(" 2. Search for similar addresses");
|
||||
Console.WriteLine(" 3. Exit");
|
||||
Console.Write("Select an option: ");
|
||||
|
||||
var option = Console.ReadLine()?.Trim();
|
||||
|
||||
if (option?.ToLower() == "exit" || option == "3")
|
||||
{
|
||||
Console.WriteLine("Goodbye!");
|
||||
return;
|
||||
}
|
||||
|
||||
switch (option)
|
||||
{
|
||||
case "1":
|
||||
await AddAddressAsync(embeddingService, qdrantService, appConfig);
|
||||
break;
|
||||
case "2":
|
||||
await SearchAddressesAsync(embeddingService, qdrantService, appConfig);
|
||||
break;
|
||||
default:
|
||||
Console.WriteLine("Invalid option. Please try again.");
|
||||
break;
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
}
|
||||
|
||||
async Task AddAddressAsync(IEmbeddingService embeddingService, IQdrantService? qdrantService, AppConfiguration config)
|
||||
{
|
||||
Console.Write("Enter the address: ");
|
||||
var addressText = Console.ReadLine()?.Trim();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(addressText))
|
||||
{
|
||||
Console.WriteLine("Address cannot be empty.");
|
||||
return;
|
||||
}
|
||||
|
||||
Console.WriteLine("Generating embedding...");
|
||||
try
|
||||
{
|
||||
var embedding = await embeddingService.GenerateEmbeddingAsync(addressText);
|
||||
Console.WriteLine($"Embedding generated (dimension: {embedding.Length})");
|
||||
|
||||
if (qdrantService != null)
|
||||
{
|
||||
var address = new Address
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
FullAddress = addressText,
|
||||
CreatedAt = DateTime.UtcNow
|
||||
};
|
||||
|
||||
Console.WriteLine("Storing in Qdrant...");
|
||||
await qdrantService.StoreAddressAsync(address, embedding);
|
||||
Console.WriteLine($"Address stored successfully! (ID: {address.Id})");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("Address embedding generated but not stored (Qdrant not available).");
|
||||
}
|
||||
|
||||
// Display first few values of embedding as confirmation
|
||||
Console.Write("Embedding preview: [");
|
||||
var previewCount = Math.Min(10, embedding.Length);
|
||||
for (int i = 0; i < previewCount; i++)
|
||||
{
|
||||
Console.Write($"{embedding[i]:F4}");
|
||||
if (i < previewCount - 1) Console.Write(", ");
|
||||
}
|
||||
if (embedding.Length > previewCount) Console.Write(", ...");
|
||||
Console.WriteLine("]");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Error generating embedding: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
async Task SearchAddressesAsync(IEmbeddingService embeddingService, IQdrantService? qdrantService, AppConfiguration config)
|
||||
{
|
||||
if (qdrantService == null)
|
||||
{
|
||||
Console.WriteLine("Search is not available because Qdrant is not connected.");
|
||||
return;
|
||||
}
|
||||
|
||||
Console.Write("Enter search query: ");
|
||||
var query = Console.ReadLine()?.Trim();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(query))
|
||||
{
|
||||
Console.WriteLine("Query cannot be empty.");
|
||||
return;
|
||||
}
|
||||
|
||||
Console.WriteLine("Generating query embedding...");
|
||||
try
|
||||
{
|
||||
var queryEmbedding = await embeddingService.GenerateEmbeddingAsync(query);
|
||||
Console.WriteLine($"Searching for similar addresses...");
|
||||
|
||||
var results = await qdrantService.SearchSimilarAddressesAsync(queryEmbedding, limit: 5);
|
||||
|
||||
if (results.Count == 0)
|
||||
{
|
||||
Console.WriteLine("No similar addresses found.");
|
||||
return;
|
||||
}
|
||||
|
||||
Console.WriteLine($"\nFound {results.Count} similar address(es):");
|
||||
for (int i = 0; i < results.Count; i++)
|
||||
{
|
||||
Console.WriteLine($" {i + 1}. {results[i].FullAddress}");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Error during search: {ex.Message}");
|
||||
}
|
||||
}
|
||||
52
VectorSearchApp/Services/EmbeddingService.cs
Normal file
52
VectorSearchApp/Services/EmbeddingService.cs
Normal file
@@ -0,0 +1,52 @@
|
||||
using System.Net.Http.Json;
|
||||
using VectorSearchApp.Configuration;
|
||||
using VectorSearchApp.Models;
|
||||
|
||||
namespace VectorSearchApp.Services;
|
||||
|
||||
public interface IEmbeddingService
|
||||
{
|
||||
Task<float[]> GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public class EmbeddingService : IEmbeddingService
|
||||
{
|
||||
private readonly HttpClient _httpClient;
|
||||
private readonly string _modelName;
|
||||
private readonly int _dimension;
|
||||
|
||||
public EmbeddingService(EmbeddingConfiguration config)
|
||||
{
|
||||
_modelName = config.ModelName;
|
||||
_dimension = config.Dimension;
|
||||
_httpClient = new HttpClient
|
||||
{
|
||||
BaseAddress = new Uri("https://api-inference.huggingface.co/models/")
|
||||
};
|
||||
_httpClient.DefaultRequestHeaders.Add("User-Agent", "VectorSearchApp");
|
||||
}
|
||||
|
||||
public async Task<float[]> GenerateEmbeddingAsync(string text, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var request = new
|
||||
{
|
||||
inputs = text
|
||||
};
|
||||
|
||||
var response = await _httpClient.PostAsJsonAsync(_modelName, request, cancellationToken);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
throw new InvalidOperationException($"Failed to generate embedding: {response.StatusCode}");
|
||||
}
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<float[][]>(cancellationToken: cancellationToken);
|
||||
|
||||
if (result?.Length > 0 && result[0].Length > 0)
|
||||
{
|
||||
return result[0];
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Failed to generate embedding");
|
||||
}
|
||||
}
|
||||
69
VectorSearchApp/Services/QdrantService.cs
Normal file
69
VectorSearchApp/Services/QdrantService.cs
Normal file
@@ -0,0 +1,69 @@
|
||||
using Qdrant.Client;
|
||||
using Qdrant.Client.Grpc;
|
||||
using VectorSearchApp.Configuration;
|
||||
using VectorSearchApp.Models;
|
||||
|
||||
namespace VectorSearchApp.Services;
|
||||
|
||||
public interface IQdrantService
|
||||
{
|
||||
Task InitializeCollectionAsync(CancellationToken cancellationToken = default);
|
||||
Task StoreAddressAsync(Address address, float[] embedding, CancellationToken cancellationToken = default);
|
||||
Task<List<AddressEmbedding>> SearchSimilarAddressesAsync(float[] queryEmbedding, int limit = 5, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public class QdrantService : IQdrantService
|
||||
{
|
||||
private readonly QdrantClient _client;
|
||||
private readonly string _collectionName;
|
||||
private readonly int _vectorDimension;
|
||||
|
||||
public QdrantService(QdrantConfiguration config, int vectorDimension)
|
||||
{
|
||||
_client = new QdrantClient(config.Host, config.GrpcPort);
|
||||
_collectionName = config.CollectionName;
|
||||
_vectorDimension = vectorDimension;
|
||||
}
|
||||
|
||||
public async Task InitializeCollectionAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var collections = await _client.ListCollectionsAsync(cancellationToken: cancellationToken);
|
||||
|
||||
if (!collections.Contains(_collectionName))
|
||||
{
|
||||
await _client.CreateCollectionAsync(_collectionName, new VectorParams
|
||||
{
|
||||
Size = (ulong)_vectorDimension,
|
||||
Distance = Distance.Cosine
|
||||
}, cancellationToken: cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task StoreAddressAsync(Address address, float[] embedding, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var point = new PointStruct
|
||||
{
|
||||
Id = new PointId { Uuid = address.Id.ToString() },
|
||||
Vectors = embedding,
|
||||
Payload =
|
||||
{
|
||||
["address"] = address.FullAddress,
|
||||
["created_at"] = address.CreatedAt.ToString("O")
|
||||
}
|
||||
};
|
||||
|
||||
await _client.UpsertAsync(_collectionName, new[] { point }, cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<List<AddressEmbedding>> SearchSimilarAddressesAsync(float[] queryEmbedding, int limit = 5, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var results = await _client.SearchAsync(_collectionName, queryEmbedding, limit: (ulong)limit, cancellationToken: cancellationToken);
|
||||
|
||||
return results.Select(r => new AddressEmbedding
|
||||
{
|
||||
Id = Guid.Parse(r.Id.Uuid),
|
||||
FullAddress = r.Payload["address"].StringValue,
|
||||
Vector = Array.Empty<float>()
|
||||
}).ToList();
|
||||
}
|
||||
}
|
||||
23
VectorSearchApp/VectorSearchApp.csproj
Normal file
23
VectorSearchApp/VectorSearchApp.csproj
Normal file
@@ -0,0 +1,23 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net8.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.2" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="10.0.2" />
|
||||
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.23.2" />
|
||||
<PackageReference Include="Qdrant.Client" Version="1.16.1" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Update="appsettings.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
15
VectorSearchApp/appsettings.json
Normal file
15
VectorSearchApp/appsettings.json
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Qdrant": {
|
||||
"Host": "localhost",
|
||||
"GrpcPort": 6334,
|
||||
"HttpPort": 6333,
|
||||
"CollectionName": "addresses"
|
||||
},
|
||||
"Embedding": {
|
||||
"ModelName": "sentence-transformers/all-MiniLM-L6-v2",
|
||||
"Dimension": 384
|
||||
},
|
||||
"App": {
|
||||
"BatchSize": 10
|
||||
}
|
||||
}
|
||||
17
VectorSearchApp/docker-compose.yml
Normal file
17
VectorSearchApp/docker-compose.yml
Normal file
@@ -0,0 +1,17 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
qdrant:
|
||||
image: qdrant/qdrant:latest
|
||||
container_name: vector-search-qdrant
|
||||
ports:
|
||||
- "6333:6333"
|
||||
- "6334:6334"
|
||||
volumes:
|
||||
- qdrant_storage:/qdrant/storage
|
||||
environment:
|
||||
- QDRANT__SERVICE__API_GRPC_PORT=6334
|
||||
- QDRANT__SERVICE__HTTP_PORT=6333
|
||||
|
||||
volumes:
|
||||
qdrant_storage:
|
||||
Reference in New Issue
Block a user