Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding support for serverless, reacting to api changes, adding tests #108

Merged
merged 15 commits into from
May 18, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions Pinecone.sln
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Example.CSharp", "example\E
EndProject
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Example.FSharp", "example\Example.FSharp\Example.FSharp.fsproj", "{48198DC6-2D75-417F-8096-9FF9AC657511}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PineconeTests", "test\PineconeTests.csproj", "{122FC8A9-A7F9-43DA-A67E-4148D07B9597}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand All @@ -27,6 +29,10 @@ Global
{48198DC6-2D75-417F-8096-9FF9AC657511}.Debug|Any CPU.Build.0 = Debug|Any CPU
{48198DC6-2D75-417F-8096-9FF9AC657511}.Release|Any CPU.ActiveCfg = Release|Any CPU
{48198DC6-2D75-417F-8096-9FF9AC657511}.Release|Any CPU.Build.0 = Release|Any CPU
{122FC8A9-A7F9-43DA-A67E-4148D07B9597}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{122FC8A9-A7F9-43DA-A67E-4148D07B9597}.Debug|Any CPU.Build.0 = Debug|Any CPU
{122FC8A9-A7F9-43DA-A67E-4148D07B9597}.Release|Any CPU.ActiveCfg = Release|Any CPU
{122FC8A9-A7F9-43DA-A67E-4148D07B9597}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
9 changes: 5 additions & 4 deletions example/Example.CSharp/Program.cs
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
using Pinecone;

using var pinecone = new PineconeClient("[api-key]", "[pinecone-env]");
using var pinecone = new PineconeClient("[api-key]");

// Check if the index exists and create it if it doesn't
// Depending on the storage type and infrastructure state this may take a while
// Free tier is limited to 1 index only
// Free tier is limited to 1 pod index and 5 serverless indexes only
var indexName = "test-index";
var indexList = await pinecone.ListIndexes();

if (!indexList.Contains(indexName))
if (!indexList.Select(x => x.Name).Contains(indexName))
{
await pinecone.CreateIndex(indexName, 1536, Metric.Cosine);
// free serverless indexes are currently only available on AWS us-east-1
await pinecone.CreateServerlessIndex(indexName, 1536, Metric.Cosine, "aws", "us-east-1");
}

// Get the Pinecone index by name (uses gRPC by default).
Expand Down
11 changes: 6 additions & 5 deletions example/Example.FSharp/Program.fs
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,17 @@ let createMetadata x =
MetadataMap(x |> Seq.map (fun (k, m) -> KeyValuePair(k,m) ))

let main = task {
use pinecone = new PineconeClient("[api-key]", "[pinecone-env]")
use pinecone = new PineconeClient("[api-key]")

// Check if the index exists and create it if it doesn't
// Depending on the storage type and infrastructure state this may take a while
// Free tier is limited to 1 index only
// Free tier is limited to 1 pod index and 5 serverless indexes only
let indexName = "test-index"
let! indexList = pinecone.ListIndexes()
if indexList |> Array.contains indexName |> not then
do! pinecone.CreateIndex(indexName, 1536u, Metric.Cosine)

if not (indexList |> Array.exists (fun index -> index.Name = indexName)) then
// free serverless indexes are currently only available on AWS us-east-1
pinecone.CreateServerlessIndex(indexName, 1536u, Metric.Cosine, "aws", "us-east-1")

// Get the Pinecone index by name (uses gRPC by default).
// The index client is thread-safe, consider caching and/or
// injecting it as a singleton into your DI container.
Expand Down
2 changes: 2 additions & 0 deletions src/Grpc/GrpcTransport.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ public async Task<IndexStats> DescribeStats(MetadataMap? filter = null)
}

using var call = Grpc.DescribeIndexStatsAsync(request, Auth);

return (await call.ConfigureAwait(false)).ToPublicType();
}

Expand Down Expand Up @@ -89,6 +90,7 @@ public async Task<uint> Upsert(IEnumerable<Vector> vectors, string? indexNamespa
request.Vectors.AddRange(vectors.Select(v => v.ToProtoVector()));

using var call = Grpc.UpsertAsync(request, Auth);

return (await call.ConfigureAwait(false)).UpsertedCount;
}

Expand Down
8 changes: 5 additions & 3 deletions src/Index.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@ public sealed partial record Index<
#endif
TTransport> where TTransport : ITransport<TTransport>
{
[JsonPropertyName("database")]
public required IndexDetails Details { get; init; }

public required string Name { get; init; }
public required uint Dimension { get; init; }
public required Metric Metric { get; init; }
public string? Host { get; init; }
public required IndexSpec Spec { get; init; }
public required IndexStatus Status { get; init; }
}

Expand Down
81 changes: 60 additions & 21 deletions src/PineconeClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ namespace Pinecone;
public sealed class PineconeClient : IDisposable
{
private readonly HttpClient Http;
private readonly string? _legacyEnvironment;

public PineconeClient(string apiKey)
: this(apiKey, new Uri($"https://api.pinecone.io"))
{
}

public PineconeClient(string apiKey, string environment)
{
Expand All @@ -18,6 +24,7 @@ public PineconeClient(string apiKey, string environment)

Http = new() { BaseAddress = new Uri($"https://controller.{environment}.pinecone.io") };
Http.DefaultRequestHeaders.Add("Api-Key", apiKey);
_legacyEnvironment = environment;
}

public PineconeClient(string apiKey, Uri baseUrl)
Expand All @@ -38,28 +45,48 @@ public PineconeClient(string apiKey, HttpClient client)
Http.DefaultRequestHeaders.Add("Api-Key", apiKey);
}

public async Task<string[]> ListIndexes()
public async Task<IndexDetails[]> ListIndexes()
{
var indexes = await Http
.GetFromJsonAsync("/databases", SerializerContext.Default.StringArray)
var listIndexesResult = (ListIndexesResult?)await Http
.GetFromJsonAsync("/indexes", typeof(ListIndexesResult), SerializerContext.Default)
.ConfigureAwait(false);

return indexes ?? [];
return listIndexesResult?.Indexes ?? [];
}

public Task CreateIndex(string name, uint dimension, Metric metric) =>
CreateIndex(new IndexDetails { Name = name, Dimension = dimension, Metric = metric });
public Task CreatePodBasedIndex(string name, uint dimiension, Metric metric, string environment, string podType, long pods)
=> CreateIndexAsync(new CreateIndexRequest
{
Name = name,
Dimension = dimiension,
Metric = metric,
Spec = new IndexSpec { Pod = new PodSpec { Environment = environment, PodType = podType, Pods = pods } }
});

public Task CreateServerlessIndex(string name, uint dimiension, Metric metric, string cloud, string region)
=> CreateIndexAsync(new CreateIndexRequest
{
Name = name,
Dimension = dimiension,
Metric = metric,
Spec = new IndexSpec { Serverless = new ServerlessSpec { Cloud = cloud, Region = region } }
});

public async Task CreateIndex(IndexDetails indexDetails, string? sourceCollection = null)
private async Task CreateIndexAsync(CreateIndexRequest request)
{
var request = CreateIndexRequest.From(indexDetails, sourceCollection);
var response = await Http
.PostAsJsonAsync("/databases", request, SerializerContext.Default.CreateIndexRequest)
.PostAsJsonAsync("/indexes", request, SerializerContext.Default.CreateIndexRequest)
.ConfigureAwait(false);

await response.CheckStatusCode().ConfigureAwait(false);
}

[Obsolete($"Use '{nameof(CreateServerlessIndex)}' or '{nameof(CreatePodBasedIndex)}' methods instead.")]
public Task CreateIndex(string name, uint dimension, Metric metric)
=> _legacyEnvironment is not null
? CreatePodBasedIndex(name, dimension, metric, _legacyEnvironment, "starter", 1)
: throw new InvalidOperationException($"Use '{nameof(CreateServerlessIndex)}' or '{nameof(CreatePodBasedIndex)}' methods instead.");

public Task<Index<GrpcTransport>> GetIndex(string name) => GetIndex<GrpcTransport>(name);

#if NET7_0_OR_GREATER
Expand All @@ -70,22 +97,34 @@ public async Task<Index<TTransport>> GetIndex<
#endif
where TTransport : ITransport<TTransport>
{
var response = await Http
var response = (IndexDetails)(await Http
.GetFromJsonAsync(
$"/databases/{UrlEncoder.Default.Encode(name)}",
typeof(Index<TTransport>),
$"/indexes/{UrlEncoder.Default.Encode(name)}",
typeof(IndexDetails),
SerializerContext.Default)
.ConfigureAwait(false) ?? throw new HttpRequestException("GetIndex request has failed.");
.ConfigureAwait(false) ?? throw new HttpRequestException("GetIndex request has failed."));

var index = (Index<TTransport>)response;
var host = index.Status.Host;
// TODO: Host is optional according to the API spec: https://docs.pinecone.io/reference/api/control-plane/describe_index
// but Transport requires it
var host = response.Host!;
var apiKey = Http.DefaultRequestHeaders.GetValues(Constants.RestApiKey).First();

var index = new Index<TTransport>
{
Name = response.Name,
Dimension = response.Dimension,
Metric = response.Metric,
Host = response.Host,
Spec = response.Spec,
Status = response.Status,
};

#if NET7_0_OR_GREATER
index.Transport = TTransport.Create(host, apiKey);
#else
index.Transport = ITransport<TTransport>.Create(host, apiKey);
#endif

return index;
}

Expand All @@ -100,7 +139,7 @@ public async Task ConfigureIndex(string name, int? replicas = null, string? podT
var request = new ConfigureIndexRequest { Replicas = replicas, PodType = podType };
var response = await Http
.PatchAsJsonAsync(
$"/databases/{UrlEncoder.Default.Encode(name)}",
$"/indexes/{UrlEncoder.Default.Encode(name)}",
request,
SerializerContext.Default.ConfigureIndexRequest)
.ConfigureAwait(false);
Expand All @@ -109,17 +148,17 @@ public async Task ConfigureIndex(string name, int? replicas = null, string? podT
}

public async Task DeleteIndex(string name) =>
await (await Http.DeleteAsync($"/databases/{UrlEncoder.Default.Encode(name)}").ConfigureAwait(false))
await (await Http.DeleteAsync($"/indexes/{UrlEncoder.Default.Encode(name)}").ConfigureAwait(false))
.CheckStatusCode()
.ConfigureAwait(false);

public async Task<string[]> ListCollections()
public async Task<CollectionDetails[]> ListCollections()
{
var collections = await Http
.GetFromJsonAsync("/collections", SerializerContext.Default.StringArray)
var listCollectionsResult = (ListCollectionsResult?)await Http
.GetFromJsonAsync("/collections", typeof(ListCollectionsResult), SerializerContext.Default)
.ConfigureAwait(false);

return collections ?? [];
return listCollectionsResult?.Collections ?? [];
}

public async Task CreateCollection(string name, string source)
Expand Down
2 changes: 2 additions & 0 deletions src/Rest/SerializerContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ namespace Pinecone.Rest;
[JsonSerializable(typeof(DeleteRequest))]
[JsonSerializable(typeof(MetadataMap))]
[JsonSerializable(typeof(MetadataValue[]))]
[JsonSerializable(typeof(ListIndexesResult))]
[JsonSerializable(typeof(ListCollectionsResult))]
[JsonSourceGenerationOptions(
GenerationMode = JsonSourceGenerationMode.Default,
PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,
Expand Down
22 changes: 5 additions & 17 deletions src/Rest/Types.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,12 @@

namespace Pinecone.Rest;

internal sealed record CreateIndexRequest : IndexDetails
internal sealed record CreateIndexRequest
{
[JsonPropertyName("source_collection")]
public string? SourceCollection { get; init; }

public static CreateIndexRequest From(
IndexDetails index,
string? sourceCollection) => new()
{
Dimension = index.Dimension,
Metric = index.Metric,
Name = index.Name,
Pods = index.Pods,
PodType = index.PodType,
Replicas = index.Replicas,
MetadataConfig = index.MetadataConfig,
SourceCollection = sourceCollection
};
public required string Name { get; init; }
public required uint Dimension { get; init; }
public required Metric Metric { get; init; }
public required IndexSpec Spec { get; init; }
}

internal readonly record struct ConfigureIndexRequest
Expand Down
19 changes: 16 additions & 3 deletions src/Types/CollectionTypes.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,25 @@

namespace Pinecone;

public record ListCollectionsResult
{
public required CollectionDetails[] Collections { get; init; }
}

public record CollectionDetails
{
public required string Name { get; init; }
public required long Size { get; init; }
public required string Status { get; init; }
public required long Dimension { get; init; }
public long? Size { get; init; }
public required CollectionStatus Status { get; init; }
public required uint Dimension { get; init; }
[JsonPropertyName("vector_count")]
public required long VectorCount { get; init; }
public required string Environment { get; init; }
}

public enum CollectionStatus
{
Initializing = 0,
Ready = 1,
Terminating = 2
}
48 changes: 38 additions & 10 deletions src/Types/IndexTypes.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,20 @@

namespace Pinecone;

public record ListIndexesResult
neon-sunset marked this conversation as resolved.
Show resolved Hide resolved
{
public required IndexDetails[] Indexes { get; init; }
}

public record IndexDetails
{
public required string Name { get; init; }
public required uint Dimension { get; init; }
public required Metric Metric { get; init; }
public long? Pods { get; init; }
[JsonPropertyName("pod_type")]
public string? PodType { get; init; }
public long? Replicas { get; init; }
[JsonPropertyName("metadata_config")]
public MetadataMap? MetadataConfig { get; init; }
public string? Host { get; init; }

public required IndexSpec Spec { get; init;}
public required IndexStatus Status { get; init; }
}

[JsonConverter(typeof(JsonStringEnumConverter<Metric>))]
Expand All @@ -29,10 +32,6 @@ public record IndexStatus
[JsonPropertyName("ready")]
public required bool IsReady { get; init; }
public required IndexState State { get; init; }
public required string Host { get; init; }
public required int Port { get; init; }
public string?[]? Waiting { get; init; }
public string?[]? Crashed { get; init; }
}

[JsonConverter(typeof(JsonStringEnumConverter<IndexState>))]
Expand All @@ -48,6 +47,35 @@ public enum IndexState
InitializationFailed = 7
}

public record IndexSpec
{
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public ServerlessSpec? Serverless { get; init; }

[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public PodSpec? Pod { get; init; }
}

public record ServerlessSpec
{
public required string Cloud { get; init; }
public required string Region { get; init; }
}

public record PodSpec
{
[JsonPropertyName("environment")]
public required string Environment { get; init; }
public long? Replicas { get; init; }
[JsonPropertyName("pod_type")]
public required string PodType { get; init; }
public long Pods { get; init; }
[JsonPropertyName("metadata_config")]
public MetadataMap? MetadataConfig { get; init; }
[JsonPropertyName("source_collection")]
public string? SourceCollection { get; init; }
}

public record IndexStats
{
[JsonConverter(typeof(IndexNamespaceArrayConverter))]
Expand Down
Loading
Loading