Adding support for serverless, reacting to api changes, adding tests

Pinecone.sln

@@ -9,6 +9,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Example.CSharp", "example\E
 EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Example.FSharp", "example\Example.FSharp\Example.FSharp.fsproj", "{48198DC6-2D75-417F-8096-9FF9AC657511}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PineconeTests", "test\PineconeTests.csproj", "{122FC8A9-A7F9-43DA-A67E-4148D07B9597}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -27,6 +29,10 @@ Global
 		{48198DC6-2D75-417F-8096-9FF9AC657511}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{48198DC6-2D75-417F-8096-9FF9AC657511}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{48198DC6-2D75-417F-8096-9FF9AC657511}.Release|Any CPU.Build.0 = Release|Any CPU
+		{122FC8A9-A7F9-43DA-A67E-4148D07B9597}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{122FC8A9-A7F9-43DA-A67E-4148D07B9597}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{122FC8A9-A7F9-43DA-A67E-4148D07B9597}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{122FC8A9-A7F9-43DA-A67E-4148D07B9597}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

example/Example.CSharp/Program.cs

@@ -1,16 +1,17 @@
 using Pinecone;
 
-using var pinecone = new PineconeClient("[api-key]", "[pinecone-env]");
+using var pinecone = new PineconeClient("[api-key]");
 
 // Check if the index exists and create it if it doesn't
 // Depending on the storage type and infrastructure state this may take a while
-// Free tier is limited to 1 index only
+// Free tier is limited to 1 pod index and 5 serverless indexes only
 var indexName = "test-index";
 var indexList = await pinecone.ListIndexes();
 
-if (!indexList.Contains(indexName))
+if (!indexList.Select(x => x.Name).Contains(indexName))
 {
-    await pinecone.CreateIndex(indexName, 1536, Metric.Cosine);
+    // free serverless indexes are currently only available on AWS us-east-1
+    await pinecone.CreateServerlessIndex(indexName, 1536, Metric.Cosine, "aws", "us-east-1");
 }
 
 // Get the Pinecone index by name (uses gRPC by default).

example/Example.FSharp/Program.fs

@@ -5,16 +5,17 @@ let createMetadata x =
     MetadataMap(x |> Seq.map (fun (k, m) -> KeyValuePair(k,m) ))
 
 let main = task {
-    use pinecone = new PineconeClient("[api-key]", "[pinecone-env]")
+    use pinecone = new PineconeClient("[api-key]")
 
     // Check if the index exists and create it if it doesn't
     // Depending on the storage type and infrastructure state this may take a while
-    // Free tier is limited to 1 index only
+    // Free tier is limited to 1 pod index and 5 serverless indexes only
     let indexName = "test-index"
     let! indexList = pinecone.ListIndexes()
-    if indexList |> Array.contains indexName |> not then
-        do! pinecone.CreateIndex(indexName, 1536u, Metric.Cosine)
-
+    if not (indexList |> Array.exists (fun index -> index.Name = indexName)) then
+        // free serverless indexes are currently only available on AWS us-east-1
+        pinecone.CreateServerlessIndex(indexName, 1536u, Metric.Cosine, "aws", "us-east-1")
+
     // Get the Pinecone index by name (uses gRPC by default).
     // The index client is thread-safe, consider caching and/or
     // injecting it as a singleton into your DI container.

src/Grpc/GrpcTransport.cs

@@ -33,6 +33,7 @@ public async Task<IndexStats> DescribeStats(MetadataMap? filter = null)
         }
 
         using var call = Grpc.DescribeIndexStatsAsync(request, Auth);
+
         return (await call.ConfigureAwait(false)).ToPublicType();
     }
 
@@ -89,6 +90,7 @@ public async Task<uint> Upsert(IEnumerable<Vector> vectors, string? indexNamespa
         request.Vectors.AddRange(vectors.Select(v => v.ToProtoVector()));
 
         using var call = Grpc.UpsertAsync(request, Auth);
+
         return (await call.ConfigureAwait(false)).UpsertedCount;
     }
 

src/Index.cs

@@ -10,9 +10,11 @@ public sealed partial record Index<
 #endif
     TTransport> where TTransport : ITransport<TTransport>
 {
-    [JsonPropertyName("database")]
-    public required IndexDetails Details { get; init; }
-
+    public required string Name { get; init; }
+    public required uint Dimension { get; init; }
+    public required Metric Metric { get; init; }
+    public string? Host { get; init; }
+    public required IndexSpec Spec { get; init; }
     public required IndexStatus Status { get; init; }
 }
 

src/PineconeClient.cs

@@ -10,6 +10,12 @@ namespace Pinecone;
 public sealed class PineconeClient : IDisposable
 {
     private readonly HttpClient Http;
+    private readonly string? _legacyEnvironment;
+
+    public PineconeClient(string apiKey)
+        : this(apiKey, new Uri($"https://api.pinecone.io"))
+    {
+    }
 
     public PineconeClient(string apiKey, string environment)
     {
@@ -18,6 +24,7 @@ public PineconeClient(string apiKey, string environment)
 
         Http = new() { BaseAddress = new Uri($"https://controller.{environment}.pinecone.io") };
         Http.DefaultRequestHeaders.Add("Api-Key", apiKey);
+        _legacyEnvironment = environment;
     }
 
     public PineconeClient(string apiKey, Uri baseUrl)
@@ -38,28 +45,48 @@ public PineconeClient(string apiKey, HttpClient client)
         Http.DefaultRequestHeaders.Add("Api-Key", apiKey);
     }
 
-    public async Task<string[]> ListIndexes()
+    public async Task<IndexDetails[]> ListIndexes()
     {
-        var indexes = await Http
-            .GetFromJsonAsync("/databases", SerializerContext.Default.StringArray)
+        var listIndexesResult = (ListIndexesResult?)await Http
+            .GetFromJsonAsync("/indexes", typeof(ListIndexesResult), SerializerContext.Default)
             .ConfigureAwait(false);
 
-        return indexes ?? [];
+        return listIndexesResult?.Indexes ?? [];
     }
 
-    public Task CreateIndex(string name, uint dimension, Metric metric) =>
-        CreateIndex(new IndexDetails { Name = name, Dimension = dimension, Metric = metric });
+    public Task CreatePodBasedIndex(string name, uint dimiension, Metric metric, string environment, string podType, long pods)
+        => CreateIndexAsync(new CreateIndexRequest
+        {
+            Name = name,
+            Dimension = dimiension,
+            Metric = metric,
+            Spec = new IndexSpec { Pod = new PodSpec { Environment = environment, PodType = podType, Pods = pods } }
+        });
+
+    public Task CreateServerlessIndex(string name, uint dimiension, Metric metric, string cloud, string region)
+        => CreateIndexAsync(new CreateIndexRequest
+        {
+            Name = name,
+            Dimension = dimiension,
+            Metric = metric,
+            Spec = new IndexSpec { Serverless = new ServerlessSpec { Cloud = cloud, Region = region } }
+        });
 
-    public async Task CreateIndex(IndexDetails indexDetails, string? sourceCollection = null)
+    private async Task CreateIndexAsync(CreateIndexRequest request)
     {
-        var request = CreateIndexRequest.From(indexDetails, sourceCollection);
         var response = await Http
-            .PostAsJsonAsync("/databases", request, SerializerContext.Default.CreateIndexRequest)
+            .PostAsJsonAsync("/indexes", request, SerializerContext.Default.CreateIndexRequest)
             .ConfigureAwait(false);
 
         await response.CheckStatusCode().ConfigureAwait(false);
     }
 
+    [Obsolete($"Use '{nameof(CreateServerlessIndex)}' or '{nameof(CreatePodBasedIndex)}' methods instead.")]
+    public Task CreateIndex(string name, uint dimension, Metric metric) 
+        => _legacyEnvironment is not null
+            ? CreatePodBasedIndex(name, dimension, metric, _legacyEnvironment, "starter", 1)
+            : throw new InvalidOperationException($"Use '{nameof(CreateServerlessIndex)}' or '{nameof(CreatePodBasedIndex)}' methods instead.");
+
     public Task<Index<GrpcTransport>> GetIndex(string name) => GetIndex<GrpcTransport>(name);
 
 #if NET7_0_OR_GREATER
@@ -70,22 +97,34 @@ public async Task<Index<TTransport>> GetIndex<
 #endif
         where TTransport : ITransport<TTransport>
     {
-        var response = await Http
+        var response = (IndexDetails)(await Http
             .GetFromJsonAsync(
-                $"/databases/{UrlEncoder.Default.Encode(name)}",
-                typeof(Index<TTransport>),
+                $"/indexes/{UrlEncoder.Default.Encode(name)}",
+                typeof(IndexDetails),
                 SerializerContext.Default)
-            .ConfigureAwait(false) ?? throw new HttpRequestException("GetIndex request has failed.");
+            .ConfigureAwait(false) ?? throw new HttpRequestException("GetIndex request has failed."));
 
-        var index = (Index<TTransport>)response;
-        var host = index.Status.Host;
+        // TODO: Host is optional according to the API spec: https://docs.pinecone.io/reference/api/control-plane/describe_index
+        // but Transport requires it
+        var host = response.Host!;
         var apiKey = Http.DefaultRequestHeaders.GetValues(Constants.RestApiKey).First();
 
+        var index = new Index<TTransport>
+        {
+            Name = response.Name,
+            Dimension = response.Dimension,
+            Metric = response.Metric,
+            Host = response.Host,
+            Spec = response.Spec,
+            Status = response.Status,
+        };
+
 #if NET7_0_OR_GREATER
         index.Transport = TTransport.Create(host, apiKey);
 #else
         index.Transport = ITransport<TTransport>.Create(host, apiKey);
 #endif
+
         return index;
     }
 
@@ -100,7 +139,7 @@ public async Task ConfigureIndex(string name, int? replicas = null, string? podT
         var request = new ConfigureIndexRequest { Replicas = replicas, PodType = podType };
         var response = await Http
             .PatchAsJsonAsync(
-                $"/databases/{UrlEncoder.Default.Encode(name)}",
+                $"/indexes/{UrlEncoder.Default.Encode(name)}",
                 request,
                 SerializerContext.Default.ConfigureIndexRequest)
             .ConfigureAwait(false);
@@ -109,17 +148,17 @@ public async Task ConfigureIndex(string name, int? replicas = null, string? podT
     }
 
     public async Task DeleteIndex(string name) =>
-        await (await Http.DeleteAsync($"/databases/{UrlEncoder.Default.Encode(name)}").ConfigureAwait(false))
+        await (await Http.DeleteAsync($"/indexes/{UrlEncoder.Default.Encode(name)}").ConfigureAwait(false))
             .CheckStatusCode()
             .ConfigureAwait(false);
 
-    public async Task<string[]> ListCollections()
+    public async Task<CollectionDetails[]> ListCollections()
     {
-        var collections = await Http
-            .GetFromJsonAsync("/collections", SerializerContext.Default.StringArray)
+        var listCollectionsResult = (ListCollectionsResult?)await Http
+            .GetFromJsonAsync("/collections", typeof(ListCollectionsResult), SerializerContext.Default)
             .ConfigureAwait(false);
 
-        return collections ?? [];
+        return listCollectionsResult?.Collections ?? [];
     }
 
     public async Task CreateCollection(string name, string source)

src/Rest/SerializerContext.cs

@@ -25,6 +25,8 @@ namespace Pinecone.Rest;
 [JsonSerializable(typeof(DeleteRequest))]
 [JsonSerializable(typeof(MetadataMap))]
 [JsonSerializable(typeof(MetadataValue[]))]
+[JsonSerializable(typeof(ListIndexesResult))]
+[JsonSerializable(typeof(ListCollectionsResult))]
 [JsonSourceGenerationOptions(
     GenerationMode = JsonSourceGenerationMode.Default,
     PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase,

src/Rest/Types.cs

@@ -2,24 +2,12 @@
 
 namespace Pinecone.Rest;
 
-internal sealed record CreateIndexRequest : IndexDetails
+internal sealed record CreateIndexRequest
 {
-    [JsonPropertyName("source_collection")]
-    public string? SourceCollection { get; init; }
-
-    public static CreateIndexRequest From(
-        IndexDetails index,
-        string? sourceCollection) => new()
-    {
-        Dimension = index.Dimension,
-        Metric = index.Metric,
-        Name = index.Name,
-        Pods = index.Pods,
-        PodType = index.PodType,
-        Replicas = index.Replicas,
-        MetadataConfig = index.MetadataConfig,
-        SourceCollection = sourceCollection
-    };
+    public required string Name { get; init; }
+    public required uint Dimension { get; init; }
+    public required Metric Metric { get; init; }
+    public required IndexSpec Spec { get; init; }
 }
 
 internal readonly record struct ConfigureIndexRequest

src/Types/CollectionTypes.cs

@@ -2,12 +2,25 @@
 
 namespace Pinecone;
 
+public record ListCollectionsResult
+{
+    public required CollectionDetails[] Collections { get; init; }
+}
+
 public record CollectionDetails
 {
     public required string Name { get; init; }
-    public required long Size { get; init; }
-    public required string Status { get; init; }
-    public required long Dimension { get; init; }
+    public long? Size { get; init; }
+    public required CollectionStatus Status { get; init; }
+    public required uint Dimension { get; init; }
     [JsonPropertyName("vector_count")]
     public required long VectorCount { get; init; }
+    public required string Environment { get; init; }
+}
+
+public enum CollectionStatus
+{
+    Initializing = 0,
+    Ready = 1,
+    Terminating = 2
 }

src/Types/IndexTypes.cs

@@ -3,17 +3,20 @@
 
 namespace Pinecone;
 
+public record ListIndexesResult
+{
+    public required IndexDetails[] Indexes { get; init; }
+}
+
 public record IndexDetails
 {
     public required string Name { get; init; }
     public required uint Dimension { get; init; }
     public required Metric Metric { get; init; }
-    public long? Pods { get; init; }
-    [JsonPropertyName("pod_type")]
-    public string? PodType { get; init; }
-    public long? Replicas { get; init; }
-    [JsonPropertyName("metadata_config")]
-    public MetadataMap? MetadataConfig { get; init; }
+    public string? Host { get; init; }
+
+    public required IndexSpec Spec { get; init;}
+    public required IndexStatus Status { get; init; }
 }
 
 [JsonConverter(typeof(JsonStringEnumConverter<Metric>))]
@@ -29,10 +32,6 @@ public record IndexStatus
     [JsonPropertyName("ready")]
     public required bool IsReady { get; init; }
     public required IndexState State { get; init; }
-    public required string Host { get; init; }
-    public required int Port { get; init; }
-    public string?[]? Waiting { get; init; }
-    public string?[]? Crashed { get; init; }
 }
 
 [JsonConverter(typeof(JsonStringEnumConverter<IndexState>))]
@@ -48,6 +47,35 @@ public enum IndexState
     InitializationFailed = 7
 }
 
+public record IndexSpec
+{
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public ServerlessSpec? Serverless { get; init; }
+
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public PodSpec? Pod { get; init; }
+}
+
+public record ServerlessSpec
+{
+    public required string Cloud { get; init; }
+    public required string Region { get; init; }
+}
+
+public record PodSpec
+{
+    [JsonPropertyName("environment")]
+    public required string Environment { get; init; }
+    public long? Replicas { get; init; }
+    [JsonPropertyName("pod_type")]
+    public required string PodType { get; init; }
+    public long Pods { get; init; }
+    [JsonPropertyName("metadata_config")]
+    public MetadataMap? MetadataConfig { get; init; }
+    [JsonPropertyName("source_collection")]
+    public string? SourceCollection { get; init; }
+}
+
 public record IndexStats
 {
     [JsonConverter(typeof(IndexNamespaceArrayConverter))]