feat: cuda feature detection.

.github/prepare_release.sh

@@ -63,14 +63,15 @@ elif [[ $type == "patch" ]]; then
         exit 1
     fi
 else
-    echo "Invalid type"
+    echo "Invalid type" 
     exit 1
 fi
 
 cd ..
 # pack the main package
 dotnet pack ./LLama/LLamaSharp.csproj -c Release -o ./temp/ /p:PackageVersion=$updated_version /p:Version=$updated_version;
 dotnet pack ./LLama.SemanticKernel/LLamaSharp.SemanticKernel.csproj -c Release -o ./temp/ /p:PackageVersion=$updated_version /p:Version=$updated_version;
+dotnet pack ./LLama.KernelMemory/LLamaSharp.KernelMemory.csproj -c Release -o ./temp/ /p:PackageVersion=$updated_version /p:Version=$updated_version;
 
 # pack the backends
 cd temp

.github/workflows/release-minor.yml

@@ -32,6 +32,7 @@ jobs:
       run: |
         dotnet build ./LLama/LLamaSharp.csproj -c Release --no-restore
         dotnet build ./LLama.SemanticKernel/LLamaSharp.SemanticKernel.csproj -c Release --no-restore
+        dotnet build ./LLama.KernelMemory/LLamaSharp.KernelMemory.csproj -c Release --no-restore
 
     - name: Pack packages
       run: |

.github/workflows/release-patch.yml

@@ -32,6 +32,7 @@ jobs:
       run: |
         dotnet build ./LLama/LLamaSharp.csproj -c Release --no-restore
         dotnet build ./LLama.SemanticKernel/LLamaSharp.SemanticKernel.csproj -c Release --no-restore
+        dotnet build ./LLama.KernelMemory/LLamaSharp.KernelMemory.csproj -c Release --no-restore
 
     - name: Pack packages
       run: |

LLama.Examples/Program.cs

@@ -7,6 +7,7 @@
 
 Console.WriteLine("======================================================================================================");
 
+NativeLibraryConfig.Default.WithCuda().WithLogs();
 
 NativeApi.llama_empty_call();
 Console.WriteLine();

LLama.KernelMemory/LLamaSharp.KernelMemory.csproj

@@ -1,11 +1,12 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFrameworks>net6.0;net7.0</TargetFrameworks>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
     <Version>0.7.1</Version>
+    <Version>0.8.0</Version>
     <Authors>Xbotter</Authors>
     <Company>SciSharp STACK</Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

LLama.SemanticKernel/LLamaSharp.SemanticKernel.csproj

@@ -10,7 +10,7 @@
 		<ImplicitUsings>enable</ImplicitUsings>
 		<Nullable>enable</Nullable>
 
-    <Version>0.7.1</Version>
+    <Version>0.8.0</Version>
     <Authors>Tim Miller, Xbotter</Authors>
     <Company>SciSharp STACK</Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>

LLama/AntipromptProcessor.cs

@@ -1,66 +1,67 @@
 using System;
 using System.Collections.Generic;
 
-namespace LLama;
-
-internal sealed class AntipromptProcessor
+namespace LLama
 {
-    private int _longestAntiprompt;
-    private readonly List<string> _antiprompts = new();
-
-    private string? _string;
-
-    public AntipromptProcessor(IEnumerable<string>? antiprompts = null)
+    internal sealed class AntipromptProcessor
     {
-        if (antiprompts != null)
-            SetAntiprompts(antiprompts);
-    }
+        private int _longestAntiprompt;
+        private readonly List<string> _antiprompts = new();
 
-    /// <summary>
-    /// Add an antiprompt to the collection
-    /// </summary>
-    /// <param name="antiprompt"></param>
-    public void AddAntiprompt(string antiprompt)
-    {
-        _antiprompts.Add(antiprompt);
-        _longestAntiprompt = Math.Max(_longestAntiprompt, antiprompt.Length);
-    }
+        private string? _string;
 
-    /// <summary>
-    /// Overwrite all current antiprompts with a new set
-    /// </summary>
-    /// <param name="antiprompts"></param>
-    public void SetAntiprompts(IEnumerable<string> antiprompts)
-    {
-        _antiprompts.Clear();
-        _antiprompts.AddRange(antiprompts);
+        public AntipromptProcessor(IEnumerable<string>? antiprompts = null)
+        {
+            if (antiprompts != null)
+                SetAntiprompts(antiprompts);
+        }
 
-        _longestAntiprompt = 0;
-        foreach (var antiprompt in _antiprompts)
+        /// <summary>
+        /// Add an antiprompt to the collection
+        /// </summary>
+        /// <param name="antiprompt"></param>
+        public void AddAntiprompt(string antiprompt)
+        {
+            _antiprompts.Add(antiprompt);
             _longestAntiprompt = Math.Max(_longestAntiprompt, antiprompt.Length);
-    }
+        }
 
-    /// <summary>
-    /// Add some text and check if the buffer now ends with any antiprompt
-    /// </summary>
-    /// <param name="text"></param>
-    /// <returns>true if the text buffer ends with any antiprompt</returns>
-    public bool Add(string text)
-    {
-        _string += text;
+        /// <summary>
+        /// Overwrite all current antiprompts with a new set
+        /// </summary>
+        /// <param name="antiprompts"></param>
+        public void SetAntiprompts(IEnumerable<string> antiprompts)
+        {
+            _antiprompts.Clear();
+            _antiprompts.AddRange(antiprompts);
+
+            _longestAntiprompt = 0;
+            foreach (var antiprompt in _antiprompts)
+                _longestAntiprompt = Math.Max(_longestAntiprompt, antiprompt.Length);
+        }
+
+        /// <summary>
+        /// Add some text and check if the buffer now ends with any antiprompt
+        /// </summary>
+        /// <param name="text"></param>
+        /// <returns>true if the text buffer ends with any antiprompt</returns>
+        public bool Add(string text)
+        {
+            _string += text;
 
-        // When the string gets very long (4x antiprompt length) trim it down (to 2x antiprompt length).
-        // This trimming leaves a lot of extra characters because two sequences can be considered "equal" in unicode
-        // even with different numbers of characters. Hopefully there are enough characters here to handle all those weird circumstances!
-        var maxLength = Math.Max(32, _longestAntiprompt * 4);
-        var trimLength = Math.Max(16, _longestAntiprompt * 2);
-        if (_string.Length > maxLength)
-            _string = _string.Substring(_string.Length - trimLength);
+            // When the string gets very long (4x antiprompt length) trim it down (to 2x antiprompt length).
+            // This trimming leaves a lot of extra characters because two sequences can be considered "equal" in unicode
+            // even with different numbers of characters. Hopefully there are enough characters here to handle all those weird circumstances!
+            var maxLength = Math.Max(32, _longestAntiprompt * 4);
+            var trimLength = Math.Max(16, _longestAntiprompt * 2);
+            if (_string.Length > maxLength)
+                _string = _string.Substring(_string.Length - trimLength);
 
-        foreach (var antiprompt in _antiprompts)
-            if (_string.EndsWith(antiprompt, StringComparison.CurrentCulture))
-                return true;
+            foreach (var antiprompt in _antiprompts)
+                if (_string.EndsWith(antiprompt, StringComparison.CurrentCulture))
+                    return true;
 
-        return false;
+            return false;
+        }
     }
 }

LLama/LLamaSharp.Runtime.targets

@@ -5,39 +5,39 @@
     <ItemGroup Condition="'$(IncludeBuiltInRuntimes)' == 'true'">
         <None Include="$(MSBuildThisFileDirectory)runtimes/libllama.dll">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-            <Link>libllama.dll</Link>
+            <Link>runtimes/win-x64/native/libllama.dll</Link>
         </None>
         <None Include="$(MSBuildThisFileDirectory)runtimes/libllama-cuda11.dll">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-            <Link>libllama-cuda11.dll</Link>
+            <Link>runtimes/win-x64/native/cuda11/libllama.dll</Link>
         </None>
         <None Include="$(MSBuildThisFileDirectory)runtimes/libllama-cuda12.dll">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-            <Link>libllama-cuda12.dll</Link>
+            <Link>runtimes/win-x64/native/cuda12/libllama.dll</Link>
         </None>
         <None Include="$(MSBuildThisFileDirectory)runtimes/libllama.so">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-            <Link>libllama.so</Link>
+            <Link>runtimes/linux-x64/native/libllama.so</Link>
         </None>
         <None Include="$(MSBuildThisFileDirectory)runtimes/libllama-cuda11.so">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-            <Link>libllama-cuda11.so</Link>
+            <Link>runtimes/linux-x64/native/cuda11/libllama.so</Link>
         </None>
         <None Include="$(MSBuildThisFileDirectory)runtimes/libllama-cuda12.so">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-            <Link>libllama-cuda12.so</Link>
+            <Link>runtimes/linux-x64/native/cuda12/libllama.so</Link>
         </None>
         <None Include="$(MSBuildThisFileDirectory)runtimes/osx-arm64/libllama.dylib">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-            <Link>runtimes/osx-arm64/libllama.dylib</Link>
+            <Link>runtimes/osx-arm64/native/libllama.dylib</Link>
         </None>
         <None Include="$(MSBuildThisFileDirectory)runtimes/osx-arm64/ggml-metal.metal">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-            <Link>runtimes/osx-arm64/ggml-metal.metal</Link>
+            <Link>runtimes/osx-arm64/native/ggml-metal.metal</Link>
         </None>
         <None Include="$(MSBuildThisFileDirectory)runtimes/osx-x64/libllama.dylib">
             <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-            <Link>runtimes/osx-x64/libllama.dylib</Link>
+            <Link>runtimes/osx-x64/native/libllama.dylib</Link>
         </None>
     </ItemGroup>
 </Project>

LLama/LLamaSharp.csproj

@@ -7,7 +7,7 @@
     <Platforms>AnyCPU;x64;Arm64</Platforms>
     <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
 
-    <Version>0.5.0</Version>
+    <Version>0.8.0</Version>
     <Authors>Yaohui Liu, Martin Evans, Haiping Chen</Authors>
     <Company>SciSharp STACK</Company>
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
@@ -17,11 +17,11 @@
     <PackageIconUrl>https://avatars3.githubusercontent.com/u/44989469?s=200&amp;v=4</PackageIconUrl>
     <PackageTags>LLama, LLM, GPT, ChatGPT, NLP, AI, Chat Bot, SciSharp</PackageTags>
     <Description>
-      The .NET binding of LLama.cpp, providing APIs to run the model and deploy it on Web. For model
+      The .NET binding of LLama.cpp, making LLM inference and deployment easy and fast. For model
       weights to run, please go to https://github.com/SciSharp/LLamaSharp for more information.
     </Description>
     <PackageReleaseNotes>
-      LLamaSharp 0.5.0 adds support for GGUF, grammar and integration with semantic-kernel.
+      LLamaSharp 0.8.0 supports automatically device feature detection, adds integration with kernel-memory and fixes some performance issues.
     </PackageReleaseNotes>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <PackageOutputPath>packages</PackageOutputPath>