Improved Tensor Splits

LLama.Web/Common/ModelOptions.cs

@@ -84,7 +84,7 @@ public class ModelOptions : IModelParams
 		/// <summary>
 		/// how split tensors should be distributed across GPUs
 		/// </summary>
-		public nint TensorSplits { get; set; }
+		public float[] TensorSplits { get; set; }
 
 		/// <summary>
 		/// Grouped-Query Attention

LLama/Abstractions/IModelParams.cs

@@ -93,7 +93,7 @@ public interface IModelParams
         /// <summary>
         /// how split tensors should be distributed across GPUs
         /// </summary>
-        nint TensorSplits { get; set; }
+        float[]? TensorSplits { get; set; }
 
         /// <summary>
         /// Grouped-Query Attention

LLama/Common/ModelParams.cs

@@ -1,14 +1,13 @@
 using LLama.Abstractions;
 using System;
-using System.Collections.Generic;
-using System.Text;
 
 namespace LLama.Common
 {
     /// <summary>
     /// The parameters for initializing a LLama model.
     /// </summary>
-    public class ModelParams : IModelParams
+    public class ModelParams
+        : IModelParams
     {
         /// <summary>
         /// Model context size (n_ctx)
@@ -85,7 +84,7 @@ public class ModelParams : IModelParams
         /// <summary>
         /// how split tensors should be distributed across GPUs
         /// </summary>
-        public nint TensorSplits { get; set; }
+        public float[]? TensorSplits { get; set; }
 
 		/// <summary>
 		/// Grouped-Query Attention

LLama/Extensions/IModelParamsExtensions.cs

@@ -0,0 +1,54 @@
+using System.IO;
+using System;
+using System.Buffers;
+using LLama.Abstractions;
+using LLama.Native;
+
+namespace LLama.Extensions
+{
+    internal static class IModelParamsExtensions
+    {
+        /// <summary>
+        /// Convert the given `IModelParams` into a `LLamaContextParams`
+        /// </summary>
+        /// <param name="params"></param>
+        /// <param name="result"></param>
+        /// <returns></returns>
+        /// <exception cref="FileNotFoundException"></exception>
+        /// <exception cref="ArgumentException"></exception>
+        public static MemoryHandle ToLlamaContextParams(this IModelParams @params, out LLamaContextParams result)
+        {
+            if (!File.Exists(@params.ModelPath))
+                throw new FileNotFoundException($"The model file does not exist: {@params.ModelPath}");
+
+            if (@params.TensorSplits != null && @params.TensorSplits.Length != 1)
+                throw new ArgumentException("Currently multi-gpu support is not supported by both llama.cpp and LLamaSharp.");
+
+            result = NativeApi.llama_context_default_params();
+            result.n_ctx = @params.ContextSize;
+            result.n_batch = @params.BatchSize;
+            result.main_gpu = @params.MainGpu;
+            result.n_gpu_layers = @params.GpuLayerCount;
+            result.seed = @params.Seed;
+            result.f16_kv = @params.UseFp16Memory;
+            result.use_mmap = @params.UseMemoryLock;
+            result.use_mlock = @params.UseMemoryLock;
+            result.logits_all = @params.Perplexity;
+            result.embedding = @params.EmbeddingMode;
+            result.low_vram = @params.LowVram;
+            result.n_gqa = @params.GroupedQueryAttention;
+            result.rms_norm_eps = @params.RmsNormEpsilon;
+            result.rope_freq_base = @params.RopeFrequencyBase;
+            result.rope_freq_scale = @params.RopeFrequencyScale;
+            result.mul_mat_q = @params.MulMatQ;
+
+            var pin = @params.TensorSplits.AsMemory().Pin();
+            unsafe
+            {
+                result.tensor_split = (nint)pin.Pointer;
+            }
+
+            return pin;
+        }
+    }
+}

LLama/Utils.cs

@@ -1,12 +1,12 @@
 using LLama.Abstractions;
-using LLama.Exceptions;
 using LLama.Native;
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Text;
+using LLama.Exceptions;
+using LLama.Extensions;
 
 namespace LLama
 {
@@ -15,46 +15,16 @@
     {
         public static SafeLLamaContextHandle InitLLamaContextFromModelParams(IModelParams @params)
         {
-            var lparams = NativeApi.llama_context_default_params();
-
-            lparams.n_ctx = @params.ContextSize;
-            lparams.n_batch = @params.BatchSize;
-            lparams.main_gpu = @params.MainGpu;
-            lparams.n_gpu_layers = @params.GpuLayerCount;
-            lparams.seed = @params.Seed;
-            lparams.f16_kv = @params.UseFp16Memory;
-            lparams.use_mmap = @params.UseMemoryLock;
-            lparams.use_mlock = @params.UseMemoryLock;
-            lparams.logits_all = @params.Perplexity;
-            lparams.embedding = @params.EmbeddingMode;
-            lparams.low_vram = @params.LowVram;
-            lparams.n_gqa = @params.GroupedQueryAttention;
-            lparams.rms_norm_eps = @params.RmsNormEpsilon;
-            lparams.rope_freq_base = @params.RopeFrequencyBase;
-            lparams.rope_freq_scale = @params.RopeFrequencyScale;
-            lparams.mul_mat_q = @params.MulMatQ;
-
-            /*
-            if (@params.TensorSplits.Length != 1)
+            using (@params.ToLlamaContextParams(out var lparams))
             {
-                throw new ArgumentException("Currently multi-gpu support is not supported by " +
-                    "both llama.cpp and LLamaSharp.");
-            }*/
+                var model = SafeLlamaModelHandle.LoadFromFile(@params.ModelPath, lparams);
+                var ctx = SafeLLamaContextHandle.Create(model, lparams);
 
-            lparams.tensor_split = @params.TensorSplits;
+                if (!string.IsNullOrEmpty(@params.LoraAdapter))
+                    model.ApplyLoraFromFile(@params.LoraAdapter, @params.LoraBase, @params.Threads);
 
-            if (!File.Exists(@params.ModelPath))
-            {
-                throw new FileNotFoundException($"The model file does not exist: {@params.ModelPath}");
+                return ctx;
             }
-
-            var model = SafeLlamaModelHandle.LoadFromFile(@params.ModelPath, lparams);
-            var ctx = SafeLLamaContextHandle.Create(model, lparams);
-
-            if (!string.IsNullOrEmpty(@params.LoraAdapter))
-                model.ApplyLoraFromFile(@params.LoraAdapter, @params.LoraBase, @params.Threads);
-
-            return ctx;
         }
 
         public static IEnumerable<llama_token> Tokenize(SafeLLamaContextHandle ctx, string text, bool add_bos, Encoding encoding)
@@ -96,15 +66,15 @@
 #if NET6_0_OR_GREATER
             if(encoding == Encoding.UTF8)
             {
                 return Marshal.PtrToStringUTF8(ptr);
             }
             else if(encoding == Encoding.Unicode)
             {
                 return Marshal.PtrToStringUni(ptr);
             }
             else
             {
                 return Marshal.PtrToStringAuto(ptr);
             }
 #else
             byte* tp = (byte*)ptr.ToPointer();