Skip to content

Commit

Permalink
Initial changes required for GGUF support
Browse files Browse the repository at this point in the history
  • Loading branch information
@martindevans
martindevans committed Aug 25, 2023
1 parent da3dffe commit 9da6c61
Show file tree
Hide file tree
Showing 19 changed files with 76 additions and 2,043 deletions.
24 changes: 24 additions & 0 deletions LLama.Examples/LLama.Examples.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,30 @@
<None Update="Assets\reason-act.txt">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/ggml-metal.metal">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama-cuda11.dll">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama-cuda11.so">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama-cuda12.dll">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama-cuda12.so">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama-metal.dylib">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama.dylib">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama.so">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
</ItemGroup>

</Project>
24 changes: 24 additions & 0 deletions LLama.Unittest/LLama.Unittest.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,30 @@
</ItemGroup>

<ItemGroup>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/ggml-metal.metal">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama-cuda11.dll">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama-cuda11.so">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama-cuda12.dll">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama-cuda12.so">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama-metal.dylib">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama.dylib">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="C:\Users\Martin\Documents\dotnet\LLamaSharp\LLama\runtimes/libllama.so">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</None>
<None Update="Models\llama-2-7b-chat.ggmlv3.q3_K_S.bin">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
Expand Down
2 changes: 0 additions & 2 deletions LLama.Unittest/ModelsParamsTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ public void SerializeRoundTripSystemTextJson()
BatchSize = 17,
ContextSize = 42,
LoraAdapter = "adapter",
GroupedQueryAttention = 7,
Seed = 42,
GpuLayerCount = 111
};
Expand All @@ -33,7 +32,6 @@ public void SerializeRoundTripNewtonsoft()
BatchSize = 17,
ContextSize = 42,
LoraAdapter = "adapter",
GroupedQueryAttention = 7,
Seed = 42,
GpuLayerCount = 111
};
Expand Down
10 changes: 0 additions & 10 deletions LLama.Web/Common/ModelOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,16 +88,6 @@ public class ModelOptions
/// </summary>
public float[] TensorSplits { get; set; }

/// <summary>
/// Grouped-Query Attention
/// </summary>
public int GroupedQueryAttention { get; set; } = 1;

/// <summary>
/// RMS Norm Epsilon
/// </summary>
public float RmsNormEpsilon { get; set; } = 5e-6f;

/// <summary>
/// RoPE base frequency
/// </summary>
Expand Down
10 changes: 0 additions & 10 deletions LLama/Abstractions/IModelParams.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,16 +98,6 @@ public interface IModelParams
/// </summary>
float[]? TensorSplits { get; set; }

/// <summary>
/// Grouped-Query Attention
/// </summary>
int GroupedQueryAttention { get; set; }

/// <summary>
/// RMS Norm Epsilon
/// </summary>
float RmsNormEpsilon { get; set; }

/// <summary>
/// RoPE base frequency
/// </summary>
Expand Down
16 changes: 1 addition & 15 deletions LLama/Common/ModelParams.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,16 +89,6 @@ public record ModelParams
/// </summary>
public float[]? TensorSplits { get; set; }

/// <summary>
/// Grouped-Query Attention
/// </summary>
public int GroupedQueryAttention { get; set; } = 1;

/// <summary>
/// RMS Norm Epsilon
/// </summary>
public float RmsNormEpsilon { get; set; } = 5e-6f;

/// <summary>
/// RoPE base frequency
/// </summary>
Expand Down Expand Up @@ -153,8 +143,6 @@ private ModelParams()
/// <param name="batchSize">Batch size for prompt processing (must be >=32 to use BLAS) (n_batch)</param>
/// <param name="convertEosToNewLine">Whether to convert eos to newline during the inference.</param>
/// <param name="embeddingMode">Whether to use embedding mode. (embedding) Note that if this is set to true, The LLamaModel won't produce text response anymore.</param>
/// <param name="groupedQueryAttention">Grouped-Query Attention</param>
/// <param name="rmsNormEpsilon">RMS Norm Epsilon</param>
/// <param name="ropeFrequencyBase">RoPE base frequency.</param>
/// <param name="ropeFrequencyScale">RoPE frequency scaling factor</param>
/// <param name="mulMatQ">Use experimental mul_mat_q kernels</param>
Expand All @@ -165,7 +153,7 @@ public ModelParams(string modelPath, int contextSize = 512, int gpuLayerCount =
bool useMemorymap = true, bool useMemoryLock = false, bool perplexity = false,
string loraAdapter = "", string loraBase = "", int threads = -1, int batchSize = 512,
bool convertEosToNewLine = false, bool embeddingMode = false,
int groupedQueryAttention = 1, float rmsNormEpsilon = 5e-6f, float ropeFrequencyBase = 10000.0f, float ropeFrequencyScale = 1f, bool mulMatQ = false,
float ropeFrequencyBase = 10000.0f, float ropeFrequencyScale = 1f, bool mulMatQ = false,
string encoding = "UTF-8")
{
ContextSize = contextSize;
Expand All @@ -182,8 +170,6 @@ public ModelParams(string modelPath, int contextSize = 512, int gpuLayerCount =
BatchSize = batchSize;
ConvertEosToNewLine = convertEosToNewLine;
EmbeddingMode = embeddingMode;
GroupedQueryAttention = groupedQueryAttention;
RmsNormEpsilon = rmsNormEpsilon;
RopeFrequencyBase = ropeFrequencyBase;
RopeFrequencyScale = ropeFrequencyScale;
MulMatQ = mulMatQ;
Expand Down
2 changes: 0 additions & 2 deletions LLama/Extensions/IModelParamsExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ public static MemoryHandle ToLlamaContextParams(this IModelParams @params, out L
result.logits_all = @params.Perplexity;
result.embedding = @params.EmbeddingMode;
result.low_vram = @params.LowVram;
result.n_gqa = @params.GroupedQueryAttention;
result.rms_norm_eps = @params.RmsNormEpsilon;
result.rope_freq_base = @params.RopeFrequencyBase;
result.rope_freq_scale = @params.RopeFrequencyScale;
result.mul_mat_q = @params.MulMatQ;
Expand Down
42 changes: 16 additions & 26 deletions LLama/Native/LLamaContextParams.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,6 @@ public struct LLamaContextParams
/// </summary>
public int n_batch;

/// <summary>
/// grouped-query attention (TEMP - will be moved to model hparams)
/// </summary>
public int n_gqa;

/// <summary>
/// rms norm epsilon (TEMP - will be moved to model hparams)
/// </summary>
public float rms_norm_eps;

/// <summary>
/// number of layers to store in VRAM
/// </summary>
Expand Down Expand Up @@ -82,8 +72,8 @@ public struct LLamaContextParams
/// if true, reduce VRAM usage at the cost of performance
/// </summary>
public bool low_vram
{
get => Convert.ToBoolean(_low_vram);
{
readonly get => Convert.ToBoolean(_low_vram);
set => _low_vram = Convert.ToSByte(value);
}
private sbyte _low_vram;
Expand All @@ -92,8 +82,8 @@ public bool low_vram
/// if true, use experimental mul_mat_q kernels
/// </summary>
public bool mul_mat_q
{
get => Convert.ToBoolean(_mul_mat_q);
{
readonly get => Convert.ToBoolean(_mul_mat_q);
set => _mul_mat_q = Convert.ToSByte(value);
}
private sbyte _mul_mat_q;
Expand All @@ -102,8 +92,8 @@ public bool mul_mat_q
/// use fp16 for KV cache
/// </summary>
public bool f16_kv
{
get => Convert.ToBoolean(_f16_kv);
{
readonly get => Convert.ToBoolean(_f16_kv);
set => _f16_kv = Convert.ToSByte(value);
}
private sbyte _f16_kv;
Expand All @@ -112,8 +102,8 @@ public bool f16_kv
/// the llama_eval() call computes all logits, not just the last one
/// </summary>
public bool logits_all
{
get => Convert.ToBoolean(_logits_all);
{
readonly get => Convert.ToBoolean(_logits_all);
set => _logits_all = Convert.ToSByte(value);
}
private sbyte _logits_all;
Expand All @@ -122,8 +112,8 @@ public bool logits_all
/// only load the vocabulary, no weights
/// </summary>
public bool vocab_only
{
get => Convert.ToBoolean(_vocab_only);
{
readonly get => Convert.ToBoolean(_vocab_only);
set => _vocab_only = Convert.ToSByte(value);
}
private sbyte _vocab_only;
Expand All @@ -132,8 +122,8 @@ public bool vocab_only
/// use mmap if possible
/// </summary>
public bool use_mmap
{
get => Convert.ToBoolean(_use_mmap);
{
readonly get => Convert.ToBoolean(_use_mmap);
set => _use_mmap = Convert.ToSByte(value);
}
private sbyte _use_mmap;
Expand All @@ -142,8 +132,8 @@ public bool use_mmap
/// force system to keep model in RAM
/// </summary>
public bool use_mlock
{
get => Convert.ToBoolean(_use_mlock);
{
readonly get => Convert.ToBoolean(_use_mlock);
set => _use_mlock = Convert.ToSByte(value);
}
private sbyte _use_mlock;
Expand All @@ -152,8 +142,8 @@ public bool use_mlock
/// embedding mode only
/// </summary>
public bool embedding
{
get => Convert.ToBoolean(_embedding);
{
readonly get => Convert.ToBoolean(_embedding);
set => _embedding = Convert.ToSByte(value);
}
private sbyte _embedding;
Expand Down
5 changes: 5 additions & 0 deletions LLama/Native/LLamaFtype.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,5 +105,10 @@ public enum LLamaFtype
/// </summary>
/// <remarks>Benchmark@7B: 5.15GB, +0.0044 ppl</remarks>
LLAMA_FTYPE_MOSTLY_Q6_K = 18,

/// <summary>
/// File type was not specified
/// </summary>
LLAMA_FTYPE_GUESSED = 1024
}
}
6 changes: 3 additions & 3 deletions LLama/Native/NativeApi.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -377,23 +377,23 @@ public static int llama_tokenize(SafeLLamaContextHandle ctx, string text, Encodi
/// <param name="model"></param>
/// <returns></returns>
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
public static extern int llama_n_vocab_from_model(SafeLlamaModelHandle model);
public static extern int llama_model_n_vocab(SafeLlamaModelHandle model);

/// <summary>
/// Get the size of the context window for the model
/// </summary>
/// <param name="model"></param>
/// <returns></returns>
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
public static extern int llama_n_ctx_from_model(SafeLlamaModelHandle model);
public static extern int llama_model_n_ctx(SafeLlamaModelHandle model);

/// <summary>
/// Get the dimension of embedding vectors from this model
/// </summary>
/// <param name="model"></param>
/// <returns></returns>
[DllImport(libraryName, CallingConvention = CallingConvention.Cdecl)]
public static extern int llama_n_embd_from_model(SafeLlamaModelHandle model);
public static extern int llama_model_n_embd(SafeLlamaModelHandle model);

/// <summary>
/// Convert a single token into text
Expand Down
6 changes: 3 additions & 3 deletions LLama/Native/SafeLlamaModelHandle.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ public sealed class SafeLlamaModelHandle
internal SafeLlamaModelHandle(IntPtr handle)
: base(handle)
{
VocabCount = NativeApi.llama_n_vocab_from_model(this);
ContextSize = NativeApi.llama_n_ctx_from_model(this);
EmbeddingSize = NativeApi.llama_n_embd_from_model(this);
VocabCount = NativeApi.llama_model_n_vocab(this);
ContextSize = NativeApi.llama_model_n_ctx(this);
EmbeddingSize = NativeApi.llama_model_n_embd(this);
}

/// <inheritdoc />
Expand Down
Loading

0 comments on commit 9da6c61

Please sign in to comment.