Merge branch 'master' into alternative_dependency_loading

LLama/Common/Logger.cs

@@ -1,21 +1,44 @@
-using System;
+using LLama.Native;
+using System;
 using System.Diagnostics;
 using System.IO;
 using static LLama.Common.ILLamaLogger;
 
 namespace LLama.Common;
 
+/// <summary>
+/// receives log messages from LLamaSharp
+/// </summary>
 public interface ILLamaLogger
 {
+    /// <summary>
+    /// Severity level of a log message
+    /// </summary>
     public enum LogLevel
     {
-        Info,
-        Debug,
-        Warning,
-        Error
+        /// <summary>
+        /// Logs that are used for interactive investigation during development.
+        /// </summary>
+        Debug = 1,
+
+        /// <summary>
+        /// Logs that highlight when the current flow of execution is stopped due to a failure.
+        /// </summary>
+        Error = 2,
+
+        /// <summary>
+        /// Logs that highlight an abnormal or unexpected event in the application flow, but do not otherwise cause the application execution to stop.
+        /// </summary>
+        Warning = 3,
+
+        /// <summary>
+        /// Logs that track the general flow of the application.
+        /// </summary>
+        Info = 4
     }
+
     /// <summary>
-    /// Write the log in cosutomized way
+    /// Write the log in customized way
     /// </summary>
     /// <param name="source">The source of the log. It may be a method name or class name.</param>
     /// <param name="message">The message.</param>
@@ -24,38 +47,66 @@ public enum LogLevel
 }
 
 /// <summary>
-/// The default logger of LLamaSharp. On default it write to console. User methods of `LLamaLogger.Default` to change the behavior.
-/// It's more recommended to inherit `ILLamaLogger` to cosutomize the behavior.
+/// The default logger of LLamaSharp. On default it write to console. Use methods of `LLamaLogger.Default` to change the behavior.
+/// It's recommended to inherit `ILLamaLogger` to customize the behavior.
 /// </summary>
-public sealed class LLamaDefaultLogger : ILLamaLogger
+public sealed class LLamaDefaultLogger
+    : ILLamaLogger
 {
     private static readonly Lazy<LLamaDefaultLogger> _instance = new Lazy<LLamaDefaultLogger>(() => new LLamaDefaultLogger());
 
     private bool _toConsole = true;
-    private bool _toFile = false;
+    private bool _toFile;
 
-    private FileStream? _fileStream = null;
-    private StreamWriter _fileWriter = null;
+    private FileStream? _fileStream;
+    private StreamWriter? _fileWriter;
 
+    /// <summary>
+    /// Get the default logger instance
+    /// </summary>
     public static LLamaDefaultLogger Default => _instance.Value;
 
     private LLamaDefaultLogger()
     {
 
     }
 
+    /// <summary>
+    /// Enable logging output from llama.cpp
+    /// </summary>
+    /// <returns></returns>
+    public LLamaDefaultLogger EnableNative()
+    {
+        EnableNativeLogCallback();
+        return this;
+    }
+
+    /// <summary>
+    /// Enable writing log messages to console
+    /// </summary>
+    /// <returns></returns>
     public LLamaDefaultLogger EnableConsole()
     {
         _toConsole = true;
         return this;
     }
 
+    /// <summary>
+    /// Disable writing messages to console
+    /// </summary>
+    /// <returns></returns>
     public LLamaDefaultLogger DisableConsole()
     {
         _toConsole = false;
         return this;
     }
 
+    /// <summary>
+    /// Enable writing log messages to file
+    /// </summary>
+    /// <param name="filename"></param>
+    /// <param name="mode"></param>
+    /// <returns></returns>
     public LLamaDefaultLogger EnableFile(string filename, FileMode mode = FileMode.Append)
     {
         _fileStream = new FileStream(filename, mode, FileAccess.Write);
@@ -64,7 +115,22 @@ public LLamaDefaultLogger EnableFile(string filename, FileMode mode = FileMode.A
         return this;
     }
 
+    /// <summary>
+    /// Disable writing log messages to file
+    /// </summary>
+    /// <param name="filename">unused!</param>
+    /// <returns></returns>
+    [Obsolete("Use DisableFile method without 'filename' parameter")]
     public LLamaDefaultLogger DisableFile(string filename)
+    {
+        return DisableFile();
+    }
+
+    /// <summary>
+    /// Disable writing log messages to file
+    /// </summary>
+    /// <returns></returns>
+    public LLamaDefaultLogger DisableFile()
     {
         if (_fileWriter is not null)
         {
@@ -80,6 +146,12 @@ public LLamaDefaultLogger DisableFile(string filename)
         return this;
     }
 
+    /// <summary>
+    /// Log a message
+    /// </summary>
+    /// <param name="source">The source of this message (e.g. class name)</param>
+    /// <param name="message">The message to log</param>
+    /// <param name="level">Severity level of this message</param>
     public void Log(string source, string message, LogLevel level)
     {
         if (level == LogLevel.Info)
@@ -100,6 +172,10 @@ public void Log(string source, string message, LogLevel level)
         }
     }
 
+    /// <summary>
+    /// Write a log message with "Info" severity
+    /// </summary>
+    /// <param name="message"></param>
     public void Info(string message)
     {
         message = MessageFormat("info", message);
@@ -117,6 +193,10 @@ public void Info(string message)
         }
     }
 
+    /// <summary>
+    /// Write a log message with "Warn" severity
+    /// </summary>
+    /// <param name="message"></param>
     public void Warn(string message)
     {
         message = MessageFormat("warn", message);
@@ -134,6 +214,10 @@ public void Warn(string message)
         }
     }
 
+    /// <summary>
+    /// Write a log message with "Error" severity
+    /// </summary>
+    /// <param name="message"></param>
     public void Error(string message)
     {
         message = MessageFormat("error", message);
@@ -151,10 +235,36 @@ public void Error(string message)
         }
     }
 
-    private string MessageFormat(string level, string message)
+    private static string MessageFormat(string level, string message)
     {
-        DateTime now = DateTime.Now;
-        string formattedDate = now.ToString("yyyy.MM.dd HH:mm:ss");
-        return $"[{formattedDate}][{level}]: {message}";
+        var now = DateTime.Now;
+        return $"[{now:yyyy.MM.dd HH:mm:ss}][{level}]: {message}";
     }
+
+    /// <summary>
+    /// Register native logging callback
+    /// </summary>
+	private void EnableNativeLogCallback()
+    {
+        // TODO: Move to a more appropriate place once we have a intitialize method
+        NativeApi.llama_log_set(NativeLogCallback);
+    }
+
+    /// <summary>
+    /// Callback for native logging function
+    /// </summary>
+    /// <param name="level">The log level</param>
+    /// <param name="message">The log message</param>
+    private void NativeLogCallback(LogLevel level, string message)
+    {
+        if (string.IsNullOrEmpty(message))
+            return;
+
+        // Note that text includes the new line character at the end for most events.
+        // If your logging mechanism cannot handle that, check if the last character is '\n' and strip it
+        // if it exists.
+        // It might not exist for progress report where '.' is output repeatedly.
+        Log(default!, message.TrimEnd('\n'), level);
+    }
+
 }

LLama/GlobalSuppressions.cs

@@ -0,0 +1,8 @@
+// This file is used by Code Analysis to maintain SuppressMessage
+// attributes that are applied to this project.
+// Project-level suppressions either have no target or are given
+// a specific target and scoped to a namespace, type, member, etc.
+
+using System.Diagnostics.CodeAnalysis;
+
+[assembly: SuppressMessage("Interoperability", "CA1401:P/Invokes should not be visible", Justification = "LLamaSharp intentionally exports the native llama.cpp API")]

LLama/LLamaQuantizer.cs

@@ -1,8 +1,6 @@
 using LLama.Native;
 using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
 
 namespace LLama
 {
@@ -36,8 +34,7 @@ public static unsafe bool Quantize(string srcFileName, string dstFilename, LLama
             quantizeParams.nthread = nthread;
             quantizeParams.allow_requantize = allowRequantize;
             quantizeParams.quantize_output_tensor = quantizeOutputTensor;
-            LLamaModelQuantizeParams* p = &quantizeParams;
-            return NativeApi.llama_model_quantize(srcFileName, dstFilename, p) == 0;
+            return NativeApi.llama_model_quantize(srcFileName, dstFilename, &quantizeParams) == 0;
         }
 
         /// <summary>
@@ -57,42 +54,71 @@ public static bool Quantize(string srcFileName, string dstFilename, string ftype
             return Quantize(srcFileName, dstFilename, StringToFtype(ftype), nthread, allowRequantize, quantizeOutputTensor);
         }
 
-        private static bool ValidateFtype(string ftype)
-        {
-            return new string[] { "q4_0", "q4_1", "q5_0", "q5_1", "q8_0" }.Contains(ftype);
-        }
-
         private static bool ValidateFtype(LLamaFtype ftype)
         {
-            return ftype is LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1
-                or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_0 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_1 or LLamaFtype.LLAMA_FTYPE_MOSTLY_Q8_0;
-        }
+            // Validation copies from here:
+            // https://github.com/ggerganov/llama.cpp/blob/e59fcb2bc129881f4a269fee748fb38bce0a64de/llama.cpp#L2960
 
-        private static string FtypeToString(LLamaFtype ftype)
-        {
-            return ftype switch
+            switch (ftype)
             {
-                LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0 => "q4_0",
-                LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1 => "q4_1",
-                LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_0 => "q5_0",
-                LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_1 => "q5_1",
-                LLamaFtype.LLAMA_FTYPE_MOSTLY_Q8_0 => "q8_0",
-                _ => throw new ArgumentException($"The type {Enum.GetName(typeof(LLamaFtype), ftype)} is not a valid type " +
-                    $"to perform quantization.")
-            };
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_0:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_1:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q8_0:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_F16:
+                case LLamaFtype.LLAMA_FTYPE_ALL_F32:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q2_K:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q3_K_S:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q3_K_M:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q3_K_L:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_K_S:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_K_M:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_K_S:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_K_M:
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q6_K:
+                    return true;
+
+                case LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16:
+                default:
+                    return false;
+            }
         }
 
+        /// <summary>
+        /// Parse a string into a LLamaFtype. This is a "relaxed" parsing, which allows any string which is contained within
+        /// the enum name to be used.
+        ///
+        /// For example "Q5_K_M" will convert to "LLAMA_FTYPE_MOSTLY_Q5_K_M"
+        /// </summary>
+        /// <param name="str"></param>
+        /// <returns></returns>
+        /// <exception cref="ArgumentException"></exception>
         private static LLamaFtype StringToFtype(string str)
         {
-            return str switch
+            // Find all variants which contain the input string
+            var matches = new List<LLamaFtype>();
+            foreach (LLamaFtype ftype in Enum.GetValues(typeof(LLamaFtype)))
             {
-                "q4_0" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_0,
-                "q4_1" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q4_1,
-                "q5_0" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_0,
-                "q5_1" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q5_1,
-                "q8_0" => LLamaFtype.LLAMA_FTYPE_MOSTLY_Q8_0,
-                _ => throw new ArgumentException($"Invalid ftype {str} to quantize.")
-            };
+                var name = Enum.GetName(typeof(LLamaFtype), ftype);
+
+                // Note: this is using "IndexOf" instead of "Contains" to be compatible with netstandard2.0
+#pragma warning disable CA2249
+                if (name != null && name.IndexOf(str, StringComparison.OrdinalIgnoreCase) >= 0)
+                    matches.Add(ftype);
+#pragma warning restore CA2249
+            }
+
+            // If there was just one match, success!
+            if (matches.Count == 1)
+                return matches[0];
+
+            // If none matched throw a generic error
+            if (matches.Count == 0)
+                throw new ArgumentException($"Unknown ftype \"{str}\" for quantization.");
+
+            // There were several matches, throw an error asking the user to be more specific
+            throw new ArgumentException($"\"{str}\" matches multiple potential ftypes: {string.Join(",", matches)}");
         }
     }
 }