Skip to content

Commit

Permalink
Include comments and include some checks
Browse files Browse the repository at this point in the history
  • Loading branch information
SignalRT committed Mar 20, 2024
1 parent 9f68be3 commit b4fb934
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 3 deletions.
49 changes: 48 additions & 1 deletion LLama/LLavaWeights.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,36 +4,83 @@

namespace LLama;

/// <summary>
/// A set of llava model weights (mmproj), loaded into memory.
/// </summary>
public sealed class LLavaWeights : IDisposable
{
/// <summary>
/// The native handle, which is used in the native APIs
/// </summary>
/// <remarks>Be careful how you use this!</remarks>
public SafeLlavaModelHandle NativeHandle { get; }

internal LLavaWeights(SafeLlavaModelHandle weights)
{
NativeHandle = weights;
}

/// <summary>
/// Load weights into memory
/// </summary>
/// <param name="mmProject">path to the "mmproj" model file</param>
/// <returns></returns>
public static LLavaWeights LoadFromFile(string mmProject)
{
var weights = SafeLlavaModelHandle.LoadFromFile(mmProject, 1);
return new LLavaWeights(weights);
}

public SafeLlavaImageEmbedHandle CreateImageEmbeddings(LLamaContext ctxLlama, Byte[] image )
/// <summary>
/// Create the Image Embeddings from the bytes of an image.
/// </summary>
/// <param name="ctxLlama"></param>
/// <param name="image">Image bytes. Supported formats:
/// <list type="bullet">
/// <item>JPG</item>
/// <item>PNG</item>
/// <item>BMP</item>
/// <item>TGA</item>
/// </list>
/// </param>
/// <returns></returns>
public SafeLlavaImageEmbedHandle CreateImageEmbeddings(LLamaContext ctxLlama, byte[] image )
{
return NativeHandle.CreateImageEmbeddings(ctxLlama, image );
}

/// <summary>
/// Create the Image Embeddings from the bytes of an image.
/// </summary>
/// <param name="ctxLlama"></param>
/// <param name="image">Path to the image file. Supported formats:
/// <list type="bullet">
/// <item>JPG</item>
/// <item>PNG</item>
/// <item>BMP</item>
/// <item>TGA</item>
/// </list>
/// </param>
/// <returns></returns>
/// <exception cref="InvalidOperationException"></exception>
public SafeLlavaImageEmbedHandle CreateImageEmbeddings(LLamaContext ctxLlama, string image )
{
return NativeHandle.CreateImageEmbeddings(ctxLlama, image );
}

/// <summary>
/// Eval the image embeddings
/// </summary>
/// <param name="ctxLlama"></param>
/// <param name="imageEmbed"></param>
/// <param name="n_past"></param>
/// <returns></returns>
public bool EvalImageEmbed(LLamaContext ctxLlama, SafeLlavaImageEmbedHandle imageEmbed, ref int n_past)
{
return NativeHandle.EvalImageEmbed( ctxLlama, imageEmbed, ref n_past );
}

/// <inheritdoc />
public void Dispose()
{
NativeHandle.Dispose();
Expand Down
40 changes: 38 additions & 2 deletions LLama/Native/SafeLlavaImageEmbedHandle.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
namespace LLama.Native
{
/// <summary>
/// A Reference to a set of llava Image Embed handle
/// A Reference to a llava Image Embed handle
/// </summary>
public sealed class SafeLlavaImageEmbedHandle
: SafeLLamaHandleBase
Expand All @@ -24,12 +24,48 @@ private SafeLlavaImageEmbedHandle(IntPtr handle)
private SafeLlavaImageEmbedHandle()
{}

/// <summary>
/// Create an image embed from an image file
/// </summary>
/// <param name="ctxLlava"></param>
/// <param name="ctxLlama"></param>
/// <param name="image">Path to the image file. Supported formats:
/// <list type="bullet">
/// <item>JPG</item>
/// <item>PNG</item>
/// <item>BMP</item>
/// <item>TGA</item>
/// </list>
/// </param>
/// <returns></returns>
/// <exception cref="InvalidOperationException"></exception>
public static SafeLlavaImageEmbedHandle CreateFromFileName( SafeLlavaModelHandle ctxLlava, LLamaContext ctxLlama, string image )
{
// Try to open the image file, this will check:
// - File exists (automatically throws FileNotFoundException)
// - File is readable (explicit check)
// This provides better error messages that llama.cpp, which would throw an access violation exception in both cases.
using (var fs = new FileStream(image, FileMode.Open))
if (!fs.CanRead)
throw new InvalidOperationException($"Llava image file '{image}' is not readable");
return NativeApi.llava_image_embed_make_with_filename(ctxLlava, (int) ctxLlama.BatchThreads, image);
}

public static SafeLlavaImageEmbedHandle CreateFromMemory( SafeLlavaModelHandle ctxLlava, LLamaContext ctxLlama, Byte[] image )
/// <summary>
/// Create an image embed from the bytes of an image.
/// </summary>
/// <param name="ctxLlava"></param>
/// <param name="ctxLlama"></param>
/// <param name="image">Image bytes. Supported formats:
/// <list type="bullet">
/// <item>JPG</item>
/// <item>PNG</item>
/// <item>BMP</item>
/// <item>TGA</item>
/// </list>
/// </param>
/// <returns></returns>
public static SafeLlavaImageEmbedHandle CreateFromMemory( SafeLlavaModelHandle ctxLlava, LLamaContext ctxLlama, byte[] image )
{
return NativeApi.llava_image_embed_make_with_bytes(ctxLlava, (int) ctxLlama.BatchThreads, image, image.Length);
}
Expand Down

0 comments on commit b4fb934

Please sign in to comment.