-
-
-
-
-
-@{ await Html.RenderPartialAsync("_ChatTemplates"); }
-
-@section Scripts {
-
-
-}
\ No newline at end of file
diff --git a/LLama.Web/Pages/Executor/Instruct.cshtml.cs b/LLama.Web/Pages/Executor/Instruct.cshtml.cs
deleted file mode 100644
index 18a58253b..000000000
--- a/LLama.Web/Pages/Executor/Instruct.cshtml.cs
+++ /dev/null
@@ -1,34 +0,0 @@
-using LLama.Web.Common;
-using LLama.Web.Models;
-using LLama.Web.Services;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.Mvc.RazorPages;
-using Microsoft.Extensions.Options;
-
-namespace LLama.Web.Pages
-{
- public class InstructModel : PageModel
- {
- private readonly ILogger
-
-
-
-
-
- Instruct
-
- Hub: Disconnected
-
-
- Model
-
-
-
-
- Parameters
-
-
-
-
- Prompt
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-@{ await Html.RenderPartialAsync("_ChatTemplates");}
-
-@section Scripts {
-
-
-}
\ No newline at end of file
diff --git a/LLama.Web/Pages/Executor/Interactive.cshtml.cs b/LLama.Web/Pages/Executor/Interactive.cshtml.cs
deleted file mode 100644
index 7179a4405..000000000
--- a/LLama.Web/Pages/Executor/Interactive.cshtml.cs
+++ /dev/null
@@ -1,34 +0,0 @@
-using LLama.Web.Common;
-using LLama.Web.Models;
-using LLama.Web.Services;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.Mvc.RazorPages;
-using Microsoft.Extensions.Options;
-
-namespace LLama.Web.Pages
-{
- public class InteractiveModel : PageModel
- {
- private readonly ILogger
-
-
-
-
-
- Interactive
-
- Hub: Disconnected
-
-
- Model
-
-
-
-
- Parameters
-
-
-
-
- Prompt
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-@{ await Html.RenderPartialAsync("_ChatTemplates"); }
-
-
-@section Scripts {
-
-
-}
\ No newline at end of file
diff --git a/LLama.Web/Pages/Executor/Stateless.cshtml.cs b/LLama.Web/Pages/Executor/Stateless.cshtml.cs
deleted file mode 100644
index f88c4b832..000000000
--- a/LLama.Web/Pages/Executor/Stateless.cshtml.cs
+++ /dev/null
@@ -1,34 +0,0 @@
-using LLama.Web.Common;
-using LLama.Web.Models;
-using LLama.Web.Services;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.AspNetCore.Mvc.RazorPages;
-using Microsoft.Extensions.Options;
-
-namespace LLama.Web.Pages
-{
- public class StatelessModel : PageModel
- {
- private readonly ILogger
-
-
-
-
-
- Stateless
-
- Hub: Disconnected
-
-
- Model
-
-
-
-
- Parameters
-
-
-
-
- Prompt
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- _logger;
+ private readonly ConnectionSessionService _modelSessionService;
- public IndexModel(ILogger logger)
+ public IndexModel(ILogger logger, IOptions options, ConnectionSessionService modelSessionService)
{
_logger = logger;
+ Options = options.Value;
+ _modelSessionService = modelSessionService;
}
+ public LLamaOptions Options { get; set; }
+
+ [BindProperty]
+ public CreateSessionModel SessionOptions { get; set; }
+
public void OnGet()
{
+ SessionOptions = new CreateSessionModel
+ {
+ Prompt = "Below is an instruction that describes a task. Write a response that appropriately completes the request.",
+ AntiPrompt = "User:",
+ OutputFilter = "User:, Response:"
+ };
+ }
+ public async Task OnPostCancel(CancelModel model)
+ {
+ await _modelSessionService.CancelAsync(model.ConnectionId);
+ return new JsonResult(default);
}
}
}
\ No newline at end of file
diff --git a/LLama.Web/Pages/Shared/_ChatTemplates.cshtml b/LLama.Web/Pages/Shared/_ChatTemplates.cshtml
index 156440124..cd768f1f5 100644
--- a/LLama.Web/Pages/Shared/_ChatTemplates.cshtml
+++ b/LLama.Web/Pages/Shared/_ChatTemplates.cshtml
@@ -12,7 +12,7 @@
Welcome
-Learn about building Web apps with ASP.NET Core.
+@Html.AntiForgeryToken() +
+
+
+
+@{
+ await Html.RenderPartialAsync("_ChatTemplates");
+}
+
+@section Scripts {
+
+
+}
\ No newline at end of file
diff --git a/LLama.Web/Pages/Index.cshtml.cs b/LLama.Web/Pages/Index.cshtml.cs
index 477c9bfbe..8cd694bfa 100644
--- a/LLama.Web/Pages/Index.cshtml.cs
+++ b/LLama.Web/Pages/Index.cshtml.cs
@@ -1,20 +1,43 @@
-using Microsoft.AspNetCore.Mvc;
+using LLama.Web.Common;
+using LLama.Web.Models;
+using LLama.Web.Services;
+using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.RazorPages;
+using Microsoft.Extensions.Options;
namespace LLama.Web.Pages
{
public class IndexModel : PageModel
{
private readonly ILogger
+
+
+
+
+
+
+ @ViewData["Title"]
+
+
+ Socket: Disconnected
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
- {{text}}
+ {{text}}
{{date}}
@@ -26,9 +26,7 @@
-
- 
-
+
();
builder.Services.AddSingleton();
var app = builder.Build();
diff --git a/LLama.Web/README.md b/LLama.Web/README.md
index 9b6786e6b..bee508652 100644
--- a/LLama.Web/README.md
+++ b/LLama.Web/README.md
@@ -1,41 +1,45 @@
## LLama.Web - Basic ASP.NET Core examples of LLamaSharp in action
-LLama.Web has no heavy dependencies and no extra frameworks ove bootstrap and jquery to keep the examples clean and easy to copy over to your own project
+LLama.Web has no heavy dependencies and no extra frameworks except bootstrap and jquery to keep the examples clean and easy to copy over to your own project
+
## Websockets
Using signalr websockets simplifys the streaming of responses and model per connection management
-
## Setup
-You can setup Models, Prompts and Inference parameters in the appsettings.json
+You can setup Models, Ports etc in the appsettings.json
**Models**
You can add multiple models to the options for quick selection in the UI, options are based on ModelParams so its fully configurable
-**Parameters**
-You can add multiple sets of inference parameters to the options for quick selection in the UI, options are based on InferenceParams so its fully configurable
-
-**Prompts**
-You can add multiple sets of prompts to the options for quick selection in the UI
Example:
```json
- {
- "Name": "Alpaca",
- "Path": "D:\\Repositories\\AI\\Prompts\\alpaca.txt",
- "Prompt": "Alternativly to can set a prompt text directly and omit the Path"
- "AntiPrompt": [
- "User:"
- ],
- "OutputFilter": [
- "Response:",
- "User:"
- ]
+ "LLamaOptions": {
+ "Models": [
+ {
+ "Name": "WizardLM-7B",
+ "MaxInstances": 2,
+ "ModelPath": "\\Models\\wizardLM-7B.ggmlv3.q4_0.bin",
+ "ContextSize": 2048
+ },
+ {
+ "Name": "WizardLM-13B",
+ "MaxInstances": 2,
+ "ModelPath": "\\Models\\wizardLM-13B.ggmlv3.q4_0.bin",
+ "ContextSize": 1024,
+ "GpuLayerCount": 16,
+ "Threads": 15
+ }
+ ]
}
```
-## Interactive UI
-The interactive UI is a simple example of using LLammaSharp
-
+## Inference Demo
+The Inference Demo UI is a simple example of using LLamaSharp
+
+
+Inference Parameters
+
diff --git a/LLama.Web/Services/ConnectionSessionService.cs b/LLama.Web/Services/ConnectionSessionService.cs
index 6c266f14f..c94968b6b 100644
--- a/LLama.Web/Services/ConnectionSessionService.cs
+++ b/LLama.Web/Services/ConnectionSessionService.cs
@@ -3,7 +3,7 @@
using LLama.Web.Models;
using Microsoft.Extensions.Options;
using System.Collections.Concurrent;
-using System.Drawing;
+using System.Diagnostics;
namespace LLama.Web.Services
{
@@ -15,80 +15,162 @@ public class ConnectionSessionService : IModelSessionService
{
private readonly LLamaOptions _options;
private readonly ILogger _logger;
+ private readonly IModelCacheService _modelCacheService;
private readonly ConcurrentDictionary _modelSessions;
- public ConnectionSessionService(ILogger logger, IOptions options)
+
+ public ConnectionSessionService(ILogger logger, IOptions options, IModelCacheService modelCacheService)
{
_logger = logger;
_options = options.Value;
+ _modelCacheService = modelCacheService;
_modelSessions = new ConcurrentDictionary();
}
- public Task GetAsync(string connectionId)
+ public Task GetAsync(string sessionId)
{
- _modelSessions.TryGetValue(connectionId, out var modelSession);
+ _modelSessions.TryGetValue(sessionId, out var modelSession);
return Task.FromResult(modelSession);
}
- public Task> CreateAsync(LLamaExecutorType executorType, string connectionId, string modelName, string promptName, string parameterName)
- {
- var modelOption = _options.Models.FirstOrDefault(x => x.Name == modelName);
- if (modelOption is null)
- return Task.FromResult(ServiceResult.FromError($"Model option '{modelName}' not found"));
- var promptOption = _options.Prompts.FirstOrDefault(x => x.Name == promptName);
- if (promptOption is null)
- return Task.FromResult(ServiceResult.FromError($"Prompt option '{promptName}' not found"));
+ public async Task> CreateAsync(string sessionId, CreateSessionModel sessionModel)
+ {
+ // Remove existing connections session
+ await RemoveAsync(sessionId);
- var parameterOption = _options.Parameters.FirstOrDefault(x => x.Name == parameterName);
- if (parameterOption is null)
- return Task.FromResult(ServiceResult.FromError($"Parameter option '{parameterName}' not found"));
+ var modelOption = _options.Models.FirstOrDefault(x => x.Name == sessionModel.Model);
+ if (modelOption is null)
+ return ServiceResult.FromError($"Model option '{sessionModel.Model}' not found");
//Max instance
var currentInstances = _modelSessions.Count(x => x.Value.ModelName == modelOption.Name);
if (modelOption.MaxInstances > -1 && currentInstances >= modelOption.MaxInstances)
- return Task.FromResult(ServiceResult.FromError("Maximum model instances reached"));
+ return ServiceResult.FromError("Maximum model instances reached");
- // Create model
- var llamaModel = new LLamaModel(modelOption);
+ // Create Model/Context
+ var llamaModelContext = await CreateModelContext(sessionId, modelOption);
// Create executor
- ILLamaExecutor executor = executorType switch
+ ILLamaExecutor executor = sessionModel.ExecutorType switch
{
- LLamaExecutorType.Interactive => new InteractiveExecutor(llamaModel),
- LLamaExecutorType.Instruct => new InstructExecutor(llamaModel),
- LLamaExecutorType.Stateless => new StatelessExecutor(llamaModel),
+ LLamaExecutorType.Interactive => new InteractiveExecutor(llamaModelContext),
+ LLamaExecutorType.Instruct => new InstructExecutor(llamaModelContext),
+ LLamaExecutorType.Stateless => new StatelessExecutor(llamaModelContext),
_ => default
};
+ // Create Prompt
+ var promptOption = new PromptOptions
+ {
+ Name = "Custom",
+ Prompt = sessionModel.Prompt,
+ AntiPrompt = CreateListFromCSV(sessionModel.AntiPrompt),
+ OutputFilter = CreateListFromCSV(sessionModel.OutputFilter),
+ };
+
// Create session
- var modelSession = new ModelSession(executor, modelOption, promptOption, parameterOption);
- if (!_modelSessions.TryAdd(connectionId, modelSession))
- return Task.FromResult(ServiceResult.FromError("Failed to create model session"));
+ var modelSession = new ModelSession(executor, modelOption, promptOption, sessionModel);
+ if (!_modelSessions.TryAdd(sessionId, modelSession))
+ return ServiceResult.FromError("Failed to create model session");
- return Task.FromResult(ServiceResult.FromValue(modelSession));
+ return ServiceResult.FromValue(default);
}
- public Task RemoveAsync(string connectionId)
+
+ public async IAsyncEnumerable InferAsync(string sessionId, string prompt, CancellationTokenSource cancellationTokenSource)
{
- if (_modelSessions.TryRemove(connectionId, out var modelSession))
+ var modelSession = await GetAsync(sessionId);
+ if (modelSession is null)
+ yield break;
+
+ // Create unique response id
+ var responseId = Guid.NewGuid().ToString();
+
+ // Send begin of response
+ var stopwatch = Stopwatch.GetTimestamp();
+ yield return new ResponseFragment
+ {
+ Id = responseId,
+ IsFirst = true
+ };
+
+ // Send content of response
+ await foreach (var fragment in modelSession.InferAsync(prompt, cancellationTokenSource))
+ {
+ yield return new ResponseFragment
+ {
+ Id = responseId,
+ Content = fragment
+ };
+ }
+
+ // Send end of response
+ var elapsedTime = Stopwatch.GetElapsedTime(stopwatch);
+ var signature = modelSession.IsInferCanceled()
+ ? $"Inference cancelled after {elapsedTime.TotalSeconds:F0} seconds"
+ : $"Inference completed in {elapsedTime.TotalSeconds:F0} seconds";
+ yield return new ResponseFragment
+ {
+ Id = responseId,
+ IsLast = true,
+ Content = signature,
+ IsCancelled = modelSession.IsInferCanceled(),
+ Elapsed = (int)elapsedTime.TotalMilliseconds
+ };
+ }
+
+
+ public async Task RemoveAsync(string sessionId)
+ {
+ if (_modelSessions.TryRemove(sessionId, out var modelSession))
{
modelSession.CancelInfer();
- modelSession.Dispose();
- return Task.FromResult(true);
+ var llamaModel = await _modelCacheService.Get(modelSession.ModelName);
+ if (llamaModel is null)
+ return false;
+
+ return await llamaModel.RemoveContext(sessionId);
}
- return Task.FromResult(false);
+ return false;
}
- public Task CancelAsync(string connectionId)
+ public Task CancelAsync(string sessionId)
{
- if (_modelSessions.TryGetValue(connectionId, out var modelSession))
+ if (_modelSessions.TryGetValue(sessionId, out var modelSession))
{
modelSession.CancelInfer();
return Task.FromResult(true);
}
return Task.FromResult(false);
}
+
+ private async Task CreateModelContext(string sessionId, ModelOptions modelOption)
+ {
+ // Create model
+ var llamaModel = await _modelCacheService.Get(modelOption.Name)
+ ?? await _modelCacheService.Create(modelOption);
+ if (llamaModel is null)
+ throw new Exception($"Failed to create model, modelName: {modelOption.Name}");
+
+ //Create context
+ var llamaModelContext = await llamaModel.GetContext(sessionId)
+ ?? await llamaModel.CreateContext(sessionId);
+ if (llamaModelContext is null)
+ throw new Exception($"Failed to create model, connectionId: {sessionId}");
+
+ return llamaModelContext;
+ }
+
+ private List CreateListFromCSV(string csv)
+ {
+ if(string.IsNullOrEmpty(csv))
+ return null;
+
+ return csv.Split(",")
+ .Select(x => x.Trim())
+ .ToList();
+ }
}
}
diff --git a/LLama.Web/Services/IModelCacheService.cs b/LLama.Web/Services/IModelCacheService.cs
new file mode 100644
index 000000000..ed627cbae
--- /dev/null
+++ b/LLama.Web/Services/IModelCacheService.cs
@@ -0,0 +1,11 @@
+using LLama.Web.Common;
+
+namespace LLama.Web.Services
+{
+ public interface IModelCacheService
+ {
+ Task Create(ModelOptions modelOptions);
+ Task Get(string modelName);
+ Task Remove(string modelName);
+ }
+}
\ No newline at end of file
diff --git a/LLama.Web/Services/IModelSessionService.cs b/LLama.Web/Services/IModelSessionService.cs
index 4ee0d483f..fb177ccf1 100644
--- a/LLama.Web/Services/IModelSessionService.cs
+++ b/LLama.Web/Services/IModelSessionService.cs
@@ -7,10 +7,9 @@ namespace LLama.Web.Services
public interface IModelSessionService
{
Task GetAsync(string sessionId);
- Task> CreateAsync(LLamaExecutorType executorType, string sessionId, string modelName, string promptName, string parameterName);
+ Task> CreateAsync(string sessionId, CreateSessionModel sessionModel);
Task RemoveAsync(string sessionId);
Task CancelAsync(string sessionId);
}
-
}
diff --git a/LLama.Web/Services/ModelCacheService.cs b/LLama.Web/Services/ModelCacheService.cs
new file mode 100644
index 000000000..f2d1f7ed5
--- /dev/null
+++ b/LLama.Web/Services/ModelCacheService.cs
@@ -0,0 +1,40 @@
+using System.Collections.Concurrent;
+using LLama.Web.Services;
+
+namespace LLama.Web.Common
+{
+ public class ModelCacheService : IModelCacheService
+ {
+ private readonly ConcurrentDictionary _modelInstances = new ConcurrentDictionary();
+
+ public Task Create(ModelOptions modelOptions)
+ {
+ if (_modelInstances.TryGetValue(modelOptions.Name, out LLamaModel model))
+ return Task.FromResult(model);
+
+ model = new LLamaModel(modelOptions);
+ if (!_modelInstances.TryAdd(modelOptions.Name, model))
+ throw new Exception($"Failed to cache model {modelOptions.Name}.");
+
+ return Task.FromResult(model);
+ }
+
+
+ public Task Get(string modelName)
+ {
+ _modelInstances.TryGetValue(modelName, out LLamaModel model);
+ return Task.FromResult(model);
+ }
+
+
+ public Task Remove(string modelName)
+ {
+ if (_modelInstances.TryRemove(modelName, out LLamaModel model))
+ {
+ model?.Dispose();
+ return Task.FromResult(true);
+ }
+ return Task.FromResult(false);
+ }
+ }
+}
diff --git a/LLama.Web/appsettings.json b/LLama.Web/appsettings.json
index 9f340a9c7..652fd36ba 100644
--- a/LLama.Web/appsettings.json
+++ b/LLama.Web/appsettings.json
@@ -6,49 +6,15 @@
}
},
"AllowedHosts": "*",
+ "Urls": "https://localhost:5000",
"LLamaOptions": {
"Models": [
{
"Name": "WizardLM-7B",
"MaxInstances": 2,
- "ModelPath": "D:\\Repositories\\AI\\Models\\wizardLM-7B.ggmlv3.q4_0.bin",
+ "ModelPath": "\\Models\\wizardLM-7B.ggmlv3.q4_0.bin",
"ContextSize": 2048
}
- ],
- "Parameters": [
- {
- "Name": "Default",
- "Temperature": 0.6
- }
- ],
- "Prompts": [
- {
- "Name": "None",
- "Prompt": ""
- },
- {
- "Name": "Alpaca",
- "Path": "D:\\Repositories\\AI\\Prompts\\alpaca.txt",
- "AntiPrompt": [
- "User:"
- ],
- "OutputFilter": [
- "Response:",
- "User:"
- ]
- },
- {
- "Name": "ChatWithBob",
- "Path": "D:\\Repositories\\AI\\Prompts\\chat-with-bob.txt",
- "AntiPrompt": [
- "User:"
- ],
- "OutputFilter": [
- "Bob:",
- "User:"
- ]
- }
]
-
}
}
diff --git a/LLama.Web/wwwroot/css/site.css b/LLama.Web/wwwroot/css/site.css
index d10ef9757..a31573cf2 100644
--- a/LLama.Web/wwwroot/css/site.css
+++ b/LLama.Web/wwwroot/css/site.css
@@ -1,4 +1,4 @@
-html, body {
+html, body {
font-size: 14px;
height: 100%;
display: flex;
@@ -31,4 +31,21 @@ footer {
box-shadow: 0 0 0 0.1rem white, 0 0 0 0.25rem #258cfb;
}
+#scroll-container {
+ flex: 1;
+ overflow-y: scroll;
+}
+
+#output-container .content {
+ white-space: break-spaces;
+}
+
+
+.slider-container > .slider {
+ width: 100%;
+}
+.slider-container > label {
+ width: 50px;
+ text-align: center;
+}
\ No newline at end of file
diff --git a/LLama.Web/wwwroot/js/sessionConnectionChat.js b/LLama.Web/wwwroot/js/sessionConnectionChat.js
index 472b59718..99c7a3418 100644
--- a/LLama.Web/wwwroot/js/sessionConnectionChat.js
+++ b/LLama.Web/wwwroot/js/sessionConnectionChat.js
@@ -1,9 +1,9 @@
-const createConnectionSessionChat = (LLamaExecutorType) => {
+const createConnectionSessionChat = () => {
const outputErrorTemplate = $("#outputErrorTemplate").html();
const outputInfoTemplate = $("#outputInfoTemplate").html();
const outputUserTemplate = $("#outputUserTemplate").html();
const outputBotTemplate = $("#outputBotTemplate").html();
- const sessionDetailsTemplate = $("#sessionDetailsTemplate").html();
+ const signatureTemplate = $("#signatureTemplate").html();
let connectionId;
const connection = new signalR.HubConnectionBuilder().withUrl("/SessionConnectionHub").build();
@@ -20,7 +20,6 @@ const createConnectionSessionChat = (LLamaExecutorType) => {
}
else if (status == Enums.SessionConnectionStatus.Loaded) {
enableControls();
- $("#session-details").html(Mustache.render(sessionDetailsTemplate, { model: getSelectedModel(), prompt: getSelectedPrompt(), parameter: getSelectedParameter() }));
onInfo(`New model session successfully started`)
}
}
@@ -53,7 +52,7 @@ const createConnectionSessionChat = (LLamaExecutorType) => {
if (response.isLast) {
enableControls();
- responseContainer.find(".signature").append(response.content);
+ responseContainer.find(".signature").append(Mustache.render(signatureTemplate, response));
scrollToBottom();
}
else {
@@ -71,10 +70,10 @@ const createConnectionSessionChat = (LLamaExecutorType) => {
const sendPrompt = async () => {
const text = chatInput.val();
if (text) {
+ chatInput.val(null);
disableControls();
outputContainer.append(Mustache.render(outputUserTemplate, { text: text, date: getDateTime() }));
await connection.invoke('SendPrompt', text);
- chatInput.val(null);
scrollToBottom(true);
}
}
@@ -84,16 +83,32 @@ const createConnectionSessionChat = (LLamaExecutorType) => {
}
const loadModel = async () => {
- const modelName = getSelectedModel();
- const promptName = getSelectedPrompt();
- const parameterName = getSelectedParameter();
- if (!modelName || !promptName || !parameterName) {
- onError("Please select a valid Model, Parameter and Prompt");
- return;
- }
-
disableControls();
- await connection.invoke('LoadModel', LLamaExecutorType, modelName, promptName, parameterName);
+ await connection.invoke('LoadModel', serializeFormToJson('SessionParameters'));
+ }
+
+
+ const serializeFormToJson = (form) => {
+ const formDataJson = {};
+ const formData = new FormData(document.getElementById(form));
+ formData.forEach((value, key) => {
+
+ if (key.includes("."))
+ key = key.split(".")[1];
+
+ // Convert number strings to numbers
+ if (!isNaN(value) && value.trim() !== "") {
+ formDataJson[key] = parseFloat(value);
+ }
+ // Convert boolean strings to booleans
+ else if (value === "true" || value === "false") {
+ formDataJson[key] = (value === "true");
+ }
+ else {
+ formDataJson[key] = value;
+ }
+ });
+ return formDataJson;
}
@@ -118,21 +133,6 @@ const createConnectionSessionChat = (LLamaExecutorType) => {
}
- const getSelectedModel = () => {
- return $("option:selected", "#Model").val();
- }
-
-
- const getSelectedParameter = () => {
- return $("option:selected", "#Parameter").val();
- }
-
-
- const getSelectedPrompt = () => {
- return $("option:selected", "#Prompt").val();
- }
-
-
const getDateTime = () => {
const dateTime = new Date();
return dateTime.toLocaleString();
@@ -165,7 +165,10 @@ const createConnectionSessionChat = (LLamaExecutorType) => {
sendPrompt();
}
});
-
+ $(".slider").on("input", function (e) {
+ const slider = $(this);
+ slider.next().text(slider.val());
+ }).trigger("input");
// Map signalr functions
diff --git a/LLama.WebAPI/Services/StatefulChatService.cs b/LLama.WebAPI/Services/StatefulChatService.cs
index ab89b5176..d2a54f702 100644
--- a/LLama.WebAPI/Services/StatefulChatService.cs
+++ b/LLama.WebAPI/Services/StatefulChatService.cs
@@ -1,7 +1,5 @@
using LLama.WebAPI.Models;
-using Microsoft;
-using System.Runtime.CompilerServices;
namespace LLama.WebAPI.Services;
@@ -9,6 +7,7 @@ public class StatefulChatService : IDisposable
{
private readonly ChatSession _session;
private readonly LLamaModel _model;
+ private readonly LLamaModelContext _context;
private bool _continue = false;
private const string SystemPrompt = "Transcript of a dialog, where the User interacts with an Assistant. Assistant is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.\n\n"
@@ -17,12 +16,13 @@ public class StatefulChatService : IDisposable
public StatefulChatService(IConfiguration configuration)
{
_model = new LLamaModel(new Common.ModelParams(configuration["ModelPath"], contextSize: 512));
- _session = new ChatSession(new InteractiveExecutor(_model));
+ _context = new LLamaModelContext(_model);
+ _session = new ChatSession(new InteractiveExecutor(_context));
}
public void Dispose()
{
- _model?.Dispose();
+ _context?.Dispose();
}
public string Send(SendMessageInput input)
diff --git a/LLama.WebAPI/Services/StatelessChatService.cs b/LLama.WebAPI/Services/StatelessChatService.cs
index c1356646e..c186954f3 100644
--- a/LLama.WebAPI/Services/StatelessChatService.cs
+++ b/LLama.WebAPI/Services/StatelessChatService.cs
@@ -1,5 +1,4 @@
using LLama.Common;
-using Microsoft.AspNetCore.Http;
using System.Text;
using static LLama.LLamaTransforms;
@@ -8,13 +7,15 @@ namespace LLama.WebAPI.Services
public class StatelessChatService
{
private readonly LLamaModel _model;
+ private readonly LLamaModelContext _context;
private readonly ChatSession _session;
public StatelessChatService(IConfiguration configuration)
{
_model = new LLamaModel(new ModelParams(configuration["ModelPath"], contextSize: 512));
+ _context = new LLamaModelContext(_model);
// TODO: replace with a stateless executor
- _session = new ChatSession(new InteractiveExecutor(_model))
+ _session = new ChatSession(new InteractiveExecutor(_context))
.WithOutputTransform(new LLamaTransforms.KeywordTextOutputStreamTransform(new string[] { "User:", "Assistant:" }, redundancyLength: 8))
.WithHistoryTransform(new HistoryTransform());
}
diff --git a/LLama/Abstractions/IInferenceParams.cs b/LLama/Abstractions/IInferenceParams.cs
new file mode 100644
index 000000000..de51f8986
--- /dev/null
+++ b/LLama/Abstractions/IInferenceParams.cs
@@ -0,0 +1,29 @@
+using System.Collections.Generic;
+using LLama.Common;
+
+namespace LLama.Abstractions
+{
+ public interface IInferenceParams
+ {
+ IEnumerable AntiPrompts { get; set; }
+ float FrequencyPenalty { get; set; }
+ string InputPrefix { get; set; }
+ string InputSuffix { get; set; }
+ Dictionary? LogitBias { get; set; }
+ int MaxTokens { get; set; }
+ MirostatType Mirostat { get; set; }
+ float MirostatEta { get; set; }
+ float MirostatTau { get; set; }
+ string PathSession { get; set; }
+ bool PenalizeNL { get; set; }
+ float PresencePenalty { get; set; }
+ int RepeatLastTokensCount { get; set; }
+ float RepeatPenalty { get; set; }
+ float Temperature { get; set; }
+ float TfsZ { get; set; }
+ int TokensKeep { get; set; }
+ int TopK { get; set; }
+ float TopP { get; set; }
+ float TypicalP { get; set; }
+ }
+}
\ No newline at end of file
diff --git a/LLama/Abstractions/ILLamaExecutor.cs b/LLama/Abstractions/ILLamaExecutor.cs
index d35e075e9..4bdcc6bd9 100644
--- a/LLama/Abstractions/ILLamaExecutor.cs
+++ b/LLama/Abstractions/ILLamaExecutor.cs
@@ -1,5 +1,4 @@
-using LLama.Common;
-using System;
+using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
@@ -14,7 +13,7 @@ public interface ILLamaExecutor
///
/// The loaded model for this executor.
///
- public LLamaModel Model { get; }
+ public LLamaModelContext Context { get; }
///
/// Infers a response from the model.
@@ -23,7 +22,7 @@ public interface ILLamaExecutor
/// Any additional parameters
/// A cancellation token.
/// Infer(string text, InferenceParams? inferenceParams = null, CancellationToken token = default);
+ IEnumerable Infer(string text, IInferenceParams? inferenceParams = null, CancellationToken token = default);
///
/// Asynchronously infers a response from the model.
@@ -32,6 +31,6 @@ public interface ILLamaExecutor
/// Any additional parameters
/// A cancellation token.
/// InferAsync(string text, InferenceParams? inferenceParams = null, CancellationToken token = default);
+ IAsyncEnumerable InferAsync(string text, IInferenceParams? inferenceParams = null, CancellationToken token = default);
}
}
diff --git a/LLama/Abstractions/IModelParams.cs b/LLama/Abstractions/IModelParams.cs
new file mode 100644
index 000000000..695de2284
--- /dev/null
+++ b/LLama/Abstractions/IModelParams.cs
@@ -0,0 +1,28 @@
+namespace LLama.Abstractions
+{
+ public interface IModelParams
+ {
+ int BatchSize { get; set; }
+ int ContextSize { get; set; }
+ bool ConvertEosToNewLine { get; set; }
+ bool EmbeddingMode { get; set; }
+ int GpuLayerCount { get; set; }
+ int GroupedQueryAttention { get; set; }
+ string LoraAdapter { get; set; }
+ string LoraBase { get; set; }
+ bool LowVram { get; set; }
+ int MainGpu { get; set; }
+ string ModelPath { get; set; }
+ string Name { get; set; }
+ bool Perplexity { get; set; }
+ float RmsNormEpsilon { get; set; }
+ float RopeFrequencyBase { get; set; }
+ float RopeFrequencyScale { get; set; }
+ int Seed { get; set; }
+ float[] TensorSplits { get; set; }
+ int Threads { get; set; }
+ bool UseFp16Memory { get; set; }
+ bool UseMemoryLock { get; set; }
+ bool UseMemorymap { get; set; }
+ }
+}
\ No newline at end of file
diff --git a/LLama/ChatSession.cs b/LLama/ChatSession.cs
index b87e8984f..5b1e7eeac 100644
--- a/LLama/ChatSession.cs
+++ b/LLama/ChatSession.cs
@@ -91,7 +91,7 @@ public virtual void SaveSession(string path)
{
Directory.CreateDirectory(path);
}
- _executor.Model.SaveState(Path.Combine(path, _modelStateFilename));
+ _executor.Context.SaveState(Path.Combine(path, _modelStateFilename));
if(Executor is StatelessExecutor)
{
@@ -116,7 +116,7 @@ public virtual void LoadSession(string path)
{
throw new FileNotFoundException($"Directory {path} does not exist.");
}
- _executor.Model.LoadState(Path.Combine(path, _modelStateFilename));
+ _executor.Context.LoadState(Path.Combine(path, _modelStateFilename));
if (Executor is StatelessExecutor)
{
diff --git a/LLama/Common/FixedSizeQueue.cs b/LLama/Common/FixedSizeQueue.cs
index e059ef705..4b082feb7 100644
--- a/LLama/Common/FixedSizeQueue.cs
+++ b/LLama/Common/FixedSizeQueue.cs
@@ -30,9 +30,11 @@ public FixedSizeQueue(int size)
///
public FixedSizeQueue(int size, IEnumerable data)
{
+#if NETCOREAPP3_0_OR_GREATER
// Try an early check on the amount of data supplied (if possible)
if (data.TryGetNonEnumeratedCount(out var count) && count > size)
throw new ArgumentException($"The max size set for the quene is {size}, but got {count} initial values.");
+#endif
// Size of "data" is unknown, copy it all into a list
_maxSize = size;
@@ -40,7 +42,7 @@ public FixedSizeQueue(int size, IEnumerable data)
// Now check if that list is a valid size
if (_storage.Count > _maxSize)
- throw new ArgumentException($"The max size set for the quene is {size}, but got {count} initial values.");
+ throw new ArgumentException($"The max size set for the quene is {size}, but got {_storage.Count} initial values.");
}
///
diff --git a/LLama/Common/InferenceParams.cs b/LLama/Common/InferenceParams.cs
index 77af7eafe..b21bd32ee 100644
--- a/LLama/Common/InferenceParams.cs
+++ b/LLama/Common/InferenceParams.cs
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
+using LLama.Abstractions;
namespace LLama.Common
{
@@ -7,7 +8,7 @@ namespace LLama.Common
///
/// The paramters used for inference.
///
- public class InferenceParams
+ public class InferenceParams : IInferenceParams
{
///
/// number of tokens to keep from initial prompt
diff --git a/LLama/Common/ModelParams.cs b/LLama/Common/ModelParams.cs
index 2a591bcd7..d917deaa8 100644
--- a/LLama/Common/ModelParams.cs
+++ b/LLama/Common/ModelParams.cs
@@ -1,13 +1,14 @@
using System;
using System.Collections.Generic;
using System.Text;
+using LLama.Abstractions;
namespace LLama.Common
{
///
/// The parameters for initializing a LLama model.
///
- public class ModelParams
+ public class ModelParams : IModelParams
{
///
/// Model context size (n_ctx)
@@ -50,9 +51,9 @@ public class ModelParams
///
public string ModelPath { get; set; }
///
- /// model alias
+ /// model name
///
- public string ModelAlias { get; set; } = "unknown";
+ public string Name { get; set; } = "unknown";
///
/// lora adapter path (lora_adapter)
///
@@ -86,6 +87,26 @@ public class ModelParams
///
public float[] TensorSplits { get; set; } = new float[] { 0 };
+ ///
+ /// Grouped-Query Attention
+ ///
+ public int GroupedQueryAttention { get; set; } = 1;
+
+ ///
+ /// RMS Norm Epsilon
+ ///
+ public float RmsNormEpsilon { get; set; } = 5e-6f;
+
+ ///
+ /// RoPE base frequency
+ ///
+ public float RopeFrequencyBase { get; set; } = 10000.0f;
+
+ ///
+ /// RoPE frequency scaling factor
+ ///
+ public float RopeFrequencyScale { get; set; } = 1.0f;
+
///
///
///
diff --git a/LLama/LLamaEmbedder.cs b/LLama/LLamaEmbedder.cs
index 4bbb61d2e..325a7520f 100644
--- a/LLama/LLamaEmbedder.cs
+++ b/LLama/LLamaEmbedder.cs
@@ -5,6 +5,7 @@
using LLama.Exceptions;
using System.Linq;
using LLama.Common;
+using LLama.Abstractions;
namespace LLama
{
@@ -28,7 +29,7 @@ internal LLamaEmbedder(SafeLLamaContextHandle ctx)
///
///
///
- public LLamaEmbedder(ModelParams @params)
+ public LLamaEmbedder(IModelParams @params)
{
@params.EmbeddingMode = true;
_ctx = Utils.InitLLamaContextFromModelParams(@params);
diff --git a/LLama/LLamaExecutorBase.cs b/LLama/LLamaExecutorBase.cs
index afbc0f258..5b4e84062 100644
--- a/LLama/LLamaExecutorBase.cs
+++ b/LLama/LLamaExecutorBase.cs
@@ -19,9 +19,9 @@ namespace LLama
public abstract class StatefulExecutorBase : ILLamaExecutor
{
///
- /// The loaded model for this executor.
+ /// The loaded context for this executor.
///
- protected readonly LLamaModel _model;
+ protected readonly LLamaModelContext _context;
///
/// The logger used by this executor.
///
@@ -63,9 +63,9 @@ public abstract class StatefulExecutorBase : ILLamaExecutor
///
protected FixedSizeQueue _last_n_tokens;
///
- /// The mode used by the executor.
+ /// The context used by the executor.
///
- public LLamaModel Model => _model;
+ public LLamaModelContext Context => _context;
///
/// Current "mu" value for mirostate sampling
@@ -75,16 +75,16 @@ public abstract class StatefulExecutorBase : ILLamaExecutor
///
///
///
- ///
+ ///
///
- protected StatefulExecutorBase(LLamaModel model, ILLamaLogger? logger = null)
+ protected StatefulExecutorBase(LLamaModelContext context, ILLamaLogger? logger = null)
{
- _model = model;
+ _context = context;
_logger = logger;
_pastTokensCount = 0;
_consumedTokensCount = 0;
_n_session_consumed = 0;
- _last_n_tokens = new FixedSizeQueue(_model.ContextSize).FillWith(0);
+ _last_n_tokens = new FixedSizeQueue(_context.ContextSize).FillWith(0);
}
///
@@ -104,9 +104,9 @@ public unsafe StatefulExecutorBase WithSessionFile(string filename)
if (File.Exists(filename))
{
_logger?.Log("LLamaExecutor", $"Attempting to load saved session from {filename}", ILLamaLogger.LogLevel.Info);
- llama_token[] session_tokens = new llama_token[_model.ContextSize];
+ llama_token[] session_tokens = new llama_token[_context.ContextSize];
ulong n_token_count_out = 0;
- if (!NativeApi.llama_load_session_file(_model.NativeHandle, _pathSession, session_tokens, (ulong)_model.ContextSize, &n_token_count_out))
+ if (!NativeApi.llama_load_session_file(_context.NativeHandle, _pathSession, session_tokens, (ulong)_context.ContextSize, &n_token_count_out))
{
_logger?.Log("LLamaExecutor", $"Failed to load session file {filename}", ILLamaLogger.LogLevel.Error);
throw new RuntimeError($"Failed to load session file {_pathSession}");
@@ -156,7 +156,7 @@ public unsafe StatefulExecutorBase WithSessionFile(string filename)
public void SaveSessionFile(string filename)
{
var session_token_array = _session_tokens.ToArray();
- NativeApi.llama_save_session_file(_model.NativeHandle, filename, session_token_array, (ulong)session_token_array.Length);
+ NativeApi.llama_save_session_file(_context.NativeHandle, filename, session_token_array, (ulong)session_token_array.Length);
}
///
@@ -173,7 +173,7 @@ protected virtual void HandleRunOutOfContext(int tokensToKeep)
_pastTokensCount = Math.Max(1, tokensToKeep);
// insert n_left/2 tokens at the start of embed from last_n_tokens
- _embeds.InsertRange(0, _last_n_tokens.Take(_last_n_tokens.Count - _embeds.Count).Skip(_model.ContextSize - n_left / 2 - _embeds.Count));
+ _embeds.InsertRange(0, _last_n_tokens.Take(_last_n_tokens.Count - _embeds.Count).Skip(_context.ContextSize - n_left / 2 - _embeds.Count));
// stop saving session if we run out of context
_pathSession = string.Empty;
@@ -231,13 +231,13 @@ protected virtual void TryReuseMathingPrefix()
///
///
/// ? extraOutputs);
+ protected abstract bool PostProcess(IInferenceParams inferenceParams, InferStateArgs args, out IEnumerable? extraOutputs);
///
/// The core inference logic.
///
///
///
- protected abstract void InferInternal(InferenceParams inferenceParams, InferStateArgs args);
+ protected abstract void InferInternal(IInferenceParams inferenceParams, InferStateArgs args);
///
/// Save the current state to a file.
///
@@ -267,7 +267,7 @@ protected virtual void TryReuseMathingPrefix()
///
///
/// Infer(string text, InferenceParams? inferenceParams = null, CancellationToken cancellationToken = default)
+ public virtual IEnumerable Infer(string text, IInferenceParams? inferenceParams = null, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
if (inferenceParams is null)
@@ -296,7 +296,7 @@ public virtual IEnumerable Infer(string text, InferenceParams? inference
if (args.ReturnValue)
{
- foreach (var item in _model.GenerateResult(_embeds))
+ foreach (var item in _context.GenerateResult(_embeds))
{
yield return item;
}
@@ -324,7 +324,7 @@ public virtual IEnumerable Infer(string text, InferenceParams? inference
///
///
/// InferAsync(string text, InferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+ public virtual async IAsyncEnumerable InferAsync(string text, IInferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
foreach (var result in Infer(text, inferenceParams, cancellationToken))
{
diff --git a/LLama/LLamaInstructExecutor.cs b/LLama/LLamaInstructExecutor.cs
index 89fbac59f..d9ba2c46f 100644
--- a/LLama/LLamaInstructExecutor.cs
+++ b/LLama/LLamaInstructExecutor.cs
@@ -1,4 +1,5 @@
-using LLama.Common;
+using LLama.Abstractions;
+using LLama.Common;
using LLama.Native;
using System;
using System.Collections.Generic;
@@ -26,11 +27,11 @@ public class InstructExecutor : StatefulExecutorBase
///
///
///
- public InstructExecutor(LLamaModel model, string instructionPrefix = "\n\n### Instruction:\n\n",
+ public InstructExecutor(LLamaModelContext model, string instructionPrefix = "\n\n### Instruction:\n\n",
string instructionSuffix = "\n\n### Response:\n\n") : base(model)
{
- _inp_pfx = _model.Tokenize(instructionPrefix, true).ToArray();
- _inp_sfx = _model.Tokenize(instructionSuffix, false).ToArray();
+ _inp_pfx = _context.Tokenize(instructionPrefix, true).ToArray();
+ _inp_sfx = _context.Tokenize(instructionSuffix, false).ToArray();
_instructionPrefix = instructionPrefix;
}
@@ -116,7 +117,7 @@ protected override void PreprocessInputs(string text, InferStateArgs args)
{
// When running the first input (prompt) in inteactive mode, we should specially process it.
text = " " + text;
- _embed_inps = _model.Tokenize(text, true).ToList();
+ _embed_inps = _context.Tokenize(text, true).ToList();
}
else
{
@@ -127,7 +128,7 @@ protected override void PreprocessInputs(string text, InferStateArgs args)
_consumedTokensCount = _embed_inps.Count;
_embed_inps.AddRange(_inp_pfx);
- var line_inp = _model.Tokenize(text, false);
+ var line_inp = _context.Tokenize(text, false);
_embed_inps.AddRange(line_inp);
_embed_inps.AddRange(_inp_sfx);
@@ -136,7 +137,7 @@ protected override void PreprocessInputs(string text, InferStateArgs args)
}
}
/// ? extraOutputs)
+ protected override bool PostProcess(IInferenceParams inferenceParams, InferStateArgs args, out IEnumerable? extraOutputs)
{
extraOutputs = null;
if (_embed_inps.Count <= _consumedTokensCount)
@@ -146,7 +147,7 @@ protected override bool PostProcess(InferenceParams inferenceParams, InferStateA
string last_output = "";
foreach (var id in _last_n_tokens)
{
- last_output += Utils.PtrToString(NativeApi.llama_token_to_str(_model.NativeHandle, id), _model.Encoding);
+ last_output += Utils.PtrToString(NativeApi.llama_token_to_str(_context.NativeHandle, id), _context.Encoding);
}
foreach (var antiprompt in args.Antiprompts)
@@ -179,18 +180,18 @@ protected override bool PostProcess(InferenceParams inferenceParams, InferStateA
return false;
}
///
///
- public InteractiveExecutor(LLamaModel model) : base(model)
+ public InteractiveExecutor(LLamaModelContext model) : base(model)
{
- _llama_token_newline = Utils.Tokenize(_model.NativeHandle, "\n", false, _model.Encoding).ToArray();
+ _llama_token_newline = Utils.Tokenize(_context.NativeHandle, "\n", false, _context.Encoding).ToArray();
}
///
///
/// ? extraOutputs)
+ protected override bool PostProcess(IInferenceParams inferenceParams, InferStateArgs args, out IEnumerable? extraOutputs)
{
extraOutputs = null;
if (_embed_inps.Count <= _consumedTokensCount)
@@ -132,7 +133,7 @@ protected override bool PostProcess(InferenceParams inferenceParams, InferStateA
string last_output = "";
foreach (var id in _last_n_tokens)
{
- last_output += Utils.PtrToString(NativeApi.llama_token_to_str(_model.NativeHandle, id), _model.Encoding);
+ last_output += Utils.PtrToString(NativeApi.llama_token_to_str(_context.NativeHandle, id), _context.Encoding);
}
foreach (var antiprompt in args.Antiprompts)
@@ -166,18 +167,18 @@ protected override bool PostProcess(InferenceParams inferenceParams, InferStateA
}
///
/// The abstraction of a LLama model, which holds the context in the native library.
///
- public class LLamaModel: IDisposable
+ public class LLamaModel : IDisposable
{
// TODO: expose more properties.
ILLamaLogger? _logger;
- Encoding _encoding;
- SafeLLamaContextHandle _ctx;
- ///
- /// The context size.
- ///
- public int ContextSize { get; }
+ SafeLlamaModelHandle _model;
+ ConcurrentDictionary _contexts;
///
/// The model params set for this model.
///
- public ModelParams Params { get; set; }
+ public IModelParams Params { get; set; }
///
/// The native handle, which is used to be passed to the native APIs. Please avoid using it
/// unless you know what is the usage of the Native API.
///
- public SafeLLamaContextHandle NativeHandle => _ctx;
- ///
- /// The encoding set for this model to deal with text input.
- ///
- public Encoding Encoding => _encoding;
+ public SafeLlamaModelHandle NativeHandle => _model;
+
///
///
///
- /// Model params.
- /// Encoding to deal with text input.
+ /// Model params.
/// The logger.
- public LLamaModel(ModelParams Params, string encoding = "UTF-8", ILLamaLogger? logger = null)
+ public LLamaModel(IModelParams modelParams, ILLamaLogger? logger = null)
{
_logger = logger;
- this.Params = Params;
- _encoding = Encoding.GetEncoding(encoding);
- _logger?.Log(nameof(LLamaModel), $"Initializing LLama model with params: {this.Params}", ILLamaLogger.LogLevel.Info);
- _ctx = Utils.InitLLamaContextFromModelParams(this.Params);
- ContextSize = NativeApi.llama_n_ctx(_ctx);
- }
-
- ///
- /// Tokenize a string.
- ///
- ///
- /// Whether to add a bos to the text.
- /// Tokenize(string text, bool addBos = true)
- {
- // TODO: reconsider whether to convert to array here.
- return Utils.Tokenize(_ctx, text, addBos, _encoding);
+ _contexts = new ConcurrentDictionary();
+ _logger?.Log(nameof(LLamaModelContext), $"Initializing LLama model with params: {modelParams}", ILLamaLogger.LogLevel.Info);
+
+ Params = modelParams;
+ var contextParams = Utils.CreateContextParams(modelParams);
+ _model = SafeLlamaModelHandle.LoadFromFile(modelParams.ModelPath, contextParams);
+ if (!string.IsNullOrEmpty(modelParams.LoraAdapter))
+ _model.ApplyLoraFromFile(modelParams.LoraAdapter, modelParams.LoraBase, modelParams.Threads);
}
///
- /// Detokenize the tokens to text.
+ /// Creates a new context session on this model
///
- ///
- /// tokens)
+ /// The unique context identifier
+ /// The contexts text encoding
+ /// LLamaModelContext for this LLamaModel
+ /// Context exists
+ public Task CreateContext(string contextId, string encoding = "UTF-8")
{
- StringBuilder sb = new();
- foreach(var token in tokens)
- {
- sb.Append(Utils.PtrToString(NativeApi.llama_token_to_str(_ctx, token), _encoding));
- }
- return sb.ToString();
- }
-
- ///
- /// Save the state to specified path.
- ///
- ///
- public void SaveState(string filename)
- {
- // Delete that file before overwriting it
- if (File.Exists(filename))
- File.Delete(filename);
-
- // Estimate size of state to write to disk, this is always equal to or greater than the actual size
- var estimatedStateSize = (long)NativeApi.llama_get_state_size(_ctx);
+ if (_contexts.TryGetValue(contextId, out var context))
+ throw new Exception($"Context with id {contextId} already exists.");
- // Map the file and write the bytes directly to it. This saves copying the bytes into a C# array
- long writtenBytes;
- using (var file = MemoryMappedFile.CreateFromFile(filename, FileMode.Create, null, estimatedStateSize))
- using (var view = file.CreateViewAccessor(0, estimatedStateSize))
- {
- unsafe
- {
- byte* ptr = null;
- view.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
- writtenBytes = (long)NativeApi.llama_copy_state_data(_ctx, ptr);
- view.SafeMemoryMappedViewHandle.ReleasePointer();
- }
- }
+ context = new LLamaModelContext(this, encoding, _logger);
+ if (_contexts.TryAdd(contextId, context))
+ return Task.FromResult(context);
- // Truncate the file to the actual size of data that was written
- using (var fileStream = new FileStream(filename, FileMode.Open))
- fileStream.SetLength(writtenBytes);
+ return Task.FromResult(null);
}
///
- /// Get the state data as a byte array.
+ /// Get a contexts belonging to this model
///
- /// LLamaModelContext for this LLamaModel with the specified contextId
+ public Task GetContext(string contextId)
{
- var stateSize = NativeApi.llama_get_state_size(_ctx);
- byte[] stateMemory = new byte[stateSize];
- NativeApi.llama_copy_state_data(_ctx, stateMemory);
- return stateMemory;
- }
+ if (_contexts.TryGetValue(contextId, out var context))
+ return Task.FromResult(context);
- ///
- /// Get the state data as an opaque handle
- ///
- /// (null);
}
///
- /// Load the state from specified path.
+ /// Remove a context from this model
///
- ///
- /// true if removed, otherwise false
+ public Task RemoveContext(string contextId)
{
- // Map state file into memory and pass that pointer directly to `llama_set_state_data` to load from
- using (var file = MemoryMappedFile.CreateFromFile(filename, FileMode.Open, null))
- using (var view = file.CreateViewAccessor())
- {
- unsafe
- {
- byte* ptr = null;
- view.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
- NativeApi.llama_set_state_data(_ctx, ptr);
- view.SafeMemoryMappedViewHandle.ReleasePointer();
- }
- }
- }
+ if (!_contexts.TryRemove(contextId, out var context))
+ return Task.FromResult(false);
- ///
- /// Load the state from memory.
- ///
- ///
- ///
- /// Load the state from memory.
- ///
- ///
- ///
- /// Perform the sampling. Please don't use it unless you fully know what it does.
- ///
- ///
- ///
- ///
- ///
- ///
- ///
- ///
- ///
- ///
- ///
- ///
- /// Apply the penalty for the tokens. Please don't use it unless you fully know what it does.
- ///
- ///
- ///
- ///
- ///
- ///
- ///
- ///
- /// lastTokens, Dictionary? logitBias = null,
- int repeatLastTokensCount = 64, float repeatPenalty = 1.1f, float alphaFrequency = .0f, float alphaPresence = .0f,
- bool penalizeNL = true)
- {
- var n_vocab = NativeApi.llama_n_vocab(_ctx);
- var logits = Utils.GetLogits(_ctx, n_vocab);
-
- // Apply params.logit_bias map
- if(logitBias is not null)
- {
- foreach (var (key, value) in logitBias)
- {
- logits[key] += value;
- }
- }
-
- var candidates = new LLamaTokenData[n_vocab];
- for (llama_token token_id = 0; token_id < n_vocab; token_id++)
- candidates[token_id] = new LLamaTokenData(token_id, logits[token_id], 0.0f);
- LLamaTokenDataArray candidates_p = new LLamaTokenDataArray(candidates);
-
- // Apply penalties
- float nl_logit = logits[NativeApi.llama_token_nl()];
- int lastTokensCount = lastTokens.Count();
- var last_n_repeat = Math.Min(Math.Min(lastTokensCount, repeatLastTokensCount), ContextSize);
- SamplingApi.llama_sample_repetition_penalty(_ctx, candidates_p,
- lastTokens.Skip(lastTokensCount - last_n_repeat).ToArray(),
- (ulong)last_n_repeat, repeatPenalty);
- SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates_p,
- lastTokens.Skip(lastTokensCount - last_n_repeat).ToArray(),
- (ulong)last_n_repeat, alphaFrequency, alphaPresence);
- if (!penalizeNL)
- {
- logits[NativeApi.llama_token_nl()] = nl_logit;
- }
-
- return candidates_p;
- }
-
- ///
- ///
- ///
- ///
- ///
- /// The updated `pastTokensCount`.
- /// GenerateResult(IEnumerable ids)
- {
- foreach(var id in ids)
- {
- yield return Utils.TokenToString(id, _ctx, _encoding);
- }
+ context?.Dispose();
+ return Task.FromResult(true);
}
///
- /// The state of this model, which can be reloaded later
- ///
- public class State
- : SafeHandleZeroOrMinusOneIsInvalid
- {
- internal State(IntPtr memory)
- : base(true)
- {
- SetHandle(memory);
- }
-
- ///
+ /// The abstraction of a context over a LLama model
+ ///
+ public class LLamaModelContext : IDisposable
+ {
+ // TODO: expose more properties.
+ ILLamaLogger? _logger;
+ Encoding _encoding;
+ SafeLLamaContextHandle _ctx;
+ ///
+ /// The context size.
+ ///
+ public int ContextSize { get; }
+ ///
+ /// The model params set for this model.
+ ///
+ public IModelParams Params { get; set; }
+ ///
+ /// The native handle, which is used to be passed to the native APIs. Please avoid using it
+ /// unless you know what is the usage of the Native API.
+ ///
+ public SafeLLamaContextHandle NativeHandle => _ctx;
+ ///
+ /// The encoding set for this model to deal with text input.
+ ///
+ public Encoding Encoding => _encoding;
+
+ ///
+ ///
+ ///
+ /// Model instance.
+ /// Encoding to deal with text input.
+ /// The logger.
+ public LLamaModelContext(LLamaModel model, string encoding = "UTF-8", ILLamaLogger? logger = null)
+ {
+ _logger = logger;
+ Params = model.Params;
+ _encoding = Encoding.GetEncoding(encoding);
+ _logger?.Log(nameof(LLamaModelContext), $"Initializing LLama model with params: {Params}", ILLamaLogger.LogLevel.Info);
+
+ var contextParams = Utils.CreateContextParams(Params);
+ _ctx = SafeLLamaContextHandle.Create(model.NativeHandle, contextParams);
+ ContextSize = NativeApi.llama_n_ctx(_ctx);
+ }
+ ///
+ /// Tokenize a string.
+ ///
+ ///
+ /// Whether to add a bos to the text.
+ /// Tokenize(string text, bool addBos = true)
+ {
+ // TODO: reconsider whether to convert to array here.
+ return Utils.Tokenize(_ctx, text, addBos, _encoding);
+ }
+
+ ///
+ /// Detokenize the tokens to text.
+ ///
+ ///
+ /// tokens)
+ {
+ StringBuilder sb = new();
+ foreach (var token in tokens)
+ {
+ sb.Append(Utils.PtrToString(NativeApi.llama_token_to_str(_ctx, token), _encoding));
+ }
+ return sb.ToString();
+ }
+
+ ///
+ /// Save the state to specified path.
+ ///
+ ///
+ public void SaveState(string filename)
+ {
+ // Delete that file before overwriting it
+ if (File.Exists(filename))
+ File.Delete(filename);
+
+ // Estimate size of state to write to disk, this is always equal to or greater than the actual size
+ var estimatedStateSize = (long)NativeApi.llama_get_state_size(_ctx);
+
+ // Map the file and write the bytes directly to it. This saves copying the bytes into a C# array
+ long writtenBytes;
+ using (var file = MemoryMappedFile.CreateFromFile(filename, FileMode.Create, null, estimatedStateSize))
+ using (var view = file.CreateViewAccessor(0, estimatedStateSize))
+ {
+ unsafe
+ {
+ byte* ptr = null;
+ view.SafeMemoryMappedViewHandle.AcquirePointer(ref ptr);
+ writtenBytes = (long)NativeApi.llama_copy_state_data(_ctx, ptr);
+ view.SafeMemoryMappedViewHandle.ReleasePointer();
+ }
+ }
+
+ // Truncate the file to the actual size of data that was written
+ using (var fileStream = new FileStream(filename, FileMode.Open))
+ fileStream.SetLength(writtenBytes);
+ }
+
+ ///
+ /// Get the state data as a byte array.
+ ///
+ ///
+ /// Get the state data as an opaque handle
+ ///
+ ///
+ /// Load the state from specified path.
+ ///
+ ///
+ ///
+ /// Load the state from memory.
+ ///
+ ///
+ ///
+ /// Load the state from memory.
+ ///
+ ///
+ ///
+ /// Perform the sampling. Please don't use it unless you fully know what it does.
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ /// Apply the penalty for the tokens. Please don't use it unless you fully know what it does.
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ ///
+ /// lastTokens, Dictionary? logitBias = null,
+ int repeatLastTokensCount = 64, float repeatPenalty = 1.1f, float alphaFrequency = .0f, float alphaPresence = .0f,
+ bool penalizeNL = true)
+ {
+ var n_vocab = NativeApi.llama_n_vocab(_ctx);
+ var logits = Utils.GetLogits(_ctx, n_vocab);
+
+ // Apply params.logit_bias map
+ if (logitBias is not null)
+ {
+ foreach (var (key, value) in logitBias)
+ {
+ logits[key] += value;
+ }
+ }
+
+ var candidates = new LLamaTokenData[n_vocab];
+ for (llama_token token_id = 0; token_id < n_vocab; token_id++)
+ candidates[token_id] = new LLamaTokenData(token_id, logits[token_id], 0.0f);
+ LLamaTokenDataArray candidates_p = new LLamaTokenDataArray(candidates);
+
+ // Apply penalties
+ float nl_logit = logits[NativeApi.llama_token_nl()];
+ int lastTokensCount = lastTokens.Count();
+ var last_n_repeat = Math.Min(Math.Min(lastTokensCount, repeatLastTokensCount), ContextSize);
+ SamplingApi.llama_sample_repetition_penalty(_ctx, candidates_p,
+ lastTokens.Skip(lastTokensCount - last_n_repeat).ToArray(),
+ (ulong)last_n_repeat, repeatPenalty);
+ SamplingApi.llama_sample_frequency_and_presence_penalties(_ctx, candidates_p,
+ lastTokens.Skip(lastTokensCount - last_n_repeat).ToArray(),
+ (ulong)last_n_repeat, alphaFrequency, alphaPresence);
+ if (!penalizeNL)
+ {
+ logits[NativeApi.llama_token_nl()] = nl_logit;
+ }
+
+ return candidates_p;
+ }
+
+ ///
+ ///
+ ///
+ ///
+ ///
+ /// The updated `pastTokensCount`.
+ /// GenerateResult(IEnumerable ids)
+ {
+ foreach (var id in ids)
+ {
+ yield return Utils.TokenToString(id, _ctx, _encoding);
+ }
+ }
+
+ ///
+ /// The state of this model, which can be reloaded later
+ ///
+ public class State
+ : SafeHandleZeroOrMinusOneIsInvalid
+ {
+ internal State(IntPtr memory)
+ : base(true)
+ {
+ SetHandle(memory);
+ }
+
+ ///
public class StatelessExecutor : ILLamaExecutor
{
- private LLamaModel _model;
- private LLamaModel.State _originalState;
+ private LLamaModelContext _model;
+ private LLamaModelContext.State _originalState;
///
/// The mode used by the executor when running the inference.
///
- public LLamaModel Model => _model;
+ public LLamaModelContext Context => _model;
///
///
///
/// The LLama model.
- public StatelessExecutor(LLamaModel model)
+ public StatelessExecutor(LLamaModelContext model)
{
_model = model;
@@ -36,7 +36,7 @@ public StatelessExecutor(LLamaModel model)
}
/// Infer(string text, InferenceParams? inferenceParams = null, CancellationToken cancellationToken = default)
+ public IEnumerable Infer(string text, IInferenceParams? inferenceParams = null, CancellationToken cancellationToken = default)
{
cancellationToken.ThrowIfCancellationRequested();
int n_past = 1;
@@ -123,7 +123,7 @@ public IEnumerable Infer(string text, InferenceParams? inferenceParams =
}
/// InferAsync(string text, InferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+ public async IAsyncEnumerable InferAsync(string text, IInferenceParams? inferenceParams = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
{
foreach (var result in Infer(text, inferenceParams, cancellationToken))
{
diff --git a/LLama/Native/LLamaContextParams.cs b/LLama/Native/LLamaContextParams.cs
index 42f2be3fc..17b79ae5f 100644
--- a/LLama/Native/LLamaContextParams.cs
+++ b/LLama/Native/LLamaContextParams.cs
@@ -47,7 +47,7 @@ public struct LLamaContextParams
///
/// how to split layers across multiple GPUs
///
- public float[] tensor_split;
+ public IntPtr tensor_split;
///
/// ref: https://github.com/ggerganov/llama.cpp/pull/2054
diff --git a/LLama/ResettableLLamaModel.cs b/LLama/ResettableLLamaModel.cs
index f2862dc77..8788aa942 100644
--- a/LLama/ResettableLLamaModel.cs
+++ b/LLama/ResettableLLamaModel.cs
@@ -8,7 +8,7 @@ namespace LLama
///
/// A LLamaModel what could be reset. Note that using this class will consume about 10% more memories.
///
- public class ResettableLLamaModel : LLamaModel
+ public class ResettableLLamaModel : LLamaModelContext
{
///
/// The initial state of the model
@@ -17,9 +17,9 @@ public class ResettableLLamaModel : LLamaModel
///
///
///
- ///
+ ///
///
- public ResettableLLamaModel(ModelParams Params, string encoding = "UTF-8") : base(Params, encoding)
+ public ResettableLLamaModel(LLamaModel model, string encoding = "UTF-8") : base(model, encoding)
{
OriginalState = GetState();
}
diff --git a/LLama/Utils.cs b/LLama/Utils.cs
index c08912cf6..3a2ece76f 100644
--- a/LLama/Utils.cs
+++ b/LLama/Utils.cs
@@ -1,4 +1,5 @@
-using LLama.Common;
+using LLama.Abstractions;
+using LLama.Common;
using LLama.Exceptions;
using LLama.Native;
using System;
@@ -13,41 +14,49 @@ namespace LLama
using llama_token = Int32;
internal static class Utils
{
- public static SafeLLamaContextHandle InitLLamaContextFromModelParams(ModelParams @params)
+ public static SafeLLamaContextHandle InitLLamaContextFromModelParams(IModelParams @params)
+ {
+ var lparams = CreateContextParams(@params);
+ var model = SafeLlamaModelHandle.LoadFromFile(@params.ModelPath, lparams);
+ var ctx = SafeLLamaContextHandle.Create(model, lparams);
+
+ if (!string.IsNullOrEmpty(@params.LoraAdapter))
+ model.ApplyLoraFromFile(@params.LoraAdapter, @params.LoraBase, @params.Threads);
+
+ return ctx;
+ }
+
+ public static LLamaContextParams CreateContextParams(IModelParams modelParams)
{
var lparams = NativeApi.llama_context_default_params();
- lparams.n_ctx = @params.ContextSize;
- lparams.n_batch = @params.BatchSize;
- lparams.main_gpu = @params.MainGpu;
- lparams.n_gpu_layers = @params.GpuLayerCount;
- lparams.seed = @params.Seed;
- lparams.f16_kv = @params.UseFp16Memory;
- lparams.use_mmap = @params.UseMemoryLock;
- lparams.use_mlock = @params.UseMemoryLock;
- lparams.logits_all = @params.Perplexity;
- lparams.embedding = @params.EmbeddingMode;
- lparams.low_vram = @params.LowVram;
-
- if (@params.TensorSplits.Length != 1)
- {
- throw new ArgumentException("Currently multi-gpu support is not supported by " +
- "both llama.cpp and LLamaSharp.");
- }
- lparams.tensor_split = @params.TensorSplits;
+ lparams.n_ctx = modelParams.ContextSize;
+ lparams.n_batch = modelParams.BatchSize;
+ lparams.main_gpu = modelParams.MainGpu;
+ lparams.n_gpu_layers = modelParams.GpuLayerCount;
+ lparams.seed = modelParams.Seed;
+ lparams.f16_kv = modelParams.UseFp16Memory;
+ lparams.use_mmap = modelParams.UseMemoryLock;
+ lparams.use_mlock = modelParams.UseMemoryLock;
+ lparams.logits_all = modelParams.Perplexity;
+ lparams.embedding = modelParams.EmbeddingMode;
+ lparams.low_vram = modelParams.LowVram;
- if (!File.Exists(@params.ModelPath))
+ if (modelParams.TensorSplits.Length != 1)
{
- throw new FileNotFoundException($"The model file does not exist: {@params.ModelPath}");
+ throw new ArgumentException("Currently multi-gpu support is not supported by both llama.cpp and LLamaSharp.");
}
- var model = SafeLlamaModelHandle.LoadFromFile(@params.ModelPath, lparams);
- var ctx = SafeLLamaContextHandle.Create(model, lparams);
+ // Allocate memory for the 'tensor_split' array in C++,
+ lparams.tensor_split = Marshal.AllocHGlobal(modelParams.TensorSplits.Length * sizeof(float));
+ Marshal.Copy(modelParams.TensorSplits, 0, lparams.tensor_split, modelParams.TensorSplits.Length);
- if (!string.IsNullOrEmpty(@params.LoraAdapter))
- model.ApplyLoraFromFile(@params.LoraAdapter, @params.LoraBase, @params.Threads);
+ if (!File.Exists(modelParams.ModelPath))
+ {
+ throw new FileNotFoundException($"The model file does not exist: {modelParams.ModelPath}");
+ }
- return ctx;
+ return lparams;
}
public static IEnumerable Tokenize(SafeLLamaContextHandle ctx, string text, bool add_bos, Encoding encoding)
diff --git a/LLama/runtimes/libllama-cuda12.dll b/LLama/runtimes/libllama-cuda12.dll
index 97c1a9b13..d392d4c29 100644
Binary files a/LLama/runtimes/libllama-cuda12.dll and b/LLama/runtimes/libllama-cuda12.dll differ
diff --git a/LLama/runtimes/libllama.dll b/LLama/runtimes/libllama.dll
index 6b9e5f9d0..e0d8ca490 100644
Binary files a/LLama/runtimes/libllama.dll and b/LLama/runtimes/libllama.dll differ
diff --git a/LLama/runtimes/libllama.so b/LLama/runtimes/libllama.so
index 5acc506a2..aabcd434b 100644
Binary files a/LLama/runtimes/libllama.so and b/LLama/runtimes/libllama.so differ
diff --git a/README.md b/README.md
index 7c126e942..e6673a33a 100644
--- a/README.md
+++ b/README.md
@@ -77,8 +77,10 @@ string modelPath = "" // change it to your own model path
var prompt = "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.\r\n\r\nUser: Hello, Bob.\r\nBob: Hello. How may I help you today?\r\nUser: Please tell me the largest city in Europe.\r\nBob: Sure. The largest city in Europe is Moscow, the capital of Russia.\r\nUser:"; // use the "chat-with-bob" prompt here.
// Initialize a chat session
-var ex = new InteractiveExecutor(new LLamaModel(new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5)));
-ChatSession session = new ChatSession(ex);
+LLamaModel model = new LLamaModel(new ModelParams(modelPath, contextSize: 1024, seed: 1337, gpuLayerCount: 5));
+LLamaModelContext context = model.CreateContext("context1");
+InteractiveExecutor executor = new InteractiveExecutor(context);
+ChatSession session = new ChatSession(executor);
// show the prompt
Console.WriteLine();
@@ -124,10 +126,25 @@ We provide the integration of ASP.NET core [here](./LLama.WebAPI). Since current
Since we are in short of hands, if you're familiar with ASP.NET core, we'll appreciate it if you would like to help upgrading the Web API integration.
-## Demo
+## Console Demo
+
+
+
+
+## Web Demo
+
+Model Parameters
+
+
+
+Inference Parameters
+
+
+
+
## Roadmap
---
-
+
@@ -41,20 +39,6 @@
-
\ No newline at end of file
diff --git a/LLama.Web/Pages/Shared/_Layout.cshtml b/LLama.Web/Pages/Shared/_Layout.cshtml
index 23132bfa4..b5d1ef02e 100644
--- a/LLama.Web/Pages/Shared/_Layout.cshtml
+++ b/LLama.Web/Pages/Shared/_Layout.cshtml
@@ -21,16 +21,7 @@
diff --git a/LLama.Web/Pages/Shared/_Parameters.cshtml b/LLama.Web/Pages/Shared/_Parameters.cshtml
new file mode 100644
index 000000000..165b65a87
--- /dev/null
+++ b/LLama.Web/Pages/Shared/_Parameters.cshtml
@@ -0,0 +1,137 @@
+@page
+@model LLama.Abstractions.IInferenceParams
+@{
+}
+
+
+
+
+
+ MaxTokens
+
+
+
+
+ TokensKeep
+
+
+
+
+
+
+
+
+ TopK
+
+
+
+
+ TopP
+
+
+
+
+
+
+ TypicalP
+
+
+
+
+ Temperature
+
+
+
+
+
+
+ RepeatPenalty
+
+
+
+
+ RepeatLastTokensCount
+
+
+
+
+
+
+ FrequencyPenalty
+
+
+
+
+ PresencePenalty
+
+
+
+
+
+
+
+ TfsZ
+
+
+
+ -
+
+
+
+ Mirostat
+ @Html.DropDownListFor(m => m.Mirostat, Html.GetEnumSelectList(), new { @class = "form-control form-select" })
+
+
+
+
\ No newline at end of file
diff --git a/LLama.Web/Program.cs b/LLama.Web/Program.cs
index 6db653a14..0d2aa35de 100644
--- a/LLama.Web/Program.cs
+++ b/LLama.Web/Program.cs
@@ -20,6 +20,7 @@ public static void Main(string[] args)
.BindConfiguration(nameof(LLamaOptions));
// Services DI
+ builder.Services.AddSingleton
+ MirostatTau
+
+
+
+
+ MirostatEta
+
+
+