From 426df282b8cd8dc66318cd116876fa2d6ef82526 Mon Sep 17 00:00:00 2001 From: Andreas Eiselt Date: Sat, 22 Aug 2026 18:12:59 +0200 Subject: [PATCH] [AI-3] Register AI features and resolve their models at runtime Adds the feature registry: each AI feature declares its kind, the capabilities it requires and whether administrators may override its model, following the FeatureDecisions idiom and living in lib_static for the same reload-safety reason. The AI models page then lets an administrator bind each registered feature to a model, fed by a query that never hides a model: an option is selectable, selectable with a warning when a required capability is unknown, or disabled with the reason when it is ruled out. Llm::Runtime is the single place a feature's model is resolved: explicit override, then binding, then the connection default, and otherwise unbound. It fails closed with a machine-readable reason rather than substituting another model, because a text transform with a different model is a different feature. Bindings reference models by identifier string, never by foreign key, so a binding survives its model disappearing and the dangling state is derived, not stored. With something to bind, the connection form gains the default embedding model selector, offering only models actually known to embed, and the destructive dialogs now name the features that would be affected. Part 10 of the AI-3 stack. https://community.openproject.org/work_packages/66020 --- .../delete_model_dialog_component.rb | 14 +- .../disconnect_dialog_component.html.erb | 8 +- .../disconnect_dialog_component.rb | 4 + .../feature_binding_component.html.erb | 53 +++++ .../feature_binding_component.rb | 111 +++++++++++ .../llm_connections/base_contract.rb | 19 +- .../admin/llm_connections_controller.rb | 2 +- .../admin/llm_feature_bindings_controller.rb | 129 ++++++++++++ app/forms/llm_connections/connection_form.rb | 81 +++++++- .../llm_connections/feature_binding_form.rb | 143 ++++++++++++++ app/models/llm_connection.rb | 1 + app/models/llm_feature_binding.rb | 135 +++++++++++++ app/models/llm_model.rb | 2 + app/services/llm/runtime.rb | 162 +++++++++++++++ .../selectable_models_query.rb | 96 +++++++++ .../admin/llm_feature_bindings/index.html.erb | 63 ++++++ config/initializers/llm_features.rb | 52 +++++ config/initializers/menus.rb | 6 + config/locales/en.yml | 57 ++++++ config/routes.rb | 4 + ...60811140100_create_llm_feature_bindings.rb | 52 +++++ lib_static/open_project/llm/features.rb | 120 +++++++++++ spec/features/admin/llm_connection_spec.rb | 14 ++ spec/models/llm_model_deactivation_spec.rb | 17 ++ spec/requests/admin/llm_connections_spec.rb | 115 +++++++++++ .../admin/llm_feature_bindings_spec.rb | 186 ++++++++++++++++++ spec/requests/admin/llm_models_spec.rb | 132 +++++++++---- spec/services/llm/runtime_spec.rb | 176 +++++++++++++++++ 28 files changed, 1907 insertions(+), 47 deletions(-) create mode 100644 app/components/llm_connections/feature_binding_component.html.erb create mode 100644 app/components/llm_connections/feature_binding_component.rb create mode 100644 app/controllers/admin/llm_feature_bindings_controller.rb create mode 100644 app/forms/llm_connections/feature_binding_form.rb create mode 100644 app/models/llm_feature_binding.rb create mode 100644 app/services/llm/runtime.rb create mode 100644 app/services/llm_connections/selectable_models_query.rb create mode 100644 app/views/admin/llm_feature_bindings/index.html.erb create mode 100644 config/initializers/llm_features.rb create mode 100644 db/migrate/20260811140100_create_llm_feature_bindings.rb create mode 100644 lib_static/open_project/llm/features.rb create mode 100644 spec/requests/admin/llm_feature_bindings_spec.rb create mode 100644 spec/services/llm/runtime_spec.rb diff --git a/app/components/llm_connections/delete_model_dialog_component.rb b/app/components/llm_connections/delete_model_dialog_component.rb index 970d5efdf961..cdba173fa107 100644 --- a/app/components/llm_connections/delete_model_dialog_component.rb +++ b/app/components/llm_connections/delete_model_dialog_component.rb @@ -41,11 +41,17 @@ def form_arguments { action: url_helpers.llm_model_path(llm_model), method: :delete } end - # Named so the message says what is actually at stake. The connection - # defaults count as bindings here -- deleting their model breaks every - # feature that inherits them. + # Named so the message says what is actually at stake: features bound to this + # model stop resolving, rather than silently falling back to another one. + # The connection defaults count as bindings here -- deleting their model + # breaks every feature that inherits them. def bound_features - affected_defaults + bindings = llm_model.llm_connection + .feature_bindings + .where(model_id: llm_model.external_id) + .filter_map { |binding| binding.feature&.label } + + bindings + affected_defaults end def affected_defaults diff --git a/app/components/llm_connections/disconnect_dialog_component.html.erb b/app/components/llm_connections/disconnect_dialog_component.html.erb index 20ed7bbbddbe..b5affa99625d 100644 --- a/app/components/llm_connections/disconnect_dialog_component.html.erb +++ b/app/components/llm_connections/disconnect_dialog_component.html.erb @@ -20,7 +20,13 @@ safe_join( [ content_tag(:li, t("admin.llm_connections.disconnect.keeps_settings")), - content_tag(:li, t("admin.llm_connections.disconnect.keeps_models")) + content_tag(:li, t("admin.llm_connections.disconnect.keeps_models")), + if bound_features.any? + content_tag( + :li, + t("admin.llm_connections.disconnect.keeps_bindings", features: bound_features.to_sentence) + ) + end ].compact ) end diff --git a/app/components/llm_connections/disconnect_dialog_component.rb b/app/components/llm_connections/disconnect_dialog_component.rb index 38e133204a7d..4df992cf05da 100644 --- a/app/components/llm_connections/disconnect_dialog_component.rb +++ b/app/components/llm_connections/disconnect_dialog_component.rb @@ -52,5 +52,9 @@ class DisconnectDialogComponent < ApplicationComponent def form_arguments { action: url_helpers.disconnect_llm_connection_path, method: :post } end + + def bound_features + connection.feature_bindings.filter_map { |binding| binding.feature&.label if binding.model_id.present? } + end end end diff --git a/app/components/llm_connections/feature_binding_component.html.erb b/app/components/llm_connections/feature_binding_component.html.erb new file mode 100644 index 000000000000..762e4a7b5705 --- /dev/null +++ b/app/components/llm_connections/feature_binding_component.html.erb @@ -0,0 +1,53 @@ +<%= render(Primer::Box.new(border: true, border_radius: 2, p: 3, mb: 3)) do %> + <%= render(Primer::Beta::Text.new(tag: :h3, font_size: 4, font_weight: :bold, mb: 1)) { feature.label } %> + + <% if feature.caption.present? %> + <%= render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 2)) { feature.caption } %> + <% end %> + + <% if dangling? %> + <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :alert)) do %> + <%= t("admin.llm_feature_bindings.dangling", model: binding.resolved_model_id) %> + <% end %> + <% end %> + + <% if deactivated? %> + <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :alert)) do %> + <%= t("admin.llm_feature_bindings.deactivated", model: binding.resolved_model_id) %> + <% end %> + <% end %> + + <% if locked? %> + <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :lock)) do %> + <%= t("admin.llm_feature_bindings.locked", model: binding.model_id) %> + <% end %> + <% end %> + + <%= primer_form_with(model: form_model, url: form_url, method: :patch, scope: :llm_feature_binding) do |f| %> + <%= render( + LlmConnections::FeatureBindingForm.new( + f, + options: model_options, + inherit_label:, + feature_key: feature.key, + locked: locked?, + embedding: feature.embedding?, + selected_model_id: binding&.model_id, + dimensions_hint: probed_dimensions + ) + ) %> + <% end %> + + <% if locked? && feature.embedding? %> + <%# Rendered as text rather than disabled inputs: a disabled input submits + nothing, so the values would arrive blank and wipe the columns. %> + <%= render(Primer::Beta::Text.new(tag: :p, font_weight: :bold, mt: 2, mb: 1)) do %> + <%= t("admin.llm_feature_bindings.locked_values_heading") %> + <% end %> + <% locked_values.each do |label, value| %> + <%= render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 0)) do %> + <%= "#{label}: #{value}" %> + <% end %> + <% end %> + <% end %> +<% end %> diff --git a/app/components/llm_connections/feature_binding_component.rb b/app/components/llm_connections/feature_binding_component.rb new file mode 100644 index 000000000000..9d5d16e5a555 --- /dev/null +++ b/app/components/llm_connections/feature_binding_component.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # One feature's row on the model assignment page. + class FeatureBindingComponent < ApplicationComponent + include ApplicationHelper + include OpPrimer::ComponentHelpers + + def initialize(feature:, connection:, binding: nil) + super(feature) + @feature = feature + @connection = connection + @binding = binding + end + + # The record the select binds to. A feature without a stored binding still + # needs one so the form has a model_id to read. + def form_model + binding || connection.feature_bindings.new(feature_key: feature.key.to_s) + end + + # Not named +options+: ApplicationComponent already owns that name and + # initialises it to an empty hash, which silently swallowed the memoisation. + def model_options + @model_options ||= SelectableModelsQuery.new(connection, feature).call + end + + def inherit_label + if default_model_id.present? + I18n.t("admin.llm_feature_bindings.inherit_with_default", model: default_model_id) + else + I18n.t("admin.llm_feature_bindings.inherit_without_default") + end + end + + def locked? = binding&.locked? + + def dangling? = binding&.dangling? + + # What the embeddings probe last saw, offered as information. Never filled + # into the field: the server decides the vector size at index time. + def probed_dimensions + return unless feature.embedding? + + model_id = binding&.resolved_model_id + return if model_id.blank? + + connection.capability_verdicts.for_model(model_id).for_capability(:embeddings).first&.dimensions + end + + # Quoted so a trailing space -- load-bearing for the E5 and BGE families -- + # is visible rather than invisible. + def locked_values + [ + [LlmFeatureBinding.human_attribute_name(:model_id), binding.model_id], + [LlmFeatureBinding.human_attribute_name(:dimensions), binding.dimensions || "—"], + [LlmFeatureBinding.human_attribute_name(:input_prefix), binding.input_prefix.to_s.inspect], + [LlmFeatureBinding.human_attribute_name(:query_prefix), binding.query_prefix.to_s.inspect] + ] + end + + # Still resolvable, so not dangling -- but an administrator has hidden it + # from the pickers, so say so rather than let the choice look unremarkable. + def deactivated? + model_id = binding&.resolved_model_id + return false if model_id.blank? + + connection.models.deactivated.exists?(external_id: model_id) + end + + private + + attr_reader :feature, :connection, :binding + + def default_model_id + feature.embedding? ? connection.default_embedding_model_id : connection.default_chat_model_id + end + + def form_url + url_helpers.llm_feature_binding_path(feature.key) + end + end +end diff --git a/app/contracts/llm_connections/base_contract.rb b/app/contracts/llm_connections/base_contract.rb index 6a01d5a7d9c9..3c7fabe022e7 100644 --- a/app/contracts/llm_connections/base_contract.rb +++ b/app/contracts/llm_connections/base_contract.rb @@ -38,8 +38,6 @@ class BaseContract < ModelContract attribute :api_key attribute :default_chat_model_id attribute :default_embedding_model_id - attribute :default_chat_model_id - attribute :default_embedding_model_id validates :base_url, presence: true validates :api_format, inclusion: { in: Llm::Adapters::FORMATS } @@ -56,6 +54,7 @@ class BaseContract < ModelContract validate :enabled_requires_connection validate :default_models_offered_by_server validate :default_chat_model_can_chat + validate :default_embedding_model_can_embed validate :not_configured_from_env def not_configured_from_env @@ -66,6 +65,22 @@ def not_configured_from_env private + # A model the server has positively told us cannot embed is not a candidate + # for the embedding default, however it got submitted. An unknown verdict + # does not block: that is the normal state for a server that publishes + # nothing about its models. + def default_embedding_model_can_embed + model_id = model.default_embedding_model_id + return if model_id.blank? + return unless model.changed_attributes.include?("default_embedding_model_id") + + unsupported = model.capability_verdicts + .for_capability(:embeddings) + .exists?(model_id:, state: "unsupported") + + errors.add(:default_embedding_model_id, :cannot_embed) if unsupported + end + # The mirror image of default_embedding_model_can_embed: a model the server # positively identifies as an embedding model is not a chat candidate. def default_chat_model_can_chat diff --git a/app/controllers/admin/llm_connections_controller.rb b/app/controllers/admin/llm_connections_controller.rb index 36c140f48991..dfba30ac110b 100644 --- a/app/controllers/admin/llm_connections_controller.rb +++ b/app/controllers/admin/llm_connections_controller.rb @@ -160,7 +160,7 @@ def redirect_with_error(message) # saved value, so submitting it unchanged posts an empty string. def llm_connection_params permitted = params.expect( - llm_connection: %i[enabled api_format base_url api_key default_chat_model_id] + llm_connection: %i[enabled api_format base_url api_key default_chat_model_id default_embedding_model_id] ) permitted.delete(:api_key) if permitted[:api_key].blank? permitted.to_h.symbolize_keys diff --git a/app/controllers/admin/llm_feature_bindings_controller.rb b/app/controllers/admin/llm_feature_bindings_controller.rb new file mode 100644 index 000000000000..c531695799ee --- /dev/null +++ b/app/controllers/admin/llm_feature_bindings_controller.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Admin + # Assigns a model to each registered AI feature. + class LlmFeatureBindingsController < ApplicationController + layout "admin" + menu_item :llm_feature_bindings + + before_action :require_feature + before_action :require_admin + before_action :set_connection + + def index + @features = OpenProject::Llm::Features.available + @bindings = bindings_by_feature_key + end + + def update + feature = OpenProject::Llm::Features[params[:id]] + assign(feature) + + redirect_to llm_feature_bindings_path, status: :see_other + rescue OpenProject::Llm::UnknownFeature + render_404 + end + + private + + def set_connection + @connection = LlmConnection.instance + end + + # The flag gates the endpoints, not only the menu entry: an unfinished page + # must not accept writes just because somebody knows the URL. + def require_feature + render_404 unless OpenProject::FeatureDecisions.llm_connection_active? + end + + def bindings_by_feature_key + @connection.feature_bindings.index_by(&:feature_key) + end + + def binding_for(feature) + @connection.feature_bindings.find_or_initialize_by(feature_key: feature.key.to_s) + end + + def assign(feature) + binding = build_binding(feature) + + unless binding.save + flash[:error] = binding.errors.full_messages.join(", ") + return + end + + confirm_assignment(feature, probe_capabilities(feature, binding)) + end + + # The probe may just have proven the chosen model cannot do what the feature + # requires; confirming that save would report a working configuration that + # Llm::Runtime immediately resolves as incapable. + def confirm_assignment(feature, verdict) + if verdict&.blocking? + flash[:error] = t("admin.llm_feature_bindings.update.model_incapable", + feature: feature.label, + capability: Llm::Capabilities.label(:embeddings)) + else + flash[:notice] = t("admin.llm_feature_bindings.update.success", feature: feature.label) + end + end + + def build_binding(feature) + binding = binding_for(feature) + binding.model_id = params.dig(:llm_feature_binding, :model_id).presence + + # Only ever accepted for the kind of feature they describe; the model + # rejects them elsewhere, and they are not read at all for a chat feature. + assign_embedding_settings(binding) if feature.embedding? + + binding + end + + # The prefixes are stored exactly as typed. The trailing space in "passage: " + # is load-bearing for the E5 and BGE families, so stripping would silently + # degrade retrieval. + def assign_embedding_settings(binding) + settings = params.fetch(:llm_feature_binding, {}) + + binding.dimensions = settings[:dimensions].presence + binding.input_prefix = settings[:input_prefix] + binding.query_prefix = settings[:query_prefix] + end + + # The verdict that actually matters is the one for the model an administrator + # just chose, so it is fetched now rather than left unknown until first use. + def probe_capabilities(feature, binding) + return if feature.requires.empty? || binding.model_id.blank? + + LlmConnections::DetectCapabilitiesService.new(@connection).detect(binding.model_id).result + end + end +end diff --git a/app/forms/llm_connections/connection_form.rb b/app/forms/llm_connections/connection_form.rb index 9d3cadcb823e..edc61445cb3d 100644 --- a/app/forms/llm_connections/connection_form.rb +++ b/app/forms/llm_connections/connection_form.rb @@ -104,6 +104,32 @@ class ConnectionForm < ApplicationForm end end + # Only worth asking for once something embeds. The column, the contract + # attribute, its validation and every translation for this field already + # existed; the input was simply never rendered, so the value could not be + # set through the UI at all. + if embedding_features? + f.autocompleter( + name: :default_embedding_model_id, + label: LlmConnection.human_attribute_name(:default_embedding_model_id), + caption: default_embedding_model_caption, + disabled: read_only?, + autocomplete_options: { + decorated: true, + inputValue: model.default_embedding_model_id, + placeholder: I18n.t("label_none_parentheses") + } + ) do |list| + list.option(label: I18n.t("label_none_parentheses"), value: "", + selected: model.default_embedding_model_id.blank?) + + default_embedding_model_options.each do |model_id| + list.option(label: embedding_option_label(model_id, embeddings_state(model_id)), + value: model_id, + selected: model.default_embedding_model_id == model_id) + end + end + end end unless read_only? @@ -136,13 +162,38 @@ def default_chat_model_options (chat_capable + [model.default_chat_model_id]).compact_blank.uniq end - # The same friendly name the model table shows; the identifier stays the value. - def option_label(model_id) - model_names[model_id].presence || model_id + # Only models actually known to embed. + # + # An unconfirmed capability is not a capability: offering a model here on the + # grounds that nothing has ruled it out invites an administrator to pick one + # that cannot embed, and the failure would surface much later, at index time. + # A catalogue from a registry-backed provider makes that vivid -- 132 models, + # 3 of which embed. + # + # This leaves nothing to choose when no model is known to embed, and that is + # the honest state rather than a dead end: an administrator who knows better + # than the registry says so on the model itself, by setting its embeddings + # capability, which is what default_embedding_model_hint points at. + # + # The model already chosen is kept regardless, so a save cannot silently + # blank a working configuration. + def default_embedding_model_options + capable = model.selectable_model_ids.select { |id| embeddings_state(id) == :supported } + + (capable + [model.default_embedding_model_id]).compact_blank.uniq end - def model_names - @model_names ||= model.models.pluck(:external_id, :display_name).to_h + # Says how to make a model eligible when none is, rather than leaving an + # empty select with no explanation. + def default_embedding_model_caption + return I18n.t("admin.llm_connections.form.default_embedding_model_caption") if + default_embedding_model_options.any? + + I18n.t("admin.llm_connections.form.default_embedding_model_none") + end + + def embedding_features? + OpenProject::Llm::Features.for_kind(:embedding).any? end # No verdict at all is the same as an inconclusive one: we do not know. @@ -157,6 +208,26 @@ def embeddings_verdicts .to_h end + def embedding_option_label(model_id, state) + case state + when :unsupported + I18n.t("admin.llm_connections.form.embedding_option_unsupported", model: option_label(model_id)) + when :unknown + I18n.t("admin.llm_connections.form.embedding_option_unknown", model: option_label(model_id)) + else + option_label(model_id) + end + end + + # The same friendly name the model table shows; the identifier stays the value. + def option_label(model_id) + model_names[model_id].presence || model_id + end + + def model_names + @model_names ||= model.models.pluck(:external_id, :display_name).to_h + end + def submit_label model.persisted? ? I18n.t(:button_save) : I18n.t("admin.llm_connections.form.button_connect") end diff --git a/app/forms/llm_connections/feature_binding_form.rb b/app/forms/llm_connections/feature_binding_form.rb new file mode 100644 index 000000000000..cf7a50fbd5f7 --- /dev/null +++ b/app/forms/llm_connections/feature_binding_form.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # The model select for one registered feature. + class FeatureBindingForm < ApplicationForm + # Primer::Forms::Base.new assigns the builder itself and calls this with the + # remaining keywords, so the builder must not appear in the signature. + def initialize(options:, inherit_label:, feature_key:, locked: false, embedding: false, dimensions_hint: nil, + selected_model_id: nil) + super() + @selected_model_id = selected_model_id + @model_options = options + @inherit_label = inherit_label + @feature_key = feature_key + @locked = locked + @embedding = embedding + @dimensions_hint = dimensions_hint + end + + form do |f| + # An autocompleter rather than a select, so a model can be found by typing + # among the hundreds a gateway reports. decorated: true serialises the list + # into the element, so no endpoint is needed. + f.autocompleter( + name: :model_id, + label: LlmFeatureBinding.human_attribute_name(:model_id), + disabled: locked, + autocomplete_options: { + decorated: true, + inputValue: selected_model_id, + placeholder: inherit_label + }, + data: { test_selector: "llm-feature-binding--model-#{feature_key}" } + ) do |list| + list.option(label: inherit_label, value: "", selected: selected_model_id.blank?) + + model_options.each do |option| + # Listed but not choosable when a required capability is known to be + # missing: hiding it would leave the reason invisible too. + list.option(label: option_label(option), + value: option.model_id, + selected: selected_model_id == option.model_id, + disabled: !option.selectable?) + end + end + + # Only for an embedding feature, and only while unlocked. A locked binding + # renders these as text instead: a disabled input submits nothing, so the + # values would arrive blank and wipe the columns. + if embedding && !locked + f.text_field( + name: :dimensions, + type: :number, + min: 1, + label: LlmFeatureBinding.human_attribute_name(:dimensions), + caption: dimensions_caption, + input_width: :small, + data: { test_selector: "llm-feature-binding--dimensions-#{feature_key}" } + ) + + f.text_field( + name: :input_prefix, + label: LlmFeatureBinding.human_attribute_name(:input_prefix), + caption: I18n.t("admin.llm_feature_bindings.form.input_prefix_caption"), + input_width: :medium, + data: { test_selector: "llm-feature-binding--input-prefix-#{feature_key}" } + ) + + f.text_field( + name: :query_prefix, + label: LlmFeatureBinding.human_attribute_name(:query_prefix), + caption: I18n.t("admin.llm_feature_bindings.form.query_prefix_caption"), + input_width: :medium, + data: { test_selector: "llm-feature-binding--query-prefix-#{feature_key}" } + ) + end + + unless locked + f.submit( + name: :submit, + label: I18n.t(:button_save), + scheme: :secondary, + data: { test_selector: "llm-feature-binding--submit-#{feature_key}" } + ) + end + end + + private + + attr_reader :model_options, :inherit_label, :feature_key, :locked, :embedding, :dimensions_hint, + :selected_model_id + + # Blank is the right default: the server decides the vector size, and baking + # in a number it may contradict helps nobody. Where the probe has already + # seen a vector, its size is offered as information rather than filled in. + def dimensions_caption + return I18n.t("admin.llm_feature_bindings.form.dimensions_caption") if dimensions_hint.blank? + + I18n.t("admin.llm_feature_bindings.form.dimensions_caption_probed", dimensions: dimensions_hint) + end + + def option_label(option) + case option.state + when :unsupported + I18n.t("admin.llm_feature_bindings.option_unsupported", + model: option.model_id, + capability: option.reasons.map { |reason| Llm::Capabilities.label(reason) }.join(", ")) + when :unknown + I18n.t("admin.llm_feature_bindings.option_unknown", model: option.model_id) + else + option.model_id + end + end + end +end diff --git a/app/models/llm_connection.rb b/app/models/llm_connection.rb index 1e35e858f24e..e146d378d3ab 100644 --- a/app/models/llm_connection.rb +++ b/app/models/llm_connection.rb @@ -41,6 +41,7 @@ class LlmConnection < ApplicationRecord has_many :health_reports, as: :subject, dependent: :delete_all has_many :models, class_name: "LlmModel", dependent: :delete_all has_many :capability_verdicts, class_name: "LlmCapabilityVerdict", dependent: :delete_all + has_many :feature_bindings, class_name: "LlmFeatureBinding", dependent: :delete_all validates :base_url, presence: true validate :only_one_connection, on: :create diff --git a/app/models/llm_feature_binding.rb b/app/models/llm_feature_binding.rb new file mode 100644 index 000000000000..321b30ad6106 --- /dev/null +++ b/app/models/llm_feature_binding.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +# Which model a registered feature uses. +# +# One row per feature, not per binding: rows are reconciled against the registry +# and are never destroyed when a feature deregisters, so flipping a feature flag +# does not lose the administrator's choice. +class LlmFeatureBinding < ApplicationRecord + belongs_to :llm_connection + + # Settings that describe how vectors are written, and so only mean anything + # for an embedding feature. + EMBEDDING_SETTINGS = %i[dimensions input_prefix query_prefix].freeze + + # Everything a stored index depends on. Changing any of it invalidates the + # vectors already written, not just the model. + LOCKED_SETTINGS = ([:model_id] + EMBEDDING_SETTINGS).freeze + + before_save :freeze_inherited_model_on_lock + + validates :feature_key, presence: true, uniqueness: { scope: :llm_connection_id } + validates :dimensions, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true + validate :feature_registered + validate :embedding_settings_only_for_embedding_features + validate :locked_settings_unchanged + + def feature + OpenProject::Llm::Features[feature_key] + rescue OpenProject::Llm::UnknownFeature + nil + end + + # NULL means "use the connection default for this kind of model". + def resolved_model_id + model_id.presence || default_model_id + end + + def inherits_default? = model_id.blank? + + # Derived, never stored. A status column would be a cache with no invalidation + # trigger, and would be stale exactly when it matters -- right after the remote + # catalogue changed. + def dangling? + resolved = resolved_model_id + resolved.present? && llm_connection.available_model_ids.exclude?(resolved) + end + + def locked? = locked_at.present? + + private + + # A lock freezes what the vectors were written with. A binding that inherits + # the connection default would keep following it after the lock, so the + # resolved model is written down at the moment of locking. + def freeze_inherited_model_on_lock + return unless locked_at.present? && locked_at_changed? && locked_at_was.nil? + return if model_id.present? + + self.model_id = default_model_id + end + + def default_model_id + return if feature.nil? + + feature.embedding? ? llm_connection.default_embedding_model_id : llm_connection.default_chat_model_id + end + + def feature_registered + return if feature.present? + + errors.add(:feature_key, :not_registered) + end + + def embedding_settings_only_for_embedding_features + return if feature.nil? || feature.embedding? + + EMBEDDING_SETTINGS.each do |attribute| + next if public_send(attribute).blank? + + errors.add(attribute, :not_for_chat_feature) + end + end + + # Vectors written under one embedding model are meaningless under another, and + # the dimension count is baked into the index, so a locked binding can only be + # changed by an explicit re-index. + # + # The prefixes are locked for the same reason and matter just as much: an index + # built with "passage: " but queried under a different prefix does not error, + # it quietly returns worse results, which is the hardest kind of failure to + # notice. + # + # TODO(#69620): re-indexing is what clears locked_at. Until that job exists a + # locked binding can only be changed in the database. + def locked_settings_unchanged + # Only constrains later edits. On the save that records the lock -- and on + # create -- every attribute reads as changed from nil, and there is nothing + # indexed yet for them to contradict. + return unless locked? && locked_at_was.present? + + LOCKED_SETTINGS.each do |attribute| + next unless public_send(:"#{attribute}_changed?") + + errors.add(attribute, :locked) + end + end +end diff --git a/app/models/llm_model.rb b/app/models/llm_model.rb index 1a5940463596..f7fbd34630c1 100644 --- a/app/models/llm_model.rb +++ b/app/models/llm_model.rb @@ -61,6 +61,8 @@ def cascade_rename!(previous_external_id) return if previous_external_id.blank? || previous_external_id == external_id llm_connection.capability_verdicts.where(model_id: previous_external_id).update_all(model_id: external_id) + llm_connection.feature_bindings.where(model_id: previous_external_id).update_all(model_id: external_id) + rename_connection_defaults(previous_external_id) end diff --git a/app/services/llm/runtime.rb b/app/services/llm/runtime.rb new file mode 100644 index 000000000000..1c2d9027bab2 --- /dev/null +++ b/app/services/llm/runtime.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + # Answers "which model should this feature use, and can it run right now?". + # + # The single place model resolution happens, so that every feature agrees on + # what an unset value means: + # + # per-item override -> feature binding -> connection default -> unbound + # + # A blank value at any level means "inherit from the level below". + class Runtime + # :ready - go ahead + # :feature_disabled - the feature's own toggle is off + # :no_connection - no LLM server configured, or AI switched off globally + # :unbound - nothing has chosen a model for this feature yet + # :model_missing - the chosen model is not in the server's catalogue + # :incapable - the chosen model is known not to support what is needed + Resolution = Data.define(:feature, :connection, :model_id, :status, :missing_capabilities) do + def ready? = status == :ready + + # A chat builder for the resolved model. + # + # Note that RubyLLM enforces none of the feature's declared requirements: + # Chat#with_schema performs no capability check, and a model absent from + # RubyLLM's registry is described by Model::Info.default, which claims + # structured output, vision and function calling for everything. The + # capability verdicts consulted in #call above are the only real gate. + # + # @return [RubyLLM::Chat] + def chat(**) + ensure_usable!(:chat) + session(**).chat(model_id) + end + + # @return [RubyLLM::Embedding] + def embed(input, dimensions: nil, **) + ensure_usable!(:embedding) + session(**).embed(input, model: model_id, dimensions:) + end + + # @return [Llm::Session] + def session(**) + Llm::Session.for(connection, **) + end + + private + + # Features are resolved by kind, so asking a chat feature to embed means a + # caller has confused two features -- a bug, not a configuration problem. + def ensure_usable!(kind) + raise Llm::Errors::NotReady, status unless ready? + return if feature.public_send(:"#{kind}?") + + raise Llm::Errors::NotReady, :wrong_kind + end + end + + class << self + # @param feature_key [Symbol] a key registered with OpenProject::Llm::Features + # @param override [String, nil] a per-item model choice, e.g. one stored on + # a description assistant action. Blank means inherit. + def for(feature_key, override: nil) + new(OpenProject::Llm::Features[feature_key], override:).call + end + end + + def initialize(feature, override: nil) + @feature = feature + @override = override + end + + def call + return resolution(:feature_disabled) unless feature.available? + return resolution(:no_connection) unless LlmConnection.available? + + model_id = resolved_model_id + return resolution(:unbound) if model_id.blank? + return resolution(:model_missing, model_id:) unless connection.available_model_ids.include?(model_id) + + missing = unsupported_capabilities(model_id) + return resolution(:incapable, model_id:, missing_capabilities: missing) if missing.any? + + resolution(:ready, model_id:) + end + + private + + attr_reader :feature, :override + + def connection + @connection ||= LlmConnection.instance + end + + def resolved_model_id + effective_override || binding_model_id || connection_default + end + + # A pinned feature declared overridable: false must not follow a caller's + # override: semantic_search's vectors were written with one model, and a + # different one at query time is silently wrong answers, not a preference. + def effective_override + return unless feature.overridable + + override.presence + end + + def binding_model_id + connection.feature_bindings.find_by(feature_key: feature.key.to_s)&.model_id.presence + end + + def connection_default + feature.embedding? ? connection.default_embedding_model_id : connection.default_chat_model_id + end + + # Only a definite :unsupported blocks. An :unknown verdict -- which is the + # normal state for a server that reports nothing about its models -- is + # surfaced in the UI as a warning but never prevents a call. + def unsupported_capabilities(model_id) + return [] if feature.requires.empty? + + blocking = connection.capability_verdicts + .for_model(model_id) + .where(capability: feature.requires.map(&:to_s), state: "unsupported") + + blocking.pluck(:capability).map(&:to_sym) + end + + def resolution(status, model_id: nil, missing_capabilities: []) + Resolution.new(feature:, connection: status == :feature_disabled ? nil : connection, + model_id:, status:, missing_capabilities:) + end + end +end diff --git a/app/services/llm_connections/selectable_models_query.rb b/app/services/llm_connections/selectable_models_query.rb new file mode 100644 index 000000000000..1ca7d8a4a974 --- /dev/null +++ b/app/services/llm_connections/selectable_models_query.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + # The models offerable to a feature, each with why it is or is not usable. + # + # Models are never hidden. Hiding one produces the single support question + # nobody can answer -- "why can I not pick the model I know works" -- and it is + # exactly wrong when most verdicts are unknown. Instead each option carries a + # state the UI renders: selectable, selectable with a warning, or disabled with + # a reason. + class SelectableModelsQuery + Option = Data.define(:model_id, :state, :reasons) do + def selectable? = state != :unsupported + + def warning? = state == :unknown + end + + def initialize(connection, feature) + @connection = connection + @feature = feature + end + + def call + offerable_model_ids.map { |model_id| option_for(model_id) } + end + + private + + attr_reader :connection, :feature + + # Models an administrator has switched off are not offered, but the one this + # feature is already bound to stays listed -- otherwise the select silently + # shows nothing where a working binding exists. + def offerable_model_ids + (connection.selectable_model_ids + [bound_model_id]).compact_blank.uniq + end + + def bound_model_id + connection.feature_bindings.find_by(feature_key: feature.key.to_s)&.model_id + end + + def option_for(model_id) + states = feature.requires.index_with { |capability| verdict_state(model_id, capability) } + + if states.value?(:unsupported) + Option.new(model_id:, state: :unsupported, + reasons: states.select { |_, s| s == :unsupported }.keys) + elsif states.value?(:unknown) + Option.new(model_id:, state: :unknown, + reasons: states.select { |_, s| s == :unknown }.keys) + else + Option.new(model_id:, state: :supported, reasons: []) + end + end + + # No verdict at all is the same as an inconclusive one: we do not know. + def verdict_state(model_id, capability) + verdicts.dig(model_id, capability.to_s)&.to_sym || :unknown + end + + def verdicts + @verdicts ||= connection.capability_verdicts + .pluck(:model_id, :capability, :state) + .group_by(&:first) + .transform_values { |rows| rows.to_h { |(_, capability, state)| [capability, state] } } + end + end +end diff --git a/app/views/admin/llm_feature_bindings/index.html.erb b/app/views/admin/llm_feature_bindings/index.html.erb new file mode 100644 index 000000000000..7c026aac983d --- /dev/null +++ b/app/views/admin/llm_feature_bindings/index.html.erb @@ -0,0 +1,63 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<% html_title t(:label_administration), t("menus.admin.llm_feature_bindings") %> + +<%= + render(Primer::OpenProject::PageHeader.new) do |header| + header.with_title { t("menus.admin.llm_feature_bindings") } + header.with_description { t(".description") } + header.with_breadcrumbs( + [{ href: admin_index_path, text: t(:label_administration) }, + { href: mcp_configurations_path, text: t("menus.admin.ai") }, + t("menus.admin.llm_feature_bindings")] + ) + end +%> + +<% if !@connection.configured? %> + <%= + render(Primer::Beta::Blankslate.new(border: true)) do |component| + component.with_visual_icon(icon: :sparkle) + component.with_heading(tag: :h2) { t(".blank_title") } + component.with_description { t(".blank_description") } + component.with_primary_action(href: llm_connection_path) { t("menus.admin.llm_connection") } + end + %> +<% else %> + <% @features.each do |feature| %> + <%= render( + LlmConnections::FeatureBindingComponent.new( + feature:, + connection: @connection, + binding: @bindings[feature.key.to_s] + ) + ) %> + <% end %> +<% end %> diff --git a/config/initializers/llm_features.rb b/config/initializers/llm_features.rb new file mode 100644 index 000000000000..568d977063fa --- /dev/null +++ b/config/initializers/llm_features.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require_relative "../../lib_static/open_project/llm/features" + +# Features that send requests to the configured LLM server. +# +# Add a feature here (or from a module engine initializer) so that +# administrators can assign it a model on the "AI models" page. + +# The description assistant rewrites work package text on explicit user action. +# Plain chat completions only: no tools, no JSON mode, no streaming. Individual +# actions may override the model, which is why it is overridable. +OpenProject::Llm::Features.register :description_assistant, + kind: :chat, + prefers: %i[structured_output], + overridable: true + +# Semantic search embeds work packages into a pgvector index. Pinned because the +# stored vectors are meaningless under a different model: changing it is a +# destructive re-index rather than a swap. +OpenProject::Llm::Features.register :semantic_search, + kind: :embedding, + requires: %i[embeddings], + pinned: true diff --git a/config/initializers/menus.rb b/config/initializers/menus.rb index bed2ac6b42cc..2872ac21a09a 100644 --- a/config/initializers/menus.rb +++ b/config/initializers/menus.rb @@ -510,6 +510,12 @@ caption: I18n.t("menus.admin.llm_connection"), parent: :ai + menu.push :llm_feature_bindings, + { controller: "/admin/llm_feature_bindings", action: :index }, + if: ->(_) { User.current.admin? && OpenProject::FeatureDecisions.llm_connection_active? }, + caption: I18n.t("menus.admin.llm_feature_bindings"), + parent: :ai + menu.push :mcp_configurations, { controller: "/admin/mcp_configurations", action: :index }, if: ->(_) { User.current.admin? }, diff --git a/config/locales/en.yml b/config/locales/en.yml index af81ae9d9885..827106843325 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -208,8 +208,16 @@ en: # ActiveRecord::Base.human_attribute_name strips the _id suffix, so these # keys deliberately do not carry it (see lib/open_project/patches/active_record_i18n.rb). default_chat_model: "Default chat model" + default_embedding_model: "Default embedding model" enabled: "Enable LLMs for this instance" last_connected_at: "Last connected" + llm_feature_binding: + dimensions: "Dimensions" + feature_key: "Feature" + input_prefix: "Document prefix" + # ActiveRecord::Base.human_attribute_name strips the _id suffix. + model: "Model" + query_prefix: "Query prefix" llm_model: display_name: "Display name" # human_attribute_name strips the _id suffix, so the key omits it. @@ -706,6 +714,21 @@ en: not_available: "is not offered by the configured LLM server." enabled: requires_connection: "cannot be turned on before a connection has been configured." + llm_feature_binding: + attributes: + dimensions: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." + feature_key: + not_registered: "does not belong to a known AI feature." + input_prefix: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." + model_id: + locked: "cannot be changed while data indexed with it still exists. Re-index to switch models." + query_prefix: + locked: "cannot be changed while data indexed with it still exists. Re-index to change it." + not_for_chat_feature: "only applies to features that create embeddings." meeting: error_conflict: "Unable to save because the meeting was updated by someone else in the meantime. Please reload the page." member: @@ -1718,6 +1741,7 @@ en: configured_from_env: "This connection is configured through the environment and cannot be changed here." description: "OpenProject will stop using the LLM server. AI features will be unavailable until you connect again." heading: "Disconnect from the LLM server?" + keeps_bindings: "The model chosen for each feature is kept: %{features}." keeps_models: "The model list, including any models you added manually, is kept." keeps_settings: "The endpoint and API format are kept. Only the stored API key is removed." menu_label: "Disconnect" @@ -1731,6 +1755,10 @@ en: The full base URL of the server, including the API version segment, exactly as your provider documents it (for example https://example.com/v1). OpenProject appends only the endpoint path. button_connect: "Connect" default_chat_model_caption: "Used by AI features that do not select a model themselves." + default_embedding_model_caption: "Used by AI features that index text for search and do not select a model themselves. Only models known to create embeddings are offered." + default_embedding_model_none: "No model is known to create embeddings. If you know one does, open it under Available models and set its embeddings capability." + embedding_option_unknown: "%{model} (not verified as an embedding model)" + embedding_option_unsupported: "%{model} (cannot create embeddings)" enabled_caption: "When turned off, all LLM-backed AI features stop working. The connection settings are kept." label_connecting: "Contacting the LLM server…" models: @@ -1762,6 +1790,27 @@ en: update: no_models: "Saved, but the server did not return a model list. Either it does not offer one, or the endpoint is wrong — a URL missing its API version segment (for example /v1) looks exactly the same from here. Add the models you want to use below, then run the health checks to confirm the server answers." success: "Successfully connected to the LLM server." + llm_feature_bindings: + dangling: "%{model} is no longer offered by the LLM server. This feature will not run until another model is selected." + deactivated: "%{model} has been hidden by an administrator. This feature keeps using it, but it can no longer be chosen elsewhere." + form: + dimensions_caption: "How many numbers each vector has. Leave blank to use whatever the server returns." + dimensions_caption_probed: "The server returned %{dimensions}-dimension vectors for this model. Leave blank to use whatever it returns at index time." + input_prefix_caption: "Prepended to each document before it is indexed. Some models expect one, for example \"passage: \" including the trailing space." + query_prefix_caption: "Prepended to each search query. Some models expect one, for example \"query: \" including the trailing space." + index: + blank_description: "Connect OpenProject to an LLM server first. Models can be assigned once the server reports which ones it offers." + blank_title: "No LLM server configured" + description: "Choose which model each AI feature uses. Features without a choice use the instance default." + inherit_with_default: "Use the default (%{model})" + inherit_without_default: "Use the default (none set)" + locked: "%{model} is in use by indexed data and cannot be changed here. Re-index to switch models." + locked_values_heading: "Values fixed by the existing index" + option_unknown: "%{model} — not verified" + option_unsupported: "%{model} — no %{capability} support" + update: + model_incapable: "The model for %{feature} has been saved, but the server just reported that it does not support %{capability}. Pick a different model, or assert the capability on the model if you know better." + success: "The model for %{feature} has been saved." llm_models: create: success: "%{model} has been added." @@ -4290,6 +4339,13 @@ en: context_window_sources: registry: "the figure published for this model" server: "reported by the server" + features: + description_assistant: + caption: "Rewrites and restructures work package text on request." + label: "Description assistant" + semantic_search: + caption: "Indexes work packages so they can be found by meaning rather than by keyword." + label: "Semantic search" model_kinds: chat: "Chat" embedding: "Embedding" @@ -4545,6 +4601,7 @@ en: ai: "Artificial Intelligence (AI)" api_and_webhooks: "API and webhooks" llm_connection: "LLM settings" + llm_feature_bindings: "AI models" mail_notification: "Email notifications" mails_and_notifications: "Emails and notifications" mcp_configurations: "Model Context Protocol (MCP)" diff --git a/config/routes.rb b/config/routes.rb index 54bb86452b7b..f85a16056e5c 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -759,6 +759,10 @@ end end + # Keyed by feature key rather than by record id: the binding is an attribute + # of a registered feature, and a feature may not have a row yet. + resources :llm_feature_bindings, only: %i[index update], controller: "admin/llm_feature_bindings" + resources :mcp_configurations, only: %i[index update], controller: "admin/mcp_configurations" do collection do post :multi_update diff --git a/db/migrate/20260811140100_create_llm_feature_bindings.rb b/db/migrate/20260811140100_create_llm_feature_bindings.rb new file mode 100644 index 000000000000..b9bbc61683e1 --- /dev/null +++ b/db/migrate/20260811140100_create_llm_feature_bindings.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class CreateLlmFeatureBindings < ActiveRecord::Migration[8.1] + def change + create_table :llm_feature_bindings do |t| + t.references :llm_connection, null: false, foreign_key: true + t.string :feature_key, null: false + # NULL means "use the connection default for this kind of model". + t.string :model_id + # Embedding features only. Frozen together with model_id once vectors exist. + t.integer :dimensions + t.string :input_prefix + t.string :query_prefix + # Set once the binding has data depending on it, after which the model + # cannot be swapped without a destructive re-index. + t.datetime :locked_at + t.datetime :last_seen_at + + t.timestamps null: false + end + + add_index :llm_feature_bindings, %i[llm_connection_id feature_key], unique: true + end +end diff --git a/lib_static/open_project/llm/features.rb b/lib_static/open_project/llm/features.rb new file mode 100644 index 000000000000..5165d24ca2e8 --- /dev/null +++ b/lib_static/open_project/llm/features.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module OpenProject + module Llm + class UnknownFeature < StandardError; end + + # A feature that sends requests to the configured LLM server. + # + # Features declare the capabilities they need so the administration UI can + # tell an administrator which models are usable for which job, and so a + # feature never silently runs against a model that cannot serve it. + Feature = Data.define(:key, :kind, :requires, :prefers, :overridable, :pinned, :available, :i18n_scope) do + def available? = available.call + + def chat? = kind == :chat + + def embedding? = kind == :embedding + + def label = I18n.t("label", scope: i18n_scope) + + def caption = I18n.t("caption", scope: i18n_scope, default: nil) + end + + # The registry of LLM-consuming features. + # + # Lives in lib_static because it is populated from initializers, which run + # before eager loading; constants defined under app/ would be unloaded on a + # development reload and lose their registrations. This is the same reason + # OpenProject::FeatureDecisions lives here. + # + # Register from config/initializers/llm_features.rb for core features, or + # from a module's engine: + # + # initializer "openproject_foo.llm_features" do + # OpenProject::Llm::Features.register :foo, kind: :chat + # end + module Features + module_function + + KINDS = %i[chat embedding].freeze + + # Mirrors Llm::Capabilities, which owns the vocabulary and knows how to + # read published values from the model registry. Duplicated as literals + # here because lib_static is autoloaded once, before app/ is available. + CAPABILITIES = { + chat: %i[function_calling structured_output vision reasoning].freeze, + embedding: %i[embeddings].freeze + }.freeze + + def register(key, + kind:, + requires: [], + prefers: [], + overridable: false, + pinned: false, + available: -> { true }, + i18n_scope: nil) + key = key.to_sym + validate!(key, kind, requires + prefers) + + all[key] = Feature.new(key:, kind:, requires: requires.map(&:to_sym).freeze, + prefers: prefers.map(&:to_sym).freeze, + overridable:, pinned:, available:, + i18n_scope: i18n_scope || "llm.features.#{key}") + end + + def all = @all ||= {} + + def [](key) + all.fetch(key.to_sym) { raise UnknownFeature, key.to_s } + end + + def registered?(key) = all.key?(key.to_sym) + + # Features whose own toggle is on. A feature that is switched off keeps its + # stored binding: flipping a flag must not lose an administrator's choice. + def available = all.values.select(&:available?) + + def for_kind(kind) = available.select { |feature| feature.kind == kind } + + def validate!(key, kind, capabilities) + raise ArgumentError, "unknown kind #{kind.inspect}" unless KINDS.include?(kind) + raise ArgumentError, "LLM feature #{key} is already registered" if all.key?(key) + + unknown = capabilities.map(&:to_sym) - CAPABILITIES.fetch(kind) + return if unknown.empty? + + raise ArgumentError, "#{unknown.join(', ')} not valid for a #{kind} feature" + end + end + end +end diff --git a/spec/features/admin/llm_connection_spec.rb b/spec/features/admin/llm_connection_spec.rb index 44691d061c5b..a1203322b941 100644 --- a/spec/features/admin/llm_connection_spec.rb +++ b/spec/features/admin/llm_connection_spec.rb @@ -123,4 +123,18 @@ def choose_action(item) expect(connection.models.count).to eq(2) end end + + describe "the AI models page" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + before { mock_llm_embeddings_response(base_url) } + + it "offers the vector settings only for features that embed" do + visit llm_feature_bindings_path + + expect(page).to have_test_selector("llm-feature-binding--dimensions-semantic_search") + expect(page).to have_no_test_selector("llm-feature-binding--dimensions-description_assistant") + expect(page).to be_axe_clean.within("#content") + end + end end diff --git a/spec/models/llm_model_deactivation_spec.rb b/spec/models/llm_model_deactivation_spec.rb index e75e259c6901..be415f977b5c 100644 --- a/spec/models/llm_model_deactivation_spec.rb +++ b/spec/models/llm_model_deactivation_spec.rb @@ -43,7 +43,24 @@ expect(connection.selectable_model_ids).to include("bge-m3") end + # The decision that makes the toggle safe: curation, not enforcement. A row an # administrator switches off must never silently break a running feature. + it "keeps a feature already bound to it working" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + expect(connection.available_model_ids).to include("qwen3.6-27b") + expect(Llm::Runtime.for(:description_assistant)).to be_ready + end + + it "still offers it to the feature that is bound to it" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + options = LlmConnections::SelectableModelsQuery + .new(connection, OpenProject::Llm::Features[:description_assistant]) + .call + + expect(options.map(&:model_id)).to include("qwen3.6-27b") + end # The reason deactivated_at exists rather than reusing active: the sync writes # active on every refresh, so an administrator's choice stored there would be diff --git a/spec/requests/admin/llm_connections_spec.rb b/spec/requests/admin/llm_connections_spec.rb index bcbfc3783792..cbf2b523eac5 100644 --- a/spec/requests/admin/llm_connections_spec.rb +++ b/spec/requests/admin/llm_connections_spec.rb @@ -240,15 +240,21 @@ before { login_as admin } it "offers the confirmation, naming what is kept" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + get disconnect_dialog_llm_connection_path, headers: { "Accept" => "text/vnd.turbo-stream.html" } expect(response).to have_http_status(:ok) expect(response.body).to include("Disconnect from the LLM server?") + expect(response.body).to include("Description assistant") end # Disconnecting is reversible on purpose: destroying the connection would + # cascade to the models, the verdicts and every binding. it "clears the credential and switches the connection off, keeping everything else" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + post disconnect_llm_connection_path connection.reload @@ -256,6 +262,7 @@ expect(connection).not_to be_enabled expect(connection.base_url).to eq("https://example.com/v1") expect(connection.models.count).to eq(2) + expect(connection.feature_bindings.first.model_id).to eq("qwen3.6-27b") end it "refuses when the connection comes from the environment" do @@ -276,6 +283,29 @@ end end + describe "the default embedding model field" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url: "https://example.com/v1") } + + before { login_as admin } + + # Column, contract attribute, validation, error key, locale key and permitted + # param all existed; the input was never rendered, so the value could not be + # set through the UI at all. + it "is rendered once something embeds" do + get llm_connection_path + + expect(response.body).to include("llm_connection[default_embedding_model_id]") + end + + it "is saved" do + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "bge-m3" } } + + expect(connection.reload.default_embedding_model_id).to eq("bge-m3") + end + end + describe "paginating the model list" do let!(:connection) { create(:llm_connection, :enabled, base_url: "https://example.com/v1") } @@ -344,4 +374,89 @@ def rendered_rows(body) = body.scan("llm-model--toggle-").size expect(rendered_rows(response.body)).to eq(3) end end + + describe "choosing a default embedding model" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url: "https://example.com/v1") } + + before { login_as admin } + + def verdict(model_id, state) + connection.capability_verdicts.create!(model_id:, capability: "embeddings", state:, + source: "probe", checked_at: Time.current) + end + + it "does not offer a model the server says cannot embed" do + verdict("qwen3.6-27b", "unsupported") + verdict("bge-m3", "supported") + + get llm_connection_path + + expect(response.body).to include("bge-m3") + expect(response.body).not_to include("cannot create embeddings") + end + + # An unconfirmed capability is not a capability: offering such a model + # invites a choice that fails much later, at index time. + it "does not offer a model whose capability is merely unconfirmed" do + get llm_connection_path + + expect(response.body).not_to include("not verified as an embedding model") + end + + it "says how to make a model eligible when none is" do + get llm_connection_path + + expect(response.body).to include("set its embeddings capability") + end + + it "offers one an administrator has asserted can embed" do + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "supported", source: "admin", checked_at: Time.current) + + get llm_connection_path + + expect(response.body).to include("bge-m3") + expect(response.body).not_to include("set its embeddings capability") + end + + # Otherwise a save would silently blank a working configuration. + it "keeps the chosen model listed even once it is ruled out" do + connection.update_column(:default_embedding_model_id, "qwen3.6-27b") + verdict("qwen3.6-27b", "unsupported") + + get llm_connection_path + + expect(response.body).to include("qwen3.6-27b") + end + + it "refuses a model the server says cannot embed" do + verdict("qwen3.6-27b", "unsupported") + + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "qwen3.6-27b" } } + + expect(connection.reload.default_embedding_model_id).to be_nil + end + + it "accepts one that can" do + verdict("bge-m3", "supported") + + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "bge-m3" } } + + expect(connection.reload.default_embedding_model_id).to eq("bge-m3") + end + + # Unknown is the normal state for a server that publishes nothing, so it + # must not block the choice. + it "allows one with no verdict at all" do + patch llm_connection_path, + params: { llm_connection: { base_url: "https://example.com/v1", + default_embedding_model_id: "bge-m3" } } + + expect(connection.reload.default_embedding_model_id).to eq("bge-m3") + end + end end diff --git a/spec/requests/admin/llm_feature_bindings_spec.rb b/spec/requests/admin/llm_feature_bindings_spec.rb new file mode 100644 index 000000000000..73139246a5c8 --- /dev/null +++ b/spec/requests/admin/llm_feature_bindings_spec.rb @@ -0,0 +1,186 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe "Admin AI model assignment", :llm_server_helpers, :skip_csrf, :webmock, + type: :rails_request, with_flag: { llm_connection: true } do + let(:admin) { create(:admin) } + let(:base_url) { "https://example.com/v1" } + + describe "GET /admin/llm_feature_bindings" do + before { login_as admin } + + it "prompts to configure a connection when there is none" do + get llm_feature_bindings_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("No LLM server configured") + end + + context "with a configured connection" do + let!(:connection) { create(:llm_connection, :with_models, base_url:) } + + it "lists every registered feature" do + get llm_feature_bindings_path + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Description assistant") + expect(response.body).to include("Semantic search") + end + + # Hiding an unusable model is the one thing that produces an unanswerable + # support question, so it stays listed and says why it cannot be chosen. + it "offers a model with no verdict, marked as unverified" do + get llm_feature_bindings_path + + expect(response.body).to include("qwen3.6-27b — not verified") + end + + it "disables a model known not to support a required capability" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "unsupported", source: "probe", checked_at: Time.current) + + get llm_feature_bindings_path + + expect(response.body).to include("qwen3.6-27b — no Embeddings support") + end + end + end + + describe "PATCH /admin/llm_feature_bindings/:id" do + let!(:connection) { create(:llm_connection, :with_models, base_url:) } + + before { login_as admin } + + it "stores the chosen model" do + patch llm_feature_binding_path("description_assistant"), + params: { llm_feature_binding: { model_id: "qwen3.6-27b" } } + + expect(response).to have_http_status(:see_other) + expect(connection.feature_bindings.find_by(feature_key: "description_assistant").model_id) + .to eq("qwen3.6-27b") + end + + it "treats a blank choice as inheriting the default" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + patch llm_feature_binding_path("description_assistant"), params: { llm_feature_binding: { model_id: "" } } + + expect(connection.feature_bindings.find_by(feature_key: "description_assistant").model_id).to be_nil + end + + # The verdict that matters is the one for the model just chosen, so it is + # fetched now rather than left unknown until the feature first runs. + it "probes the model when the feature requires a capability" do + request = stub_request(:post, "#{base_url}/embeddings") + .to_return(status: 200, + headers: { "Content-Type" => "application/json" }, + body: { data: [{ embedding: [0.1, 0.2] }] }.to_json) + + patch llm_feature_binding_path("semantic_search"), params: { llm_feature_binding: { model_id: "bge-m3" } } + + expect(request).to have_been_made.once + verdict = connection.capability_verdicts.find_by(model_id: "bge-m3", capability: "embeddings") + expect(verdict.state).to eq("supported") + expect(verdict.dimensions).to eq(2) + end + + it "does not probe for a feature that requires nothing" do + patch llm_feature_binding_path("description_assistant"), + params: { llm_feature_binding: { model_id: "qwen3.6-27b" } } + + expect(a_request(:post, "#{base_url}/embeddings")).not_to have_been_made + end + + it "404s for a feature that is not registered" do + patch llm_feature_binding_path("no_such_feature"), params: { llm_feature_binding: { model_id: "x" } } + + expect(response).to have_http_status(:not_found) + end + end + + describe "embedding settings" do + let!(:connection) { create(:llm_connection, :with_models, :enabled, base_url:) } + + before do + login_as admin + # Binding an embedding feature probes the model for a vector. + mock_llm_embeddings_response(base_url) + end + + it "stores the vector settings, keeping the prefixes exactly as typed" do + patch llm_feature_binding_path(:semantic_search), + params: { llm_feature_binding: { model_id: "bge-m3", + dimensions: "1024", + input_prefix: "passage: ", + query_prefix: "query: " } } + + binding = connection.feature_bindings.find_by(feature_key: "semantic_search") + + expect(binding.dimensions).to eq(1024) + # The trailing space is load-bearing for the E5 and BGE families. + expect(binding.input_prefix).to eq("passage: ") + expect(binding.query_prefix).to eq("query: ") + end + + it "rejects a dimension count that is not a positive integer" do + patch llm_feature_binding_path(:semantic_search), + params: { llm_feature_binding: { model_id: "bge-m3", dimensions: "0" } } + + expect(connection.feature_bindings.find_by(feature_key: "semantic_search")&.dimensions).to be_nil + end + + it "ignores vector settings sent to a chat feature" do + patch llm_feature_binding_path(:description_assistant), + params: { llm_feature_binding: { model_id: "qwen3.6-27b", dimensions: "1024" } } + + binding = connection.feature_bindings.find_by(feature_key: "description_assistant") + + expect(binding.model_id).to eq("qwen3.6-27b") + expect(binding.dimensions).to be_nil + end + + # A locked binding is the record that a vector index exists. Everything the + # index depends on is frozen, not just the model. + it "refuses to change anything a locked index depends on" do + binding = connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "bge-m3", + dimensions: 1024, input_prefix: "passage: ", + locked_at: Time.current) + + patch llm_feature_binding_path(:semantic_search), + params: { llm_feature_binding: { model_id: "bge-m3", dimensions: "512", input_prefix: "other: " } } + + binding.reload + expect(binding.dimensions).to eq(1024) + expect(binding.input_prefix).to eq("passage: ") + end + end +end diff --git a/spec/requests/admin/llm_models_spec.rb b/spec/requests/admin/llm_models_spec.rb index 85ac7966f933..c329e4cb8a27 100644 --- a/spec/requests/admin/llm_models_spec.rb +++ b/spec/requests/admin/llm_models_spec.rb @@ -72,6 +72,16 @@ expect(connection.models.where(external_id: "already-there").count).to eq(1) end + + it "makes the model bindable straight away" do + post llm_models_path, params: { llm_model: { external_id: "qwen3.6-35b-a3b" } } + + patch llm_feature_binding_path("description_assistant"), + params: { llm_feature_binding: { model_id: "qwen3.6-35b-a3b" } } + + expect(connection.feature_bindings.find_by(feature_key: "description_assistant").model_id) + .to eq("qwen3.6-35b-a3b") + end end describe "a refresh that cannot see the manual model" do @@ -129,6 +139,15 @@ expect(llm_model.context_window_source).to eq(:server) end + it "makes an asserted capability satisfy a feature that requires it" do + patch llm_model_path(llm_model), params: { llm_model: { capability_embeddings: "supported" } } + + patch llm_feature_binding_path("semantic_search"), + params: { llm_feature_binding: { model_id: "hand-typed" } } + + expect(connection.feature_bindings.find_by(feature_key: "semantic_search").model_id).to eq("hand-typed") + end + # Clearing an assertion records nothing rather than recording ignorance as # fact, so detection can still fill it in later. it "clears an assertion when set back to unspecified" do @@ -188,6 +207,21 @@ end end + describe "GET /admin/llm_models/:id/delete_dialog" do + it "offers a confirmation naming the features that would break" do + llm_model = create(:llm_model, :manual, llm_connection: connection, external_id: "hand-typed") + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "hand-typed") + + # Requested by the async-dialog Stimulus controller, which asks for a + # turbo stream rather than HTML. + get delete_dialog_llm_model_path(llm_model), + headers: { "Accept" => "text/vnd.turbo-stream.html" } + + expect(response).to have_http_status(:ok) + expect(response.body).to include("Description assistant") + end + end + describe "renaming to a taken id" do it "re-renders the form with the error instead of failing" do create(:llm_model, llm_connection: connection, external_id: "taken") @@ -221,6 +255,50 @@ end end + describe "POST /admin/llm_models/:id/toggle" do + let!(:llm_model) { create(:llm_model, llm_connection: connection, external_id: "qwen3.6-27b") } + + it "hides the model from the pickers and puts it back" do + post toggle_llm_model_path(llm_model) + + expect(response).to have_http_status(:ok) + expect(llm_model.reload).to be_deactivated + expect(connection.selectable_model_ids).not_to include("qwen3.6-27b") + + post toggle_llm_model_path(llm_model) + + expect(llm_model.reload).not_to be_deactivated + expect(connection.selectable_model_ids).to include("qwen3.6-27b") + end + + # Curation, not enforcement: a feature already pointing at the model keeps + # resolving, so switching a row off cannot silently break anything. + it "leaves an existing binding working" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") + + post toggle_llm_model_path(llm_model) + + expect(connection.available_model_ids).to include("qwen3.6-27b") + end + + it "refuses a model the server has withdrawn" do + withdrawn = create(:llm_model, :withdrawn, llm_connection: connection, external_id: "gone") + + post toggle_llm_model_path(withdrawn) + + expect(response).to have_http_status(:unprocessable_entity) + expect(withdrawn.reload).not_to be_deactivated + end + + it "is refused to a non-admin" do + login_as create(:user) + + post toggle_llm_model_path(llm_model) + + expect(llm_model.reload).not_to be_deactivated + end + end + describe "renaming a manually added model" do let!(:llm_model) do create(:llm_model, :manual, llm_connection: connection, external_id: "qwen/qwen3.6-35b-a3b") @@ -228,6 +306,8 @@ before do connection.update!(default_chat_model_id: "qwen/qwen3.6-35b-a3b") + connection.feature_bindings.create!(feature_key: "description_assistant", + model_id: "qwen/qwen3.6-35b-a3b") connection.capability_verdicts.create!(model_id: "qwen/qwen3.6-35b-a3b", capability: "embeddings", state: "unsupported", source: "probe", checked_at: Time.current) end @@ -255,9 +335,27 @@ expect(llm_model.reload.external_id).to eq("qwen/qwen3.6-35b-a3b:bf16") expect(connection.reload.default_chat_model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + expect(connection.feature_bindings.first.model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") expect(connection.capability_verdicts.first.model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") end + it "keeps the feature resolving afterwards", with_flag: { llm_connection: true } do + connection.update!(enabled: true) + + patch llm_model_path(llm_model), params: { llm_model: { external_id: "qwen/qwen3.6-35b-a3b:bf16" } } + + expect(Llm::Runtime.for(:description_assistant).model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + end + + it "follows a model a locked binding depends on" do + binding = connection.feature_bindings.first + binding.update!(locked_at: Time.current) + + patch llm_model_path(llm_model), params: { llm_model: { external_id: "qwen/qwen3.6-35b-a3b:bf16" } } + + expect(binding.reload.model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") + end + # The server names its own models; renaming one here would only be undone by # the next refresh. it "refuses to rename a discovered model" do @@ -302,38 +400,4 @@ expect(response.body).not_to include("Supported (set by an administrator)") end end - - describe "POST /admin/llm_models/:id/toggle" do - let!(:llm_model) { create(:llm_model, llm_connection: connection, external_id: "qwen3.6-27b") } - - it "hides the model from the pickers and puts it back" do - post toggle_llm_model_path(llm_model) - - expect(response).to have_http_status(:ok) - expect(llm_model.reload).to be_deactivated - expect(connection.selectable_model_ids).not_to include("qwen3.6-27b") - - post toggle_llm_model_path(llm_model) - - expect(llm_model.reload).not_to be_deactivated - expect(connection.selectable_model_ids).to include("qwen3.6-27b") - end - - it "refuses a model the server has withdrawn" do - withdrawn = create(:llm_model, :withdrawn, llm_connection: connection, external_id: "gone") - - post toggle_llm_model_path(withdrawn) - - expect(response).to have_http_status(:unprocessable_entity) - expect(withdrawn.reload).not_to be_deactivated - end - - it "is refused to a non-admin" do - login_as create(:user) - - post toggle_llm_model_path(llm_model) - - expect(llm_model.reload).not_to be_deactivated - end - end end diff --git a/spec/services/llm/runtime_spec.rb b/spec/services/llm/runtime_spec.rb new file mode 100644 index 000000000000..d462972431e7 --- /dev/null +++ b/spec/services/llm/runtime_spec.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" + +RSpec.describe Llm::Runtime, with_flag: { llm_connection: true } do + subject(:resolution) { described_class.for(feature_key, override:) } + + let(:feature_key) { :description_assistant } + let(:override) { nil } + + context "without a connection" do + it { expect(resolution.status).to eq(:no_connection) } + end + + context "with a connection that is not enabled" do + before { create(:llm_connection, :with_models, enabled: false) } + + it { expect(resolution.status).to eq(:no_connection) } + end + + context "with an enabled connection" do + let!(:connection) { create(:llm_connection, :with_models, :enabled) } + + it "is unbound until a model is chosen" do + expect(resolution.status).to eq(:unbound) + expect(resolution.model_id).to be_nil + end + + it "falls back to the connection default" do + connection.update!(default_chat_model_id: "qwen3.6-27b") + + expect(resolution).to be_ready + expect(resolution.model_id).to eq("qwen3.6-27b") + end + + it "prefers the feature binding over the connection default" do + connection.update!(default_chat_model_id: "qwen3.6-27b") + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "bge-m3") + + expect(resolution.model_id).to eq("bge-m3") + end + + context "with a per-item override" do + let(:override) { "qwen3.6-27b" } + + it "wins over the binding" do + connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "bge-m3") + + expect(resolution.model_id).to eq("qwen3.6-27b") + end + + # semantic_search's vectors were written with the bound model; a different + # one at query time is silently wrong answers, not a preference. + it "is ignored by a feature that is not overridable" do + connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "bge-m3") + connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", + state: "supported", source: "admin", checked_at: Time.current) + + resolution = described_class.for(:semantic_search, override: "qwen3.6-27b") + + expect(resolution.model_id).to eq("bge-m3") + end + end + + # Substituting the default here would silently change the output of a + # transform an administrator configured deliberately. + context "when the chosen model is gone from the catalogue" do + let(:override) { "vanished-model" } + + before { connection.update!(default_chat_model_id: "qwen3.6-27b") } + + it "fails closed rather than falling back" do + expect(resolution.status).to eq(:model_missing) + expect(resolution.model_id).to eq("vanished-model") + end + end + end + + describe "capability gating" do + let(:feature_key) { :semantic_search } + let!(:connection) { create(:llm_connection, :with_models, :enabled) } + + before { connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "qwen3.6-27b") } + + it "blocks on a definite unsupported verdict" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "unsupported", source: "probe", checked_at: Time.current) + + expect(resolution.status).to eq(:incapable) + expect(resolution.missing_capabilities).to eq([:embeddings]) + end + + # Refusing on "we could not tell" would make most self-hosted servers + # unusable, since the model list carries no capability information at all. + it "does not block when the verdict is unknown" do + connection.capability_verdicts.create!(model_id: "qwen3.6-27b", capability: "embeddings", + state: "unknown", source: "probe", checked_at: Time.current) + + expect(resolution).to be_ready + end + + it "does not block when there is no verdict at all" do + expect(resolution).to be_ready + end + end + + describe "running a request", :llm_server_helpers, :webmock do + let!(:connection) { create(:llm_connection, :with_models, :enabled, default_chat_model_id: "qwen3.6-27b") } + + it "sends a completion for the resolved model" do + mock_llm_chat_response("https://example.com/v1", content: "pong") + + expect(resolution.chat(max_retries: 0).ask("ping").content).to eq("pong") + expect(WebMock).to have_requested(:post, "https://example.com/v1/chat/completions") + .with(body: hash_including("model" => "qwen3.6-27b")) + end + + it "refuses when the feature is not ready" do + connection.update!(enabled: false) + + expect { resolution.chat }.to raise_error(Llm::Errors::NotReady) { |e| expect(e.status).to eq(:no_connection) } + end + + # Features are resolved by kind, so asking a chat feature to embed means a + # caller has confused two features. + it "refuses to embed through a chat feature" do + expect { resolution.embed("hello") } + .to raise_error(Llm::Errors::NotReady) { |e| expect(e.status).to eq(:wrong_kind) } + end + + context "with an embedding feature" do + let(:feature_key) { :semantic_search } + + before { connection.update!(default_embedding_model_id: "bge-m3") } + + it "requests a vector for the resolved model" do + mock_llm_embeddings_response("https://example.com/v1", dimensions: 8) + + expect(resolution.embed("hello", max_retries: 0).vectors.length).to eq(8) + end + + it "refuses to chat through an embedding feature" do + expect { resolution.chat } + .to raise_error(Llm::Errors::NotReady) { |e| expect(e.status).to eq(:wrong_kind) } + end + end + end +end