Image Settings rework (#1049)

- merge image settings into one place
- merge updates and notifications
- better control of image model config
- change default image dims
- open currently used image settings
- pass through chat image settings correctly
- refactor char form into smaller components
- add model override to generate avatar
- allow ltm to be disabled
- Make ltm off by default

common/horde-gen.ts

@@ -104,8 +104,8 @@ export async function generateImage(
   const payload = {
     prompt: `${prompt.slice(0, 500)} ### ${negative}`,
     params: {
-      height: base?.height ?? 384,
-      width: base?.width ?? 384,
+      height: base?.height ?? 1024,
+      width: base?.width ?? 1024,
       cfg_scale: base?.cfg ?? 9,
       seed: Math.trunc(Math.random() * 1_000_000_000).toString(),
       karras: false,

common/types/admin.ts

@@ -43,8 +43,10 @@ export interface AppConfig {
 }
 
 export type ImageModel = {
+  id: string
   name: string
   desc: string
+  override: string
   init: { clipSkip?: number; steps: number; cfg: number; height: number; width: number }
   limit: { clipSkip?: number; steps: number; cfg: number; height: number; width: number }
 }

common/types/library.ts

@@ -1,6 +1,6 @@
 import { PersonaFormat } from '../adapters'
 import { JsonField } from '../prompt'
-import { BaseImageSettings, ImageSettings } from './image-schema'
+import { ImageSettings } from './image-schema'
 import { MemoryBook } from './memory'
 import { FullSprite } from './sprite'
 import { VoiceSettings } from './texttospeech-schema'
@@ -44,8 +44,6 @@ export interface Character extends BaseCharacter {
   voice?: VoiceSettings
   voiceDisabled?: boolean
 
-  image?: ImageSettings
-
   json?: ResponseSchema
 
   folder?: string
@@ -58,7 +56,7 @@ export interface Character extends BaseCharacter {
   insert?: { depth: number; prompt: string }
   creator?: string
   characterVersion?: string
-  imageSettings?: BaseImageSettings
+  imageSettings?: ImageSettings
 }
 
 export interface LibraryCharacter extends Omit<Character, 'kind' | 'tags'> {

common/types/presets.ts

@@ -1,6 +1,6 @@
 import { AIAdapter, OpenRouterModel, ThirdPartyFormat } from '../adapters'
 import { ModelFormat } from '../presets/templates'
-import { BaseImageSettings } from './image-schema'
+import { BaseImageSettings, ImageSettings } from './image-schema'
 import { ResponseSchema } from './library'
 
 export interface SubscriptionTier {
@@ -197,3 +197,11 @@ export interface PromptTemplate {
   createdAt: string
   updatedAt: string
 }
+
+export interface ImagePreset extends ImageSettings {
+  kind: 'image-preset'
+  _id: string
+  userId: string
+  name: string
+  description: string
+}

common/types/schema.ts

@@ -1,7 +1,7 @@
 import type { AIAdapter, ChatAdapter, ThirdPartyFormat } from '../adapters'
 import * as Memory from './memory'
 import type { GenerationPreset } from '../presets'
-import type { BaseImageSettings, ImageSettings } from './image-schema'
+import type { ImageSettings } from './image-schema'
 import type { TTSSettings } from './texttospeech-schema'
 import type { UISettings } from './ui'
 import * as Saga from './saga'
@@ -92,6 +92,8 @@ export namespace AppSchema {
     admin: boolean
     role?: 'moderator' | 'admin'
 
+    disableLTM?: boolean
+
     novelApiKey: string
     novelModel: string
     novelVerified?: boolean
@@ -261,7 +263,7 @@ export namespace AppSchema {
     treeLeafId?: string
 
     imageSource?: 'last-character' | 'main-character' | 'chat' | 'settings'
-    imageSettings?: BaseImageSettings
+    imageSettings?: ImageSettings
   }
 
   export interface ChatMember {

srv/api/character.ts

@@ -323,7 +323,8 @@ const editPartCharacter = handle(async ({ body, params, userId }) => {
 
   if (body.imageSettings) {
     try {
-      update.imageSettings = JSON.parse(body.imageSettings)
+      update.imageSettings =
+        typeof body.imageSettings === 'string' ? JSON.parse(body.imageSettings) : body.imageSettings
     } catch (ex: any) {
       throw new StatusError(`Character 'imageSettings' could not be parsed: ${ex.message}`, 400)
     }
@@ -480,12 +481,14 @@ export const createImage = handle(async ({ body, userId, socketId, log }) => {
       chatId: 'string?',
       requestId: 'string?',
       parent: 'string?',
+      model: 'string?',
     },
     body
   )
   const user = userId ? await store.users.getUser(userId) : body.user
 
   const guestId = userId ? undefined : socketId
+  const requestId = body.requestId || v4()
   generateImage(
     {
       user,
@@ -495,13 +498,14 @@ export const createImage = handle(async ({ body, userId, socketId, log }) => {
       noAffix: body.noAffix,
       chatId: body.chatId,
       characterId: body.characterId,
-      requestId: body.requestId,
+      requestId,
       parentId: body.parent,
+      model: body.model,
     },
     log,
     guestId
   )
-  return { success: true }
+  return { success: true, requestId }
 })
 
 router.post('/image', createImage)

srv/api/user/settings.ts

@@ -179,6 +179,7 @@ const validConfig = {
   defaultPreset: 'string?',
   chargenPreset: 'string?',
   adapterConfig: 'any?',
+  disableLTM: 'boolean?',
 } as const
 
 /**
@@ -198,6 +199,8 @@ export const updatePartialConfig = handle(async ({ userId, body }) => {
       announcement: 'string?',
       defaultPreset: 'string?',
       chargenPreset: 'string?',
+      images: 'any?',
+      disableLTM: 'boolean?',
     },
     body
   )
@@ -212,6 +215,10 @@ export const updatePartialConfig = handle(async ({ userId, body }) => {
     update.defaultPreset = body.defaultPreset
   }
 
+  if (body.disableLTM !== undefined) {
+    update.disableLTM = body.disableLTM
+  }
+
   if (body.chargenPreset) {
     const preset = await store.presets.getUserPreset(body.chargenPreset)
     if (!preset || preset.userId !== userId) {
@@ -256,6 +263,10 @@ export const updatePartialConfig = handle(async ({ userId, body }) => {
     update.thirdPartyPassword = encryptText(body.thirdPartyPassword)
   }
 
+  if (body.images) {
+    update.images = body.images
+  }
+
   await store.users.updateUser(userId, update)
   const next = await getSafeUserConfig(userId)
   return next
@@ -276,6 +287,10 @@ export const updateConfig = handle(async ({ userId, body }) => {
     useLocalPipeline: body.useLocalPipeline,
   }
 
+  if (body.disableLTM !== undefined) {
+    update.disableLTM = body.disableLTM
+  }
+
   if (body.hordeKey || body.hordeApiKey) {
     const prevKey = prevUser.hordeKey
     const incomingKey = body.hordeKey || body.hordeApiKey!

srv/db/user.ts

@@ -102,6 +102,7 @@ export async function createUser(newUser: NewUser, admin?: boolean) {
   const user: AppSchema.User = {
     _id: v4(),
     kind: 'user',
+    disableLTM: true,
     username,
     hash,
     admin: !!admin,

srv/image/index.ts

@@ -108,7 +108,7 @@ export async function generateImage(
       case 'sd':
       case 'agnai':
         image = await handleSDImage(
-          { user, prompt, negative, settings: imageSettings },
+          { user, prompt, negative, settings: imageSettings, override: opts.model },
           log,
           guestId
         )

srv/image/stable-diffusion.ts

@@ -35,11 +35,12 @@ export type SDRequest = {
   hr_scale?: number
   hr_upscaler?: string
   hr_second_pass_steps?: number
+  model_override?: string
 }
 
 export const handleSDImage: ImageAdapter = async (opts, log, guestId) => {
   const config = await getConfig(opts)
-  const payload = getPayload(config.kind, opts, config.model)
+  const payload = getPayload(config.kind, opts, config.model, config.temp)
 
   logger.debug(payload, 'Image: Stable Diffusion payload')
 
@@ -80,14 +81,16 @@ export const handleSDImage: ImageAdapter = async (opts, log, guestId) => {
   return { ext: 'png', content: buffer }
 }
 
-async function getConfig({ user, settings }: ImageRequestOpts): Promise<{
+async function getConfig({ user, settings, override }: ImageRequestOpts): Promise<{
   kind: 'user' | 'agnai'
   host: string
   params?: string
   model?: AppSchema.ImageModel
+  temp?: AppSchema.ImageModel
 }> {
   const type = settings?.type || user.images?.type
 
+  // Stable Diffusion URL always comes from user settings
   const userHost = user.images?.sd.url || defaultSettings.url
   if (type !== 'agnai') {
     return { kind: 'user', host: userHost }
@@ -102,12 +105,16 @@ async function getConfig({ user, settings }: ImageRequestOpts): Promise<{
   if (!sub?.tier?.imagesAccess && !user.admin) return { kind: 'user', host: userHost }
 
   const models = getAgnaiModels(srv.imagesModels)
-  const model =
-    models.length === 1
-      ? models[0]
-      : models.find((m) => m.name === user.images?.agnai?.model) ?? models[0]
 
-  if (!model) {
+  const temp = override ? models.find((m) => m.id === override || m.name === override) : undefined
+
+  const match = models.find((m) => {
+    return m.id === settings?.agnai?.model || m.name === settings?.agnai?.model
+  })
+
+  const model = models.length === 1 ? models[0] : match ?? models[0]
+
+  if (!temp && !model) {
     return { kind: 'user', host: userHost }
   }
 
@@ -116,35 +123,41 @@ async function getConfig({ user, settings }: ImageRequestOpts): Promise<{
     `key=${config.auth.inferenceKey}`,
     `id=${user._id}`,
     `level=${user.admin ? 99999 : sub?.level ?? -1}`,
-    `model=${model.name}`,
-  ].join('&')
+    `model=${temp?.name || model.name}`,
+  ]
 
-  return { kind: 'agnai', host: srv.imagesHost, params: `?${params}`, model }
+  return { kind: 'agnai', host: srv.imagesHost, params: `?${params.join('&')}`, model, temp }
 }
 
-function getPayload(kind: 'agnai' | 'user', opts: ImageRequestOpts, model?: AppSchema.ImageModel) {
+function getPayload(
+  kind: 'agnai' | 'user',
+  opts: ImageRequestOpts,
+  model: AppSchema.ImageModel | undefined,
+  temp: AppSchema.ImageModel | undefined
+) {
   const sampler =
-    (kind === 'agnai' ? opts.user.images?.agnai?.sampler : opts.user.images?.sd?.sampler) ||
+    (kind === 'agnai' ? opts.settings?.agnai?.sampler : opts.settings?.sd?.sampler) ||
     defaultSettings.sampler
   const payload: SDRequest = {
     prompt: opts.prompt,
     // enable_hr: true,
     // hr_scale: 1.5,
     // hr_second_pass_steps: 15,
     // hr_upscaler: "",
-    clip_skip: opts.user.images?.clipSkip ?? model?.init.clipSkip ?? 0,
-    height: opts.user.images?.height ?? model?.init.height ?? 384,
-    width: opts.user?.images?.width ?? model?.init.width ?? 384,
+    clip_skip: opts.settings?.clipSkip ?? model?.init.clipSkip ?? 0,
+    height: opts.settings?.height ?? model?.init.height ?? 1024,
+    width: opts.settings?.width ?? model?.init.width ?? 1024,
     n_iter: 1,
     batch_size: 1,
     negative_prompt: opts.negative,
     sampler_name: (SD_SAMPLER_REV as any)[sampler],
-    cfg_scale: opts.user.images?.cfg ?? model?.init.cfg ?? 9,
+    cfg_scale: opts.settings?.cfg ?? model?.init.cfg ?? 9,
     seed: Math.trunc(Math.random() * 1_000_000_000),
-    steps: opts.user.images?.steps ?? model?.init.steps ?? 28,
+    steps: opts.settings?.steps ?? model?.init.steps ?? 28,
     restore_faces: false,
     save_images: false,
     send_images: true,
+    model_override: temp ? temp.override : model?.override,
   }
 
   if (model) {

srv/image/types.ts

@@ -1,11 +1,12 @@
 import { AppSchema } from '../../common/types/schema'
 import { AppLog } from '../middleware'
-import { BaseImageSettings } from '/common/types/image-schema'
+import { ImageSettings } from '/common/types/image-schema'
 
 export type ImageGenerateRequest = {
   user: AppSchema.User
   prompt: string
   chatId?: string
+  model?: string
   messageId?: string
   ephemeral?: boolean
   append?: boolean
@@ -20,7 +21,8 @@ export type ImageRequestOpts = {
   user: AppSchema.User
   prompt: string
   negative: string
-  settings: BaseImageSettings | undefined
+  settings: ImageSettings | undefined
+  override?: string
 }
 
 export type ImageAdapter = (

web/App.tsx

@@ -42,6 +42,7 @@ import SoundsPage from './pages/Sounds'
 import PatreonOauth from './pages/Settings/PatreonOauth'
 import { SagaDetail } from './pages/Saga/Detail'
 import { SagaList } from './pages/Saga/List'
+import { ImageSettingsModal } from './pages/Settings/Image/ImageSettings'
 
 const App: Component = () => {
   const state = userStore()
@@ -212,6 +213,9 @@ const Layout: Component<{ children?: any }> = (props) => {
       <ProfileModal />
       <For each={rootModals.modals}>{(modal) => modal.element}</For>
       <ImageModal />
+      <Show when={cfg.showImgSettings}>
+        <ImageSettingsModal />
+      </Show>
       <SettingsModal />
       <div
         class="absolute bottom-0 left-0 right-0 top-0 z-10 h-[100vh] w-full bg-black bg-opacity-20 sm:hidden"

web/Layout.tsx

@@ -1,14 +1,35 @@
-import { Component } from 'solid-js'
+import { Component, createMemo } from 'solid-js'
 import { Portal } from 'solid-js/web'
 
 export const HeaderPortal: Component<{ children: any }> = (props) => {
   return <Portal mount={document.getElementById('site-header')!}>{props.children}</Portal>
 }
 
 export const Page: Component<{ children: any; class?: string; classList?: any }> = (props) => {
+  const hasHorzPaddingClass = createMemo(() => {
+    if (isHorzPadding(props.class)) return true
+
+    for (const [cls, value] of Object.entries(props.classList || {})) {
+      if (!isHorzPadding(cls)) continue
+      if (value) return true
+    }
+
+    return false
+  })
+
   return (
-    <main class={`h-full w-full px-2 sm:px-3 ${props.class || ''}`} classList={props.classList}>
+    <main
+      class={`h-full w-full ${props.class || ''}`}
+      classList={{
+        'px-2 sm:px-3': !hasHorzPaddingClass(),
+        ...props.classList,
+      }}
+    >
       {props.children}
     </main>
   )
 }
+
+function isHorzPadding(cls?: string) {
+  return cls?.startsWith('p-') || cls?.startsWith('px-')
+}