ArliAI support (#1095)

* log image gen failures
* client-side horde image gen
* add arliai support

Author: sceuick

common/adapters.ts

@@ -99,6 +99,7 @@ export const THIRDPARTY_HANDLERS: { [svc in ThirdPartyFormat]: AIAdapter } = {
   vllm: 'kobold',
   featherless: 'kobold',
   gemini: 'kobold',
+  arli: 'kobold',
 }
 
 export const BASIC_PROMPT_ONLY: { [svc in ThirdPartyFormat]?: boolean } = {
@@ -122,6 +123,7 @@ export const THIRDPARTY_FORMATS = [
   'ollama',
   'vllm',
   'featherless',
+  'arli',
   'gemini',
 ] as const
 

common/horde-gen.ts

@@ -1,8 +1,14 @@
+import crypto from 'crypto'
 import { AppSchema } from './types/schema'
 import { defaultPresets } from './default-preset'
 import { SD_SAMPLER } from './image'
 import { toArray } from './util'
 import type { AppLog } from '../srv/middleware'
+import { v4 } from 'uuid'
+
+export const HORDE_SEED = v4()
+
+const ALGO = 'aes256'
 
 const HORDE_GUEST_KEY = '0000000000'
 const baseUrl = 'https://aihorde.net/api/v2'
@@ -107,7 +113,8 @@ export async function generateImage(
       height: base?.height ?? 1024,
       width: base?.width ?? 1024,
       cfg_scale: base?.cfg ?? 9,
-      seed: Math.trunc(Math.random() * 1_000_000_000).toString(),
+      clip_skip: base?.clipSkip,
+      denoising_strength: 1,
       karras: false,
       n: 1,
       post_processing: [],
@@ -125,10 +132,17 @@ export async function generateImage(
   log?.debug({ ...payload, prompt: null }, 'Horde payload')
   log?.debug(`Prompt:\n${payload.prompt}`)
 
+  let key = user.hordeKey
+  if (!key) {
+    key = HORDE_GUEST_KEY
+  } else {
+    key = decryptText(user.hordeKey)
+  }
+
   const image = await generate({
     type: 'image',
     payload,
-    key: user.hordeKey || HORDE_GUEST_KEY,
+    key,
     onTick,
   })
 
@@ -280,6 +294,16 @@ function wait(secs: number) {
   return new Promise((resolve) => setTimeout(resolve, secs * 1000))
 }
 
+function decryptText(text: string) {
+  try {
+    const decipher = crypto.createDecipher(ALGO, HORDE_SEED)
+    const decrypted = decipher.update(text, 'hex', 'utf8') + decipher.final('utf8')
+    return decrypted
+  } catch (ex) {
+    return text
+  }
+}
+
 export type FindUserResponse = {
   kudos_details: {
     accumulated: number

common/presets.ts

@@ -57,6 +57,7 @@ export const presetValidator = {
   oaiModel: 'string',
   openRouterModel: 'any?',
   featherlessModel: 'string?',
+  arliModel: 'string?',
   googleModel: 'string?',
 
   mirostatTau: 'number?',
@@ -73,6 +74,11 @@ export const presetValidator = {
   thirdPartyUrlNoSuffix: 'boolean?',
   thirdPartyModel: 'string?',
 
+  dryAllowedLength: 'number?',
+  dryBase: 'number?',
+  drySequenceBreakers: ['string?'],
+  dryMultiplier: 'number?',
+
   novelModel: 'string?',
   novelModelOverride: 'string?',
 

common/types/presets.ts

@@ -164,6 +164,7 @@ export interface GenSettings {
   openRouterModel?: OpenRouterModel
   googleModel?: string
   featherlessModel?: string
+  arliModel?: string
 
   thirdPartyUrl?: string
   thirdPartyFormat?: ThirdPartyFormat

common/types/schema.ts

@@ -130,6 +130,9 @@ export namespace AppSchema {
     featherlessApiKey?: string
     featherlessApiKeySet?: boolean
 
+    arliApiKey?: string
+    arliApiKeySet?: boolean
+
     defaultAdapter: AIAdapter
     defaultPresets?: { [key in AIAdapter]?: string }
     defaultPreset?: string

srv/adapter/agnaistic.ts

@@ -427,6 +427,7 @@ export function getHandlers(settings: Partial<AppSchema.GenSettings>) {
       return handlers.openai
 
     case 'featherless':
+    case 'arli':
       return handlers.kobold
 
     case 'gemini':

srv/adapter/arli.ts

@@ -0,0 +1,84 @@
+import { logger } from '../middleware'
+
+type V1Model = {
+  name: string
+  promptFormat: string
+  contextSize: string
+  modelLink: string
+  modelSize: string
+  status: boolean
+}
+
+export type ArliModel = {
+  id: string
+  name: string
+  model_class: string
+  status: 'active' | 'not_deployed' | 'pending_deploy'
+  health?: 'OFFLINE' | 'UNHEALTHY' | 'HEALTHY'
+
+  ctx: number
+  res: number
+}
+
+let modelCache: ArliModel[] = []
+let classCache: Record<string, { ctx: number; res: number }> = {}
+
+export function getArliModels() {
+  return { models: modelCache, classes: classCache }
+}
+
+async function getModelList() {
+  try {
+    const models = await fetch('https://api.arliai.com/model/all', {
+      headers: {
+        accept: '*/*',
+      },
+      method: 'GET',
+    })
+
+    const next: ArliModel[] = []
+
+    if (models.status && models.status > 200) {
+      const body = await models.json()
+      logger.warn({ body, status: models.status }, `ArliAI model list failed`)
+      return
+    }
+
+    const map = await models.json().then((res) => {
+      const list = res as V1Model[]
+      if (!list) return {}
+
+      const map: { [key: string]: V1Model } = {}
+      for (const model of list) {
+        if (!classCache[model.modelSize]) {
+          classCache[model.modelSize] = {
+            ctx: +model.contextSize,
+            res: 500,
+          }
+        }
+
+        next.push({
+          id: model.name,
+          model_class: model.modelSize,
+          name: model.name,
+          res: 500,
+          status: model.status ? 'active' : 'not_deployed',
+          ctx: +model.contextSize,
+        })
+
+        map[model.name] = model
+      }
+      return map
+    })
+
+    modelCache = next
+
+    return map
+  } catch (ex) {
+    logger.error({ err: ex }, `Featherless model list failed`)
+  }
+}
+
+getModelList()
+
+setInterval(getModelList, 120000)

srv/adapter/kobold.ts

@@ -47,7 +47,8 @@ export const handleThirdParty: ModelAdapter = async function* (opts) {
     opts.gen.thirdPartyFormat === 'llamacpp' ||
     opts.gen.thirdPartyFormat === 'exllamav2' ||
     opts.gen.thirdPartyFormat === 'koboldcpp' ||
-    opts.gen.thirdPartyFormat === 'featherless'
+    opts.gen.thirdPartyFormat === 'featherless' ||
+    opts.gen.thirdPartyFormat === 'arli'
       ? getThirdPartyPayload(opts)
       : { ...base, ...mappedSettings, prompt }
 
@@ -186,6 +187,13 @@ async function dispatch(opts: AdapterProps, body: any) {
         : fullCompletion(url, body, headers, opts.gen.thirdPartyFormat, opts.log)
     }
 
+    case 'arli': {
+      const url = 'https://api.arliai.com/v1/completions'
+      return opts.gen.streamResponse
+        ? streamCompletion(url, body, headers, opts.gen.thirdPartyFormat, opts.log)
+        : fullCompletion(url, body, headers, opts.gen.thirdPartyFormat, opts.log)
+    }
+
     default:
       const isStreamSupported = await checkStreamSupported(`${baseURL}/api/extra/version`)
       return opts.gen.streamResponse && isStreamSupported
@@ -253,6 +261,24 @@ async function getHeaders(opts: AdapterProps) {
       break
     }
 
+    case 'arli': {
+      if (!opts.gen.arliModel) {
+        throw new Error(`ArliAI model not set. Check your preset`)
+      }
+
+      const key = opts.gen.thirdPartyKey || opts.user.arliApiKey
+      if (!key) {
+        throw new Error(`ArliAI API key not set. Check your Settings->AI->Third-party settings`)
+      }
+
+      const apiKey = key ? (opts.guest ? key : decryptText(key)) : ''
+      if (apiKey) {
+        headers['Authorization'] = `Bearer ${apiKey}`
+      }
+      headers['Content-Type'] = 'application/json'
+      break
+    }
+
     case 'mistral': {
       const key = opts.user.mistralKey
       if (!key)

srv/adapter/payloads.ts

@@ -216,6 +216,55 @@ function getBasePayload(opts: AdapterProps, stops: string[] = []) {
     return payload
   }
 
+  if (format === 'arli') {
+    const body: any = {
+      model: gen.arliModel,
+      prompt,
+      stop: getStoppingStrings(opts, stops),
+      presence_penalty: gen.presencePenalty,
+      frequency_penalty: gen.frequencyPenalty,
+      length_penalty: gen.repetitionPenalty,
+      tfs: gen.tailFreeSampling,
+      temperature: gen.temp,
+      top_p: gen.topP,
+      top_k: gen.topK,
+      min_p: gen.minP,
+      typical_p: gen.typicalP,
+      ignore_eos: false,
+      max_tokens: gen.maxTokens,
+      smoothing_factor: gen.smoothingFactor,
+      smoothing_curve: gen.smoothingCurve,
+
+      stream: gen.streamResponse,
+    }
+
+    if (gen.dryMultiplier) {
+      body.dry_multiplier = gen.dryMultiplier
+      body.dry_base = gen.dryBase
+      body.dry_allowed_length = gen.dryAllowedLength
+      body.dry_range = gen.dryRange
+      body.dry_sequence_breakers = sequenceBreakers
+    }
+
+    if (gen.dynatemp_range) {
+      body.dynamic_temperature = true
+      body.dynatemp_min = (gen.temp ?? 1) - (gen.dynatemp_range ?? 0)
+      body.dynatemp_max = (gen.temp ?? 1) + (gen.dynatemp_range ?? 0)
+      body.dynatemp_exponent = gen.dynatemp_exponent
+    }
+
+    if (gen.xtcThreshold) {
+      body.xtc_threshold = gen.xtcThreshold
+      body.xtc_probability = gen.xtcProbability
+    }
+
+    if (body.top_k <= 0) {
+      body.top_k = -1
+    }
+
+    return body
+  }
+
   if (format === 'ollama') {
     const payload: any = {
       prompt,
@@ -426,12 +475,26 @@ function getBasePayload(opts: AdapterProps, stops: string[] = []) {
       epsilon_cutoff: gen.epsilonCutoff,
     }
 
+    if (gen.dryMultiplier) {
+      body.dry_multiplier = gen.dryMultiplier
+      body.dry_base = gen.dryBase
+      body.dry_allowed_length = gen.dryAllowedLength
+      body.dry_range = gen.dryRange
+      body.dry_sequence_breakers = sequenceBreakers
+    }
+
     if (gen.dynatemp_range) {
+      body.dynamic_temperature = true
       body.dynatemp_min = (gen.temp ?? 1) - (gen.dynatemp_range ?? 0)
       body.dynatemp_max = (gen.temp ?? 1) + (gen.dynatemp_range ?? 0)
       body.dynatemp_exponent = gen.dynatemp_exponent
     }
 
+    if (gen.xtcThreshold) {
+      body.xtc_threshold = gen.xtcThreshold
+      body.xtc_probability = gen.xtcProbability
+    }
+
     return body
   }
 

srv/api/settings.ts

@@ -13,6 +13,7 @@ import { getOpenRouterModels } from '../adapter/openrouter'
 import { updateRegisteredSubs } from '../adapter/agnaistic'
 import { getFeatherModels } from '../adapter/featherless'
 import { filterImageModels } from '/common/image-util'
+import { getArliModels } from '../adapter/arli'
 
 const router = Router()
 
@@ -35,6 +36,10 @@ router.get('/featherless', (_, res) => {
   const { models, classes } = getFeatherModels()
   res.json({ models, classes })
 })
+router.get('/arli', (_, res) => {
+  const { models, classes } = getArliModels()
+  res.json({ models, classes })
+})
 
 export default router