From a8b85cdb0df0e20c433d1fb6b0e52bb46d9693a6 Mon Sep 17 00:00:00 2001 From: Ali BARIN Date: Wed, 20 Dec 2023 16:15:30 +0000 Subject: [PATCH] feat(self-hosted-llm): add send prompt and send chat prompt actions --- .../src/apps/self-hosted-llm/actions/index.ts | 4 + .../actions/send-chat-prompt/index.ts | 137 ++++++++++++++++++ .../actions/send-prompt/index.ts | 104 +++++++++++++ .../apps/self-hosted-llm/assets/favicon.svg | 6 + .../src/apps/self-hosted-llm/auth/index.ts | 44 ++++++ .../self-hosted-llm/auth/is-still-verified.ts | 8 + .../auth/verify-credentials.ts | 7 + .../self-hosted-llm/common/add-auth-header.ts | 11 ++ .../self-hosted-llm/common/set-base-url.ts | 11 ++ .../self-hosted-llm/dynamic-data/index.ts | 3 + .../dynamic-data/list-models/index.ts | 19 +++ .../src/apps/self-hosted-llm/index.d.ts | 0 .../backend/src/apps/self-hosted-llm/index.ts | 21 +++ 13 files changed, 375 insertions(+) create mode 100644 packages/backend/src/apps/self-hosted-llm/actions/index.ts create mode 100644 packages/backend/src/apps/self-hosted-llm/actions/send-chat-prompt/index.ts create mode 100644 packages/backend/src/apps/self-hosted-llm/actions/send-prompt/index.ts create mode 100644 packages/backend/src/apps/self-hosted-llm/assets/favicon.svg create mode 100644 packages/backend/src/apps/self-hosted-llm/auth/index.ts create mode 100644 packages/backend/src/apps/self-hosted-llm/auth/is-still-verified.ts create mode 100644 packages/backend/src/apps/self-hosted-llm/auth/verify-credentials.ts create mode 100644 packages/backend/src/apps/self-hosted-llm/common/add-auth-header.ts create mode 100644 packages/backend/src/apps/self-hosted-llm/common/set-base-url.ts create mode 100644 packages/backend/src/apps/self-hosted-llm/dynamic-data/index.ts create mode 100644 packages/backend/src/apps/self-hosted-llm/dynamic-data/list-models/index.ts create mode 100644 packages/backend/src/apps/self-hosted-llm/index.d.ts create mode 100644 packages/backend/src/apps/self-hosted-llm/index.ts diff --git a/packages/backend/src/apps/self-hosted-llm/actions/index.ts b/packages/backend/src/apps/self-hosted-llm/actions/index.ts new file mode 100644 index 00000000..f4e1ffcb --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/actions/index.ts @@ -0,0 +1,4 @@ +import sendPrompt from './send-prompt'; +import sendChatPrompt from './send-chat-prompt'; + +export default [sendChatPrompt, sendPrompt]; diff --git a/packages/backend/src/apps/self-hosted-llm/actions/send-chat-prompt/index.ts b/packages/backend/src/apps/self-hosted-llm/actions/send-chat-prompt/index.ts new file mode 100644 index 00000000..ea239036 --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/actions/send-chat-prompt/index.ts @@ -0,0 +1,137 @@ +import defineAction from '../../../../helpers/define-action'; + +type TMessage = { + role: string; + content: string; +} + +const castFloatOrUndefined = (value: string | null) => { + return value === '' ? undefined : parseFloat(value); +} + +export default defineAction({ + name: 'Send chat prompt', + key: 'sendChatPrompt', + description: 'Creates a completion for the provided prompt and parameters.', + arguments: [ + { + label: 'Model', + key: 'model', + type: 'dropdown' as const, + required: true, + variables: true, + source: { + type: 'query', + name: 'getDynamicData', + arguments: [ + { + name: 'key', + value: 'listModels', + }, + ], + }, + }, + { + label: 'Messages', + key: 'messages', + type: 'dynamic' as const, + required: true, + description: 'Add or remove messages as needed', + value: [{ role: 'system', body: '' }], + fields: [ + { + label: 'Role', + key: 'role', + type: 'dropdown' as const, + required: true, + options: [ + { + label: 'System', + value: 'system', + }, + { + label: 'User', + value: 'user', + } + ], + }, + { + label: 'Content', + key: 'content', + type: 'string' as const, + required: true, + variables: true, + } + ], + }, + { + label: 'Temperature', + key: 'temperature', + type: 'string' as const, + required: false, + variables: true, + description: 'What sampling temperature to use. Higher values mean the model will take more risk. Try 0.9 for more creative applications, and 0 for ones with a well-defined answer. We generally recommend altering this or Top P but not both.' + }, + { + label: 'Maximum tokens', + key: 'maxTokens', + type: 'string' as const, + required: false, + variables: true, + description: 'The maximum number of tokens to generate in the completion.' + }, + { + label: 'Stop Sequence', + key: 'stopSequence', + type: 'string' as const, + required: false, + variables: true, + description: 'Single stop sequence where the API will stop generating further tokens. The returned text will not contain the stop sequence.' + }, + { + label: 'Top P', + key: 'topP', + type: 'string' as const, + required: false, + variables: true, + description: 'An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with Top P probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.' + }, + { + label: 'Frequency Penalty', + key: 'frequencyPenalty', + type: 'string' as const, + required: false, + variables: true, + description: `Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.` + }, + { + label: 'Presence Penalty', + key: 'presencePenalty', + type: 'string' as const, + required: false, + variables: true, + description: `Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.` + }, + ], + + async run($) { + const payload = { + model: $.step.parameters.model as string, + temperature: castFloatOrUndefined($.step.parameters.temperature as string), + max_tokens: castFloatOrUndefined($.step.parameters.maxTokens as string), + stop: ($.step.parameters.stopSequence as string || null), + top_p: castFloatOrUndefined($.step.parameters.topP as string), + frequency_penalty: castFloatOrUndefined($.step.parameters.frequencyPenalty as string), + presence_penalty: castFloatOrUndefined($.step.parameters.presencePenalty as string), + messages: ($.step.parameters.messages as TMessage[]).map(message => ({ + role: message.role, + content: message.content, + })), + }; + const { data } = await $.http.post('/v1/chat/completions', payload); + + $.setActionItem({ + raw: data, + }); + }, +}); diff --git a/packages/backend/src/apps/self-hosted-llm/actions/send-prompt/index.ts b/packages/backend/src/apps/self-hosted-llm/actions/send-prompt/index.ts new file mode 100644 index 00000000..cff0ae8f --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/actions/send-prompt/index.ts @@ -0,0 +1,104 @@ +import defineAction from '../../../../helpers/define-action'; + +const castFloatOrUndefined = (value: string | null) => { + return value === '' ? undefined : parseFloat(value); +} + +export default defineAction({ + name: 'Send prompt', + key: 'sendPrompt', + description: 'Creates a completion for the provided prompt and parameters.', + arguments: [ + { + label: 'Model', + key: 'model', + type: 'dropdown' as const, + required: true, + variables: true, + source: { + type: 'query', + name: 'getDynamicData', + arguments: [ + { + name: 'key', + value: 'listModels', + }, + ], + }, + }, + { + label: 'Prompt', + key: 'prompt', + type: 'string' as const, + required: true, + variables: true, + description: 'The text to analyze.' + }, + { + label: 'Temperature', + key: 'temperature', + type: 'string' as const, + required: false, + variables: true, + description: 'What sampling temperature to use. Higher values mean the model will take more risk. Try 0.9 for more creative applications, and 0 for ones with a well-defined answer. We generally recommend altering this or Top P but not both.' + }, + { + label: 'Maximum tokens', + key: 'maxTokens', + type: 'string' as const, + required: false, + variables: true, + description: 'The maximum number of tokens to generate in the completion.' + }, + { + label: 'Stop Sequence', + key: 'stopSequence', + type: 'string' as const, + required: false, + variables: true, + description: 'Single stop sequence where the API will stop generating further tokens. The returned text will not contain the stop sequence.' + }, + { + label: 'Top P', + key: 'topP', + type: 'string' as const, + required: false, + variables: true, + description: 'An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with Top P probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.' + }, + { + label: 'Frequency Penalty', + key: 'frequencyPenalty', + type: 'string' as const, + required: false, + variables: true, + description: `Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.` + }, + { + label: 'Presence Penalty', + key: 'presencePenalty', + type: 'string' as const, + required: false, + variables: true, + description: `Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.` + }, + ], + + async run($) { + const payload = { + model: $.step.parameters.model as string, + prompt: $.step.parameters.prompt as string, + temperature: castFloatOrUndefined($.step.parameters.temperature as string), + max_tokens: castFloatOrUndefined($.step.parameters.maxTokens as string), + stop: ($.step.parameters.stopSequence as string || null), + top_p: castFloatOrUndefined($.step.parameters.topP as string), + frequency_penalty: castFloatOrUndefined($.step.parameters.frequencyPenalty as string), + presence_penalty: castFloatOrUndefined($.step.parameters.presencePenalty as string), + }; + const { data } = await $.http.post('/v1/completions', payload); + + $.setActionItem({ + raw: data, + }); + }, +}); diff --git a/packages/backend/src/apps/self-hosted-llm/assets/favicon.svg b/packages/backend/src/apps/self-hosted-llm/assets/favicon.svg new file mode 100644 index 00000000..b62b84eb --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/assets/favicon.svg @@ -0,0 +1,6 @@ + + OpenAI + + + + \ No newline at end of file diff --git a/packages/backend/src/apps/self-hosted-llm/auth/index.ts b/packages/backend/src/apps/self-hosted-llm/auth/index.ts new file mode 100644 index 00000000..29bd90c8 --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/auth/index.ts @@ -0,0 +1,44 @@ +import verifyCredentials from './verify-credentials'; +import isStillVerified from './is-still-verified'; + +export default { + fields: [ + { + key: 'screenName', + label: 'Screen Name', + type: 'string' as const, + required: true, + readOnly: false, + value: null, + placeholder: null, + description: + 'Screen name of your connection to be used on Automatisch UI.', + clickToCopy: false, + }, + { + key: 'apiUrl', + label: 'API URL', + type: 'string' as const, + required: true, + readOnly: false, + value: null, + placeholder: null, + docUrl: 'https://automatisch.io/docs/self-hosted-llm#api-url', + clickToCopy: false, + }, + { + key: 'apiKey', + label: 'API Key', + type: 'string' as const, + required: true, + readOnly: false, + value: null, + placeholder: null, + docUrl: 'https://automatisch.io/docs/self-hosted-llm#api-key', + clickToCopy: false, + }, + ], + + verifyCredentials, + isStillVerified, +}; diff --git a/packages/backend/src/apps/self-hosted-llm/auth/is-still-verified.ts b/packages/backend/src/apps/self-hosted-llm/auth/is-still-verified.ts new file mode 100644 index 00000000..da69d50c --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/auth/is-still-verified.ts @@ -0,0 +1,8 @@ +import { IGlobalVariable } from '@automatisch/types'; + +const isStillVerified = async ($: IGlobalVariable) => { + const r = await $.http.get('/v1/models'); + return true; +}; + +export default isStillVerified; diff --git a/packages/backend/src/apps/self-hosted-llm/auth/verify-credentials.ts b/packages/backend/src/apps/self-hosted-llm/auth/verify-credentials.ts new file mode 100644 index 00000000..e9fb2540 --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/auth/verify-credentials.ts @@ -0,0 +1,7 @@ +import { IGlobalVariable } from '@automatisch/types'; + +const verifyCredentials = async ($: IGlobalVariable) => { + await $.http.get('/v1/models'); +}; + +export default verifyCredentials; diff --git a/packages/backend/src/apps/self-hosted-llm/common/add-auth-header.ts b/packages/backend/src/apps/self-hosted-llm/common/add-auth-header.ts new file mode 100644 index 00000000..2b0de0ce --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/common/add-auth-header.ts @@ -0,0 +1,11 @@ +import { TBeforeRequest } from '@automatisch/types'; + +const addAuthHeader: TBeforeRequest = ($, requestConfig) => { + if ($.auth.data?.apiKey) { + requestConfig.headers.Authorization = `Bearer ${$.auth.data.apiKey}`; + } + + return requestConfig; +}; + +export default addAuthHeader; diff --git a/packages/backend/src/apps/self-hosted-llm/common/set-base-url.ts b/packages/backend/src/apps/self-hosted-llm/common/set-base-url.ts new file mode 100644 index 00000000..5dcade86 --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/common/set-base-url.ts @@ -0,0 +1,11 @@ +import { TBeforeRequest } from '@automatisch/types'; + +const setBaseUrl: TBeforeRequest = ($, requestConfig) => { + if ($.auth.data.apiUrl) { + requestConfig.baseURL = $.auth.data.apiUrl as string; + } + + return requestConfig; +}; + +export default setBaseUrl; diff --git a/packages/backend/src/apps/self-hosted-llm/dynamic-data/index.ts b/packages/backend/src/apps/self-hosted-llm/dynamic-data/index.ts new file mode 100644 index 00000000..4072dcdd --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/dynamic-data/index.ts @@ -0,0 +1,3 @@ +import listModels from './list-models'; + +export default [listModels]; diff --git a/packages/backend/src/apps/self-hosted-llm/dynamic-data/list-models/index.ts b/packages/backend/src/apps/self-hosted-llm/dynamic-data/list-models/index.ts new file mode 100644 index 00000000..645b9f1e --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/dynamic-data/list-models/index.ts @@ -0,0 +1,19 @@ +import { IGlobalVariable } from '@automatisch/types'; + +export default { + name: 'List models', + key: 'listModels', + + async run($: IGlobalVariable) { + const response = await $.http.get('/v1/models'); + + const models = response.data.data.map((model: { id: string }) => { + return { + value: model.id, + name: model.id, + }; + }); + + return { data: models }; + }, +}; diff --git a/packages/backend/src/apps/self-hosted-llm/index.d.ts b/packages/backend/src/apps/self-hosted-llm/index.d.ts new file mode 100644 index 00000000..e69de29b diff --git a/packages/backend/src/apps/self-hosted-llm/index.ts b/packages/backend/src/apps/self-hosted-llm/index.ts new file mode 100644 index 00000000..2f2fd5e3 --- /dev/null +++ b/packages/backend/src/apps/self-hosted-llm/index.ts @@ -0,0 +1,21 @@ +import defineApp from '../../helpers/define-app'; +import addAuthHeader from './common/add-auth-header'; +import setBaseUrl from './common/set-base-url'; +import auth from './auth'; +import actions from './actions'; +import dynamicData from './dynamic-data'; + +export default defineApp({ + name: 'Self-hosted LLM', + key: 'self-hosted-llm', + baseUrl: '', + apiBaseUrl: '', + iconUrl: '{BASE_URL}/apps/self-hosted-llm/assets/favicon.svg', + authDocUrl: 'https://automatisch.io/docs/apps/self-hosted-llm/connection', + primaryColor: '000000', + supportsConnections: true, + beforeRequest: [setBaseUrl, addAuthHeader], + auth, + actions, + dynamicData, +});