From a45744a894ee17d4ffa4e6229e5cc82dc6ed1560 Mon Sep 17 00:00:00 2001 From: Adish M <44204658+adishM98@users.noreply.github.com> Date: Thu, 27 Feb 2025 11:47:58 +0530 Subject: [PATCH 01/42] Adding git status-all command to .gitconfig file (#12061) --- .gitconfig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitconfig b/.gitconfig index 3a59d51cdd..5d7b70339d 100644 --- a/.gitconfig +++ b/.gitconfig @@ -139,3 +139,10 @@ echo \"Pushing changes in submodules...\"; \ git submodule foreach --quiet --recursive \"git push\"; \ }; f" + + status-all = "!f() { \ + echo \"Status of base repo...\"; \ + git status; \ + echo \"Status of submodules...\"; \ + git submodule foreach --quiet --recursive \"git status\"; \ + }; f" From 5276bfb667ddb62c32cfc11378c7af98dc89a7c3 Mon Sep 17 00:00:00 2001 From: Adish M <44204658+adishM98@users.noreply.github.com> Date: Fri, 28 Feb 2025 14:32:25 +0530 Subject: [PATCH 02/42] Feature: Cypress for Modularisation (#12071) * Feature: Cypress for Modularisation * Adding submodule authentication * remove render key * adding fix to fallback to default branch * adding fix to fallback to default branch * adding fix to fallback to default branch * adding submodule change to appbuilder cypress * adding submodule change to marketplace cypress --- .github/workflows/cypress-appbuilder.yml | 36 ++++++++++++++++- .github/workflows/cypress-marketplace.yml | 38 ++++++++++++++++-- .github/workflows/cypress-platform.yml | 44 +++++++++++++++------ .github/workflows/render-preview-deploy.yml | 8 ---- 4 files changed, 102 insertions(+), 24 deletions(-) diff --git a/.github/workflows/cypress-appbuilder.yml b/.github/workflows/cypress-appbuilder.yml index acd5d9c08e..47c7bda175 100644 --- a/.github/workflows/cypress-appbuilder.yml +++ b/.github/workflows/cypress-appbuilder.yml @@ -14,7 +14,23 @@ jobs: Cypress-App-Builder: runs-on: ubuntu-22.04 - if: ${{ github.event.action == 'labeled' && (github.event.label.name == 'run-cypress-app-builder' || github.event.label.name == 'run-cypress') }} + if: | + github.event.action == 'labeled' && + ( + github.event.label.name == 'run-cypress' || + github.event.label.name == 'run-ce-app-builder' || + github.event.label.name == 'run-ee-app-builder' + ) + + strategy: + matrix: + edition: >- + ${{ + contains(github.event.pull_request.labels.*.name, 'run-cypress') && fromJson('["ce", "ee"]') || + contains(github.event.pull_request.labels.*.name, 'run-ce-app-builder') && fromJson('["ce"]') || + contains(github.event.pull_request.labels.*.name, 'run-ee-app-builder') && fromJson('["ee"]') || + fromJson('[]') + }} steps: - name: Setup Node.js @@ -22,6 +38,23 @@ jobs: with: node-version: 18.18.2 + - name: Set up Git authentication for private submodules + run: | + git config --global url."https://x-access-token:${{ secrets.CUSTOM_GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" + + - name: Checkout with Submodules + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + + - name: Checking out the correct branch for submodules EE + if: matrix.edition == 'ee' + run: | + git submodule update --init --recursive + git submodule foreach --recursive ' + git checkout ${{ env.BRANCH_NAME }} 2>/dev/null || git checkout modularisation/v3' + + - name: Set up Docker uses: docker-practice/actions-setup-docker@master @@ -45,6 +78,7 @@ jobs: - name: Set up environment variables run: | + echo "TOOLJET_EDITION=${{ matrix.edition == 'ee' && 'EE' || 'CE' }}" >> .env echo "TOOLJET_HOST=http://localhost:8082" >> .env echo "LOCKBOX_MASTER_KEY=cd97331a419c09387bef49787f7da8d2a81d30733f0de6bed23ad8356d2068b2" >> .env echo "SECRET_KEY_BASE=7073b9a35a15dd20914ae17e36a693093f25b74b96517a5fec461fc901c51e011cd142c731bee48c5081ec8bac321c1f259ef097ef2a16f25df17a3798c03426" >> .env diff --git a/.github/workflows/cypress-marketplace.yml b/.github/workflows/cypress-marketplace.yml index 7d2141c41c..c24fe5ae72 100644 --- a/.github/workflows/cypress-marketplace.yml +++ b/.github/workflows/cypress-marketplace.yml @@ -14,7 +14,23 @@ jobs: Cypress-Marketplace: runs-on: ubuntu-22.04 - if: ${{ github.event.action == 'labeled' && (github.event.label.name == 'run-cypress-marketplace' || github.event.label.name == 'run-cypress') }} + if: | + github.event.action == 'labeled' && + ( + github.event.label.name == 'run-cypress' || + github.event.label.name == 'run-ce-cypress-marketplace' || + github.event.label.name == 'run-ee-cypress-marketplace' + ) + + strategy: + matrix: + edition: >- + ${{ + contains(github.event.pull_request.labels.*.name, 'run-cypress') && fromJson('["ce", "ee"]') || + contains(github.event.pull_request.labels.*.name, 'run-ce-cypress-marketplace') && fromJson('["ce"]') || + contains(github.event.pull_request.labels.*.name, 'run-ee-cypress-marketplace') && fromJson('["ee"]') || + fromJson('[]') + }} steps: - name: Checkout @@ -46,12 +62,25 @@ jobs: - name: Set SAFE_BRANCH_NAME run: echo "SAFE_BRANCH_NAME=$(echo ${{ env.BRANCH_NAME }} | tr '/' '-')" >> $GITHUB_ENV - - name: Build and Push Docker image + - name: Build CE Docker image uses: docker/build-push-action@v4 with: context: . - file: docker/production.Dockerfile - push: true + file: docker/ce-production.Dockerfile + push: false + tags: tooljet/tj-osv:${{ env.SAFE_BRANCH_NAME }} + platforms: linux/amd64 + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build EE Docker image + if: matrix.edition == 'ee' + uses: docker/build-push-action@v4 + with: + context: . + file: docker/ee/ee-production.Dockerfile + push: false tags: tooljet/tj-osv:${{ env.SAFE_BRANCH_NAME }} platforms: linux/amd64 env: @@ -60,6 +89,7 @@ jobs: - name: Set up environment variables run: | + echo "TOOLJET_EDITION=${{ matrix.edition == 'ee' && 'EE' || 'CE' }}" >> .env echo "TOOLJET_HOST=http://localhost:3000" >> .env echo "LOCKBOX_MASTER_KEY=cd97331a419c09387bef49787f7da8d2a81d30733f0de6bed23ad8356d2068b2" >> .env echo "SECRET_KEY_BASE=7073b9a35a15dd20914ae17e36a693093f25b74b96517a5fec461fc901c51e011cd142c731bee48c5081ec8bac321c1f259ef097ef2a16f25df17a3798c03426" >> .env diff --git a/.github/workflows/cypress-platform.yml b/.github/workflows/cypress-platform.yml index 622bd43ec9..0d15c01f9c 100644 --- a/.github/workflows/cypress-platform.yml +++ b/.github/workflows/cypress-platform.yml @@ -13,9 +13,22 @@ jobs: Cypress-Platform: runs-on: ubuntu-22.04 if: | - github.event.action == 'labeled' && - (github.event.label.name == 'run-cypress-platform' || - github.event.label.name == 'run-cypress') + github.event.action == 'labeled' && + ( + github.event.label.name == 'run-cypress' || + github.event.label.name == 'run-ce-cypress-platform' || + github.event.label.name == 'run-ee-cypress-platform' + ) + + strategy: + matrix: + edition: >- + ${{ + contains(github.event.pull_request.labels.*.name, 'run-cypress') && fromJson('["ce", "ee"]') || + contains(github.event.pull_request.labels.*.name, 'run-ce-cypress-platform') && fromJson('["ce"]') || + contains(github.event.pull_request.labels.*.name, 'run-ee-cypress-platform') && fromJson('["ee"]') || + fromJson('[]') + }} steps: - name: Setup Node.js @@ -23,11 +36,22 @@ jobs: with: node-version: 18.18.2 - - name: Checkout + - name: Set up Git authentication for private submodules + run: | + git config --global url."https://x-access-token:${{ secrets.CUSTOM_GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" + + - name: Checkout with Submodules uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.ref }} + - name: Checking out the correct branch for submodules EE + if: matrix.edition == 'ee' + run: | + git submodule update --init --recursive + git submodule foreach --recursive ' + git checkout ${{ env.BRANCH_NAME }} 2>/dev/null || git checkout modularisation/v3' + - name: Set up Docker uses: docker-practice/actions-setup-docker@master @@ -55,9 +79,10 @@ jobs: - name: Set up environment variables run: | + echo "TOOLJET_EDITION=${{ matrix.edition == 'ee' && 'EE' || 'CE' }}" >> .env echo "TOOLJET_HOST=http://localhost:8082" >> .env echo "LOCKBOX_MASTER_KEY=cd97331a419c09387bef49787f7da8d2a81d30733f0de6bed23ad8356d2068b2" >> .env - echo "SECRET_KEY_BASE=7073b9a35a15dd20914ae17e36a693093f25b74b96517a5fec461fc901c51e011cd142c731bee48c5081ec8bac321c1f259ef097ef2a16f25df17a3798c03426" >> .env + echo "SECRET_KEY_BASE=7073b9a35a15dd20914ae17e36a693093f25b74b96517a5fec461fc901c51e011cd142c731bee48c5081ec8bac321c1f259ef097ef2a16f25df17a3798c03426" >> .env echo "PG_DB=tooljet_development" >> .env echo "PG_USER=postgres" >> .env echo "PG_HOST=localhost" >> .env @@ -69,7 +94,7 @@ jobs: echo "TOOLJET_DB_HOST=localhost" >> .env echo "TOOLJET_DB_PASS=postgres" >> .env echo "TOOLJET_DB_STATEMENT_TIMEOUT=60000" >> .env - echo "TOOLJET_DB_RECONFIG=true" >> .env + echo "TOOLJET_DB_RECONFIG=true" >> .env echo "PGRST_JWT_SECRET=r9iMKoe5CRMgvJBBtp4HrqN7QiPpUToj" >> .env echo "PGRST_HOST=localhost:3001" >> .env echo "PGRST_DB_PRE_CONFIG=postgrest.pre_config" >> .env @@ -77,10 +102,6 @@ jobs: echo "ENABLE_MARKETPLACE_FEATURE=true" >> .env echo "ENABLE_MARKETPLACE_DEV_MODE=true" >> .env echo "ENABLE_PRIVATE_APP_EMBED=true" >> .env - echo "SSO_GOOGLE_OAUTH2_CLIENT_ID=123456789.apps.googleusercontent.com" >> .env - echo "SSO_GOOGLE_OAUTH2_CLIENT_SECRET=ABCGFDNF-FHSDVFY-bskfh6234" >> .env - echo "SSO_GIT_OAUTH2_CLIENT_ID=1234567890" >> .env - echo "SSO_GIT_OAUTH2_CLIENT_SECRET=3346shfvkdjjsfkvxce32854e026a4531ed" >> .env - name: Set up database run: | @@ -123,9 +144,10 @@ jobs: uses: actions/upload-artifact@v4 if: always() with: - name: screenshots + name: screenshots-${{ matrix.edition }} path: cypress-tests/cypress/screenshots + Cypress-Platform-Subpath: runs-on: ubuntu-22.04 if: | diff --git a/.github/workflows/render-preview-deploy.yml b/.github/workflows/render-preview-deploy.yml index 6f24e3b95f..a025290d39 100644 --- a/.github/workflows/render-preview-deploy.yml +++ b/.github/workflows/render-preview-deploy.yml @@ -492,10 +492,6 @@ jobs: "key": "REDIS_PORT", "value": "${{ secrets.RENDER_REDIS_PORT }}" }, - { - "key": "LICENSE_KEY", - "value": "${{ secrets.RENDER_LICENSE_KEY }}" - }, { "key": "TEMPORAL_SERVER_ADDRESS", "value": "https://auto-setup-1-25-1.onrender.com" @@ -866,10 +862,6 @@ jobs: # "value": "${{ secrets.RENDER_REDIS_PORT }}" # }, # { - # "key": "LICENSE_KEY", - # "value": "${{ secrets.RENDER_LICENSE_KEY }}" - # }, - # { # "key": "TEMPORAL_SERVER_ADDRESS", # "value": "https://auto-setup-1-25-1.onrender.com" # }, From babef456cf3189832de6465bce98cb7b44fb8639 Mon Sep 17 00:00:00 2001 From: Kavin Venkatachalam Date: Wed, 26 Feb 2025 16:02:02 +0530 Subject: [PATCH 03/42] hotfix: expose the proper value while mounting the component --- frontend/src/Editor/Components/RadioButton.jsx | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/frontend/src/Editor/Components/RadioButton.jsx b/frontend/src/Editor/Components/RadioButton.jsx index 4e77f2c928..360efab4af 100644 --- a/frontend/src/Editor/Components/RadioButton.jsx +++ b/frontend/src/Editor/Components/RadioButton.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useRef, useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { v4 as uuidv4 } from 'uuid'; export const RadioButton = function RadioButton({ @@ -12,8 +12,8 @@ export const RadioButton = function RadioButton({ darkMode, dataCy, }) { - const isInitialRender = useRef(true); const { label, value, values, display_values } = properties; + const { visibility, disabledState, activeColor, boxShadow } = styles; const textColor = darkMode && styles.textColor === '#000' ? '#fff' : styles.textColor; const [checkedValue, setValue] = useState(() => value); @@ -37,12 +37,6 @@ export const RadioButton = function RadioButton({ fireEvent('onSelectionChange'); } - useEffect(() => { - if (isInitialRender.current) return; - setExposedVariable('value', value); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - useEffect(() => { const exposedVariables = { value: value, @@ -51,9 +45,8 @@ export const RadioButton = function RadioButton({ }, }; setExposedVariables(exposedVariables); - isInitialRender.current = false; // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); + }, [value]); return (
Date: Mon, 3 Mar 2025 11:53:20 +0530 Subject: [PATCH 04/42] Making CE and Basic Plan logic same for environments (#12066) * Making CE and Basic plan flow same * Added basic plan organization constant update api changes * Added edition downgrade checker --- server/src/helpers/edition.helper.ts | 56 ++++++++++++++++++ server/src/main.ts | 4 ++ .../app-environments/interfaces/IService.ts | 9 ++- .../interfaces/IUtilService.ts | 12 +++- .../src/modules/app-environments/service.ts | 57 ++++++++++--------- .../modules/app-environments/util.service.ts | 43 ++++++++++++-- server/src/modules/apps/service.ts | 35 +----------- .../organization-constants/controller.ts | 8 ++- .../interfaces/IService.ts | 6 +- .../modules/organization-constants/service.ts | 55 ++++++++++++------ 10 files changed, 194 insertions(+), 91 deletions(-) create mode 100644 server/src/helpers/edition.helper.ts diff --git a/server/src/helpers/edition.helper.ts b/server/src/helpers/edition.helper.ts new file mode 100644 index 0000000000..9f90927eee --- /dev/null +++ b/server/src/helpers/edition.helper.ts @@ -0,0 +1,56 @@ +import { DataSource } from 'typeorm'; +import { getTooljetEdition } from './utils.helper'; +import { INestApplication } from '@nestjs/common'; +import { Metadata } from '@entities/metadata.entity'; + +const EDITION_PRIORITY = { + ce: 0, + ee: 1, + cloud: 2, +}; + +export function getEditionPriority(edition: string): number { + return EDITION_PRIORITY[edition?.toLowerCase()] ?? -1; +} + +export function isEditionDowngrade(currentEdition: string, newEdition: string): boolean { + if (!currentEdition || !newEdition) return false; + return getEditionPriority(currentEdition) > getEditionPriority(newEdition); +} + +export async function validateEdition(app: INestApplication) { + if (process.env.NODE_ENV === 'production') return; + + const dataSource = app.get(DataSource); + const metadataRepository = dataSource.getRepository(Metadata); + const editionMetadata = (await metadataRepository.find())?.[0]; + + const currentEdition = getTooljetEdition(); + const savedEdition = editionMetadata?.data?.edition; + + if (savedEdition) { + if (isEditionDowngrade(savedEdition, currentEdition)) { + console.error( + `Cannot downgrade from ${savedEdition} to ${currentEdition}. Please use an equal or higher edition or reset the database.` + ); + await app.close(); + process.exit(0); + } + // change edition + if (savedEdition !== currentEdition) { + await metadataRepository.update( + { id: editionMetadata.id }, + { data: { ...editionMetadata.data, edition: currentEdition } } + ); + } + } else if (editionMetadata) { + await metadataRepository.update( + { id: editionMetadata.id }, + { data: { ...editionMetadata.data, edition: currentEdition } } + ); + } else { + await metadataRepository.save({ + data: { edition: currentEdition }, + }); + } +} diff --git a/server/src/main.ts b/server/src/main.ts index 742e9609ce..4c94d64ac0 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -27,6 +27,7 @@ import { GuardValidator } from '@modules/app/validators/feature-guard.validator' import { ILicenseUtilService } from '@modules/licensing/interfaces/IUtilService'; import { ITemporalService } from '@modules/workflows/interfaces/ITemporalService'; import { getTooljetEdition } from '@helpers/utils.helper'; +import { validateEdition } from '@helpers/edition.helper'; let appContext: INestApplicationContext = undefined; @@ -148,6 +149,9 @@ async function bootstrap() { abortOnError: false, }); + // Get DataSource from the app + await validateEdition(app); + globalThis.TOOLJET_VERSION = `${fs.readFileSync('./.version', 'utf8').trim()}-${getTooljetEdition()}`; process.env['RELEASE_VERSION'] = globalThis.TOOLJET_VERSION; diff --git a/server/src/modules/app-environments/interfaces/IService.ts b/server/src/modules/app-environments/interfaces/IService.ts index 5380381c43..38d6c9811c 100644 --- a/server/src/modules/app-environments/interfaces/IService.ts +++ b/server/src/modules/app-environments/interfaces/IService.ts @@ -2,6 +2,7 @@ import { AppEnvironment } from 'src/entities/app_environments.entity'; import { AppVersion } from 'src/entities/app_version.entity'; import { AppEnvironmentActionParametersDto } from '../dto'; import { IAppEnvironmentResponse } from './IAppEnvironmentResponse'; +import { EntityManager } from 'typeorm'; export interface IAppEnvironmentService { init(editingVersionId: string, organizationId: string): Promise; @@ -10,7 +11,13 @@ export interface IAppEnvironmentService { action: string, actionParameters: AppEnvironmentActionParametersDto ): Promise; - get(organizationId: string, id?: string, priorityCheck?: boolean, licenseCheck?: boolean): Promise; + get( + organizationId: string, + id?: string, + priorityCheck?: boolean, + licenseCheck?: boolean, + manager?: EntityManager + ): Promise; create(organizationId: string, name: string, isDefault?: boolean, priority?: number): Promise; update(id: string, name: string, organizationId: string): Promise; getAll(organizationId: string, appId?: string): Promise; diff --git a/server/src/modules/app-environments/interfaces/IUtilService.ts b/server/src/modules/app-environments/interfaces/IUtilService.ts index 4e2edf63f8..343d387bb8 100644 --- a/server/src/modules/app-environments/interfaces/IUtilService.ts +++ b/server/src/modules/app-environments/interfaces/IUtilService.ts @@ -1,8 +1,8 @@ -import { EntityManager, DeleteResult } from 'typeorm'; +import { EntityManager } from 'typeorm'; import { AppEnvironment } from 'src/entities/app_environments.entity'; -import { OrgEnvironmentConstantValue } from 'src/entities/org_environment_constant_values.entity'; import { DataSourceOptions } from '@entities/data_source_options.entity'; -import { OrganizationConstantType } from '@modules/organization-constants/constants'; +import { IAppEnvironmentResponse } from './IAppEnvironmentResponse'; +import { AppVersion } from '@entities/app_version.entity'; export interface IAppEnvironmentUtilService { getByPriority(organizationId: string, ASC?: boolean, manager?: EntityManager): Promise; @@ -27,4 +27,10 @@ export interface IAppEnvironmentUtilService { ): Promise; getAll(organizationId: string, appId?: string, manager?: EntityManager): Promise; getOptions(dataSourceId: string, organizationId: string, environmentId?: string): Promise; + init( + editorVersion: Partial, + organizationId: string, + isMultiEnvironmentEnabled: boolean, + manager?: EntityManager + ): Promise; } diff --git a/server/src/modules/app-environments/service.ts b/server/src/modules/app-environments/service.ts index 473da7bafd..7f591ef034 100644 --- a/server/src/modules/app-environments/service.ts +++ b/server/src/modules/app-environments/service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; import { AppEnvironment } from 'src/entities/app_environments.entity'; -import { EntityManager, In } from 'typeorm'; +import { EntityManager, FindOneOptions, In } from 'typeorm'; import { AppVersion } from 'src/entities/app_version.entity'; import { AppEnvironmentActions } from './constants'; import { IAppEnvironmentService } from './interfaces/IService'; @@ -8,38 +8,22 @@ import { AppEnvironmentActionParametersDto } from './dto'; import { dbTransactionWrap } from '@helpers/database.helper'; import { IAppEnvironmentResponse } from './interfaces/IAppEnvironmentResponse'; import { AppEnvironmentUtilService } from './util.service'; +import { LicenseTermsService } from '@modules/licensing/interfaces/IService'; +import { LICENSE_FIELD } from '@modules/licensing/constants'; @Injectable() export class AppEnvironmentService implements IAppEnvironmentService { - constructor(protected readonly appEnvironmentUtilService: AppEnvironmentUtilService) {} + constructor( + protected readonly appEnvironmentUtilService: AppEnvironmentUtilService, + protected readonly licenseTermsService: LicenseTermsService + ) {} async init(editingVersionId: string, organizationId: string): Promise { return await dbTransactionWrap(async (manager: EntityManager) => { const editorVersion = await manager.findOne(AppVersion, { - select: ['id', 'name', 'currentEnvironmentId', 'appId'], + select: ['id', 'name', 'appId'], where: { id: editingVersionId }, }); - const environments: AppEnvironment[] = await this.appEnvironmentUtilService.getAll( - organizationId, - editorVersion.appId, - manager - ); - const editorEnvironment = environments.find((env) => env.id === editorVersion.currentEnvironmentId); - const { shouldRenderPromoteButton, shouldRenderReleaseButton } = - await this.appEnvironmentUtilService.calculateButtonVisibility( - false, - editorEnvironment as AppEnvironment, - editorVersion.appId, - editorVersion.id - ); - const response: IAppEnvironmentResponse = { - editorVersion, - editorEnvironment: editorEnvironment as AppEnvironment, - appVersionEnvironment: editorEnvironment as AppEnvironment, - shouldRenderPromoteButton, - shouldRenderReleaseButton, - environments, - }; - return response; + return await this.appEnvironmentUtilService.init(editorVersion, organizationId, false, manager); }); } @@ -154,10 +138,27 @@ export class AppEnvironmentService implements IAppEnvironmentService { }); } - async get(organizationId: string, id?: string, priorityCheck = false): Promise { + async get( + organizationId: string, + id?: string, + priorityCheck = false, + licenseCheck = false, + manager?: EntityManager + ): Promise { + const isMultiEnvironmentEnabled = licenseCheck + ? await this.licenseTermsService.getLicenseTerms(LICENSE_FIELD.MULTI_ENVIRONMENT) + : false; + return await dbTransactionWrap(async (manager: EntityManager) => { - return await this.appEnvironmentUtilService.get(organizationId, id, priorityCheck, manager); - }); + const condition: FindOneOptions = { + where: { + organizationId, + ...(id ? { id } : !isMultiEnvironmentEnabled ? { priority: 1 } : !priorityCheck ? { isDefault: true } : {}), + }, + ...(priorityCheck && { order: { priority: 'ASC' } }), + }; + return await manager.findOneOrFail(AppEnvironment, condition); + }, manager); } async create(organizationId: string, name: string, isDefault = false, priority: number): Promise { diff --git a/server/src/modules/app-environments/util.service.ts b/server/src/modules/app-environments/util.service.ts index 6361ab89ee..bc4621f888 100644 --- a/server/src/modules/app-environments/util.service.ts +++ b/server/src/modules/app-environments/util.service.ts @@ -8,10 +8,13 @@ import { AppVersion } from '@entities/app_version.entity'; import { App } from '@entities/app.entity'; import { FindOneOptions } from 'typeorm'; import { defaultAppEnvironments } from '@helpers/utils.helper'; +import { LICENSE_FIELD } from '@modules/licensing/constants'; +import { LicenseTermsService } from '@modules/licensing/interfaces/IService'; +import { IAppEnvironmentResponse } from './interfaces/IAppEnvironmentResponse'; @Injectable() export class AppEnvironmentUtilService implements IAppEnvironmentUtilService { - constructor() {} + constructor(protected readonly licenseTermsService: LicenseTermsService) {} async updateOptions(options: object, environmentId: string, dataSourceId: string, manager?: EntityManager) { await dbTransactionWrap(async (manager: EntityManager) => { await manager.update( @@ -124,14 +127,15 @@ export class AppEnvironmentUtilService implements IAppEnvironmentUtilService { organizationId: string, id?: string, priorityCheck = false, - manager?: EntityManager, - licenseCheck = false + manager?: EntityManager ): Promise { + const isMultiEnvironmentEnabled = await this.licenseTermsService.getLicenseTerms(LICENSE_FIELD.MULTI_ENVIRONMENT); + return await dbTransactionWrap(async (manager: EntityManager) => { const condition: FindOneOptions = { where: { organizationId, - ...(id ? { id } : !priorityCheck && { isDefault: true }), + ...(id ? { id } : !isMultiEnvironmentEnabled ? { priority: 1 } : !priorityCheck ? { isDefault: true } : {}), }, ...(priorityCheck && { order: { priority: 'ASC' } }), }; @@ -181,4 +185,35 @@ export class AppEnvironmentUtilService implements IAppEnvironmentUtilService { }); }); } + + async init( + editorVersion: Partial, + organizationId: string, + isMultiEnvironmentEnabled = false, + manager?: EntityManager + ): Promise { + const environments: AppEnvironment[] = await this.getAll(organizationId, editorVersion.appId, manager); + let editorEnvironment: AppEnvironment; + if (!isMultiEnvironmentEnabled) { + editorEnvironment = environments.find((env) => env.priority === 1); + } else { + editorEnvironment = environments.find((env) => env.id === editorVersion.currentEnvironmentId); + } + const { shouldRenderPromoteButton, shouldRenderReleaseButton } = await this.calculateButtonVisibility( + isMultiEnvironmentEnabled, + editorEnvironment, + editorVersion.appId, + editorVersion.id, + manager + ); + const response: IAppEnvironmentResponse = { + editorVersion, + editorEnvironment, + appVersionEnvironment: editorEnvironment, + shouldRenderPromoteButton, + shouldRenderReleaseButton, + environments, + }; + return response; + } } diff --git a/server/src/modules/apps/service.ts b/server/src/modules/apps/service.ts index 3a855b65bd..7ad0ca351a 100644 --- a/server/src/modules/apps/service.ts +++ b/server/src/modules/apps/service.ts @@ -382,7 +382,7 @@ export class AppsService implements IAppsService { async release(app: App, user: User, versionReleaseDto: VersionReleaseDto) { await dbTransactionWrap(async (manager: EntityManager) => { const { versionToBeReleased } = versionReleaseDto; - const { id: appId, currentVersionId: lastReleasedVersion, organizationId } = app; + const { id: appId } = app; //check if the app version is eligible for release const currentEnvironment: AppEnvironment = await manager .createQueryBuilder(AppEnvironment, 'app_environments') @@ -404,39 +404,6 @@ export class AppsService implements IAppsService { throw new BadRequestException('You can only release when the version is promoted to production'); } - let promotedFromQuery: string; - if (!isMultiEnvironmentEnabled) { - if (!currentEnvironment.isDefault) { - /* For basic plan users, Promote to the production environment first then release it */ - const productionEnv = await this.appEnvironmentUtilService.get(organizationId, null, false, manager); - await manager.update(AppVersion, versionToBeReleased, { - currentEnvironmentId: productionEnv.id, - promotedFrom: currentEnvironment.id, - }); - } - - /* demote the last released environment back to the promoted_from (if not null) */ - if (lastReleasedVersion) { - promotedFromQuery = ` - UPDATE app_versions - SET current_environment_id = promoted_from - WHERE promoted_from IS NOT NULL - AND id = $1;`; - } - } else { - if (lastReleasedVersion) { - promotedFromQuery = ` - UPDATE app_versions - SET promoted_from = NULL - WHERE promoted_from IS NOT NULL - AND id = $1;`; - } - } - - if (promotedFromQuery) { - await manager.query(promotedFromQuery, [lastReleasedVersion]); - } - await manager.update(App, appId, { currentVersionId: versionToBeReleased }); this.eventEmitter.emit('auditLogEntry', { userId: user.id, diff --git a/server/src/modules/organization-constants/controller.ts b/server/src/modules/organization-constants/controller.ts index 84f40e7121..9a787ffb48 100644 --- a/server/src/modules/organization-constants/controller.ts +++ b/server/src/modules/organization-constants/controller.ts @@ -43,8 +43,12 @@ export class OrganizationConstantController implements IOrganizationConstantCont @Get(':app_slug') @InitFeature(FEATURE_KEY.GET_FROM_APP) async getConstantsFromApp(@User() user, @Query('environmentId') environmentId) { - const result = await this.organizationConstantsService.allEnvironmentConstants(user.organizationId); - return { constants: {} }; + const result = await this.organizationConstantsService.getConstantsForEnvironment( + user.organizationId, + environmentId, + OrganizationConstantType.GLOBAL + ); + return { constants: result }; } @UseGuards(JwtAuthGuard, FeatureAbilityGuard) diff --git a/server/src/modules/organization-constants/interfaces/IService.ts b/server/src/modules/organization-constants/interfaces/IService.ts index 79c9e079bf..ec9cc157eb 100644 --- a/server/src/modules/organization-constants/interfaces/IService.ts +++ b/server/src/modules/organization-constants/interfaces/IService.ts @@ -16,12 +16,14 @@ export interface IOrganizationConstantsService { ): Promise; create( organizationConstant: CreateOrganizationConstantDto, - organizationId: string + organizationId: string, + isMultiEnvEnabled?: boolean ): Promise; update( constantId: string, organizationId: string, - params: UpdateOrganizationConstantDto + params: UpdateOrganizationConstantDto, + isMultiEnvEnabled?: boolean ): Promise; delete(constantId: string, organizationId: string, environmentId?: string): Promise; } diff --git a/server/src/modules/organization-constants/service.ts b/server/src/modules/organization-constants/service.ts index bda46fb82f..60b0b84019 100644 --- a/server/src/modules/organization-constants/service.ts +++ b/server/src/modules/organization-constants/service.ts @@ -83,7 +83,8 @@ export class OrganizationConstantsService implements IOrganizationConstantsServi async create( organizationConstant: CreateOrganizationConstantDto, - organizationId: string + organizationId: string, + isMultiEnvEnabled?: boolean ): Promise { return await dbTransactionWrap(async (manager: EntityManager) => { const newOrganizationConstant = this.organizationConstantRepository.create({ @@ -101,14 +102,22 @@ export class OrganizationConstantsService implements IOrganizationConstantsServi manager ); - const environmentsIds = organizationConstant.environments; - - const environmentToUpdate = environmentsIds.map(async (environmentId) => { - return await this.appEnvironmentUtilService.get(organizationId, environmentId, false); - }); + let environmentsToUpdate = []; + if (isMultiEnvEnabled) { + const environmentsIds = organizationConstant.environments; + environmentsToUpdate = environmentsIds.map(async (environmentId) => { + return await this.appEnvironmentUtilService.get(organizationId, environmentId, false); + }); + } else { + /* + Basic plan customer. lets update all environment constant values. + this will help us to run the apps successfully when the user buys enterprise plan + */ + environmentsToUpdate = await this.appEnvironmentUtilService.getAll(organizationId); + } await Promise.all( - environmentToUpdate.map(async (environment) => { + environmentsToUpdate.map(async (environment) => { const encryptedValue = await this.organizationConstantsUtilService.encryptSecret( organizationId, organizationConstant.value @@ -131,7 +140,8 @@ export class OrganizationConstantsService implements IOrganizationConstantsServi async update( constantId: string, organizationId: string, - params: UpdateOrganizationConstantDto + params: UpdateOrganizationConstantDto, + isMultiEnvironmentEnabled = false ): Promise { const { constant_name, environment_id, value } = params; @@ -153,16 +163,27 @@ export class OrganizationConstantsService implements IOrganizationConstantsServi } await manager.save(constantToUpdate); + let environmentsToUpdate = []; + if (!isMultiEnvironmentEnabled) { + environmentsToUpdate = await this.appEnvironmentUtilService.getAll(organizationId); + } else { + const environment = await this.appEnvironmentUtilService.get(organizationId, environment_id, false); + environmentsToUpdate.push(environment); + } - const environmentToUpdate = await this.appEnvironmentUtilService.get(organizationId, environment_id, false); - const encryptedValue = await this.organizationConstantsUtilService.encryptSecret(organizationId, value); - - await this.organizationConstantsUtilService.updateOrgEnvironmentConstant( - encryptedValue, - environmentToUpdate.id, - constantToUpdate.id, - manager - ); + if (value) { + await Promise.all( + environmentsToUpdate.map(async (environment) => { + const encryptedValue = await this.organizationConstantsUtilService.encryptSecret(organizationId, value); + await this.organizationConstantsUtilService.updateOrgEnvironmentConstant( + encryptedValue, + environment.id, + constantToUpdate.id, + manager + ); + }) + ); + } return constantToUpdate; }); From 0342d75278ac070d8a5b86d7912e5864ece4662b Mon Sep 17 00:00:00 2001 From: Adish M <44204658+adishM98@users.noreply.github.com> Date: Mon, 3 Mar 2025 14:22:24 +0530 Subject: [PATCH 05/42] Feature: Docker image build with Modularisation changes (#12089) - Added change for Edition specfic docker image build - Added Cloud specific dockerfile --- .github/workflows/docker-release.yml | 232 +++++++++++++++++++++++++++ docker/cloud/cloud-server.Dockerfile | 117 ++++++++++++++ 2 files changed, 349 insertions(+) create mode 100644 .github/workflows/docker-release.yml create mode 100644 docker/cloud/cloud-server.Dockerfile diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml new file mode 100644 index 0000000000..54774c1d7a --- /dev/null +++ b/.github/workflows/docker-release.yml @@ -0,0 +1,232 @@ +name: ToolJet Edition docker images builds + +on: + release: + types: [published] + +jobs: + build-tooljet-image-for-ce-edtion: + runs-on: ubuntu-latest + if: "${{ github.event.release }}" + + steps: + + - name: Checkout code to main for Beta CE edition + if: "!contains(github.event.release.tag_name, 'ce-lts')" + uses: actions/checkout@v2 + with: + ref: refs/heads/main + + - name: Checkout code to LTS for CE LTS edition + if: "contains(github.event.release.tag_name, '-ce-lts')" + uses: actions/checkout@v2 + with: + ref: refs/heads/lts-4.0 + + # Create Docker Buildx builder with platform configuration + - name: Set up Docker Buildx + run: | + mkdir -p ~/.docker/cli-plugins + curl -SL https://github.com/docker/buildx/releases/download/v0.11.0/buildx-v0.11.0.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx + chmod a+x ~/.docker/cli-plugins/docker-buildx + docker buildx create --name mybuilder --platform linux/arm64,linux/amd64,linux/amd64/v2,linux/riscv64,linux/ppc64le,linux/s390x,linux/386,linux/mips64le,linux/mips64,linux/arm/v7,linux/arm/v6 + docker buildx use mybuilder + + - name: Set DOCKER_CLI_EXPERIMENTAL + run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV + + - name: use mybuilder buildx + run: docker buildx use mybuilder + + - name: Docker Login + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and Push Docker image for Beta tag + if: "!contains(github.event.release.tag_name, '-ce-lts')" + uses: docker/build-push-action@v4 + with: + context: . + file: docker/ce-production.Dockerfile + push: true + tags: tooljet/tooljet-ce:${{ github.event.release.tag_name }},tooljet/tooljet-ce:ce-latest + platforms: linux/amd64 + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and Push Docker image for LTS tag + if: "contains(github.event.release.tag_name, '-ce-lts')" + uses: docker/build-push-action@v4 + with: + context: . + file: docker/ce-production.Dockerfile + push: true + tags: tooljet/tooljet-ce:${{ github.event.release.tag_name }},tooljet/tooljet-ce:ce-lts-latest + platforms: linux/amd64 + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + - name: Send Slack Notification + run: | + if [[ "${{ job.status }}" == "success" ]]; then + message="ToolJet community image published:\n\`tooljet/tooljet-ce:${{ github.event.release.tag_name }}\`" + else + message="Job '${{ env.JOB_NAME }}' failed! tooljet/tooljet-ce:${{ github.event.release.tag_name }}" + fi + + curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$message\"}" ${{ secrets.SLACK_WEBHOOK_URL }} + + + - name: Send Slack Notification + run: | + if [[ "${{ job.status }}" == "success" ]]; then + message="ToolJet community image published:\n\`tooljet/tooljet-ce:${{ github.event.release.tag_name }}\`" + else + message="Job '${{ env.JOB_NAME }}' failed! tooljet/tooljet-ce:${{ github.event.release.tag_name }}" + fi + + curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$message\"}" ${{ secrets.SLACK_WEBHOOK_URL }} + + + build-tooljet-image-for-ee-edtion: + + runs-on: ubuntu-latest + if: "${{ github.event.release }}" + + steps: + - name: Checkout code to main for Beta EE edition + if: "!contains(github.event.release.tag_name, 'ee-lts')" + uses: actions/checkout@v2 + with: + ref: refs/heads/main + + - name: Checkout code to LTS for EE LTS edition + if: "contains(github.event.release.tag_name, '-ee-lts')" + uses: actions/checkout@v2 + with: + ref: refs/heads/lts-4.0 + + # Create Docker Buildx builder with platform configuration + - name: Set up Docker Buildx + run: | + mkdir -p ~/.docker/cli-plugins + curl -SL https://github.com/docker/buildx/releases/download/v0.11.0/buildx-v0.11.0.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx + chmod a+x ~/.docker/cli-plugins/docker-buildx + docker buildx create --name mybuilder --platform linux/arm64,linux/amd64,linux/amd64/v2,linux/riscv64,linux/ppc64le,linux/s390x,linux/386,linux/mips64le,linux/mips64,linux/arm/v7,linux/arm/v6 + docker buildx use mybuilder + + - name: Set DOCKER_CLI_EXPERIMENTAL + run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV + + - name: use mybuilder buildx + run: docker buildx use mybuilder + + - name: Docker Login + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + + - name: Build and Push Docker image for Beta tag + if: "!contains(github.event.release.tag_name, '-ee-lts')" + uses: docker/build-push-action@v4 + with: + context: . + args: ${{ secrets.CUSTOM_GITHUB_TOKEN }} + file: docker/ee-production.Dockerfile + push: true + tags: tooljet/tooljet-ee:${{ github.event.release.tag_name }},tooljet/tooljet-ee:ee-lts-latest,tooljet/tooljet:ee-lts-latest,tooljet/tooljet:${{ github.event.release.tag_name }} + platforms: linux/amd64 + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + + - name: Build and Push Docker image for LTS tag + if: "contains(github.event.release.tag_name, '-ee-lts')" + uses: docker/build-push-action@v4 + with: + context: . + args: ${{ secrets.CUSTOM_GITHUB_TOKEN }} + file: docker/ee-production.Dockerfile + push: true + tags: tooljet/tooljet-ee:${{ github.event.release.tag_name }},tooljet/tooljet-ee:ee-lts-latest,tooljet/tooljet:ee-lts-latest,tooljet/tooljet:${{ github.event.release.tag_name }} + platforms: linux/amd64 + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + - name: Send Slack Notification + run: | + if [[ "${{ job.status }}" == "success" ]]; then + message="ToolJet enterprise image published:\n\`tooljet/tooljet-ee:${{ github.event.release.tag_name }}\`\n\`tooljet/tooljet:${{ github.event.release.tag_name }}\`" + else + message="Job '${{ env.JOB_NAME }}' failed! Image built:\n\`tooljet/tooljet-ee:${{ github.event.release.tag_name }}\`\n\`tooljet/tooljet:${{ github.event.release.tag_name }}\`" + fi + + curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$message\"}" ${{ secrets.SLACK_WEBHOOK_URL }} + + + build-tooljet-image-for-cloud-edtion: + + runs-on: ubuntu-latest + if: "${{ github.event.release }}" + + steps: + - name: Checkout code to LTS for Cloud LTS edition + if: "contains(github.event.release.tag_name, '-cloud-lts')" + uses: actions/checkout@v2 + with: + ref: refs/heads/lts-4.0 + + # Create Docker Buildx builder with platform configuration + - name: Set up Docker Buildx + run: | + mkdir -p ~/.docker/cli-plugins + curl -SL https://github.com/docker/buildx/releases/download/v0.11.0/buildx-v0.11.0.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx + chmod a+x ~/.docker/cli-plugins/docker-buildx + docker buildx create --name mybuilder --platform linux/arm64,linux/amd64,linux/amd64/v2,linux/riscv64,linux/ppc64le,linux/s390x,linux/386,linux/mips64le,linux/mips64,linux/arm/v7,linux/arm/v6 + docker buildx use mybuilder + + - name: Set DOCKER_CLI_EXPERIMENTAL + run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV + + - name: use mybuilder buildx + run: docker buildx use mybuilder + + - name: Docker Login + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and Push Docker image for LTS tag + if: "contains(github.event.release.tag_name, '-cloud-lts')" + uses: docker/build-push-action@v4 + with: + context: . + args: ${{ secrets.CUSTOM_GITHUB_TOKEN }} + file: docker/cloud/cloud-server.Dockerfile + push: true + tags: tooljet/saas:${{ github.event.release.tag_name }} + platforms: linux/amd64 + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + + - name: Send Slack Notification + run: | + if [[ "${{ job.status }}" == "success" ]]; then + message="ToolJet cloud image published:\n\`tooljet/saas:${{ github.event.release.tag_name }}\`" + else + message="Job '${{ env.JOB_NAME }}' failed! Image built:\n\`tooljet/saas:${{ github.event.release.tag_name }}\`" + fi + + curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$message\"}" ${{ secrets.SLACK_WEBHOOK_URL }} + + diff --git a/docker/cloud/cloud-server.Dockerfile b/docker/cloud/cloud-server.Dockerfile new file mode 100644 index 0000000000..a808506ffb --- /dev/null +++ b/docker/cloud/cloud-server.Dockerfile @@ -0,0 +1,117 @@ +FROM node:18.18.2-buster as builder + +# Fix for JS heap limit allocation issue +ENV NODE_OPTIONS="--max-old-space-size=4096" + +RUN npm i -g npm@9.8.1 +RUN npm install -g @nestjs/cli + +RUN mkdir -p /app +WORKDIR /app + +ARG CUSTOM_GITHUB_TOKEN +ARG BRANCH_NAME=modularisation/v3 + +# Clone and checkout the frontend repository +RUN git config --global url."https://x-access-token:${CUSTOM_GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" + +RUN git config --global http.version HTTP/1.1 +RUN git config --global http.postBuffer 524288000 +RUN git clone https://github.com/ToolJet/ToolJet.git . + +# The branch name needs to be changed the branch with modularisation in CE repo +RUN git checkout ${BRANCH_NAME} + +RUN git submodule update --init --recursive + +# Checkout the same branch in submodules if it exists, otherwise stay on default branch +RUN git submodule foreach 'git checkout ${BRANCH_NAME} || true' + +COPY ./package.json ./package.json + +# Building ToolJet plugins +COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/ +RUN npm --prefix plugins install +COPY ./plugins/ ./plugins/ +ENV NODE_ENV=production +RUN npm --prefix plugins run build +RUN npm --prefix plugins prune --production + +# Building ToolJet server +COPY ./server/package.json ./server/package-lock.json ./server/ +RUN npm --prefix server install --only=production +COPY ./server/ ./server/ +RUN npm --prefix server run build + +FROM debian:11 + +RUN apt-get update -yq \ + && apt-get install curl gnupg zip -yq \ + && apt-get install -yq build-essential \ + && apt-get clean -y + +RUN curl -O https://nodejs.org/dist/v18.18.2/node-v18.18.2-linux-x64.tar.xz \ + && tar -xf node-v18.18.2-linux-x64.tar.xz \ + && mv node-v18.18.2-linux-x64 /usr/local/lib/nodejs \ + && echo 'export PATH="/usr/local/lib/nodejs/bin:$PATH"' >> /etc/profile.d/nodejs.sh \ + && /bin/bash -c "source /etc/profile.d/nodejs.sh" \ + && rm node-v18.18.2-linux-x64.tar.xz +ENV PATH=/usr/local/lib/nodejs/bin:$PATH + +ENV NODE_ENV=production +ENV NODE_OPTIONS="--max-old-space-size=4096" +RUN apt-get update && apt-get install -y postgresql-client freetds-dev libaio1 wget + +# Install Instantclient Basic Light Oracle and Dependencies +WORKDIR /opt/oracle +RUN wget https://tooljet-plugins-production.s3.us-east-2.amazonaws.com/marketplace-assets/oracledb/instantclients/instantclient-basiclite-linuxx64.zip && \ + wget https://tooljet-plugins-production.s3.us-east-2.amazonaws.com/marketplace-assets/oracledb/instantclients/instantclient-basiclite-linux.x64-11.2.0.4.0.zip && \ + unzip instantclient-basiclite-linuxx64.zip && rm -f instantclient-basiclite-linuxx64.zip && \ + unzip instantclient-basiclite-linux.x64-11.2.0.4.0.zip && rm -f instantclient-basiclite-linux.x64-11.2.0.4.0.zip && \ + cd /opt/oracle/instantclient_21_10 && rm -f *jdbc* *occi* *mysql* *mql1* *ipc1* *jar uidrvci genezi adrci && \ + cd /opt/oracle/instantclient_11_2 && rm -f *jdbc* *occi* *mysql* *mql1* *ipc1* *jar uidrvci genezi adrci && \ + echo /opt/oracle/instantclient* > /etc/ld.so.conf.d/oracle-instantclient.conf && ldconfig +# Set the Instant Client library paths +ENV LD_LIBRARY_PATH="/opt/oracle/instantclient_11_2:/opt/oracle/instantclient_21_10:${LD_LIBRARY_PATH}" + +WORKDIR / + +RUN mkdir -p /app + +# copy npm scripts +COPY --from=builder /app/package.json ./app/package.json + +# copy plugins dependencies +COPY --from=builder /app/plugins/dist ./app/plugins/dist +COPY --from=builder /app/plugins/client.js ./app/plugins/client.js +COPY --from=builder /app/plugins/node_modules ./app/plugins/node_modules +COPY --from=builder /app/plugins/packages/common ./app/plugins/packages/common +COPY --from=builder /app/plugins/package.json ./app/plugins/package.json + +# copy server build +COPY --from=builder /app/server/package.json ./app/server/package.json +COPY --from=builder /app/server/.version ./app/server/.version +COPY --from=builder /app/server/entrypoint.sh ./app/server/entrypoint.sh +COPY --from=builder /app/server/node_modules ./app/server/node_modules +COPY --from=builder /app/server/templates ./app/server/templates +COPY --from=builder /app/server/scripts ./app/server/scripts +COPY --from=builder /app/server/dist ./app/server/dist + +# Define non-sudo user +RUN useradd --create-home --home-dir /home/appuser appuser \ + && chown -R appuser:0 /app \ + && chown -R appuser:0 /home/appuser \ + && chmod u+x /app \ + && chmod -R g=u /app + +# Set npm cache directory +ENV npm_config_cache /home/appuser/.npm + +ENV HOME=/home/appuser +USER appuser + +WORKDIR /app +# Dependencies for scripts outside nestjs +RUN npm install dotenv@10.0.0 joi@17.4.1 + +ENTRYPOINT ["./server/entrypoint.sh"] From b8ae574e2d4b9b8d96dd05962fdab3762c567543 Mon Sep 17 00:00:00 2001 From: Adish M <44204658+adishM98@users.noreply.github.com> Date: Mon, 3 Mar 2025 14:26:41 +0530 Subject: [PATCH 06/42] Removed unused Github actions (#12091) --- .github/workflows/doc-release.yml | 39 ------ .../tooljet-docker-develop-build.yml | 65 --------- .github/workflows/update-lts-test-system.yml | 40 ------ .github/workflows/update-test-system.yml | 132 ------------------ 4 files changed, 276 deletions(-) delete mode 100644 .github/workflows/doc-release.yml delete mode 100644 .github/workflows/tooljet-docker-develop-build.yml delete mode 100644 .github/workflows/update-lts-test-system.yml delete mode 100644 .github/workflows/update-test-system.yml diff --git a/.github/workflows/doc-release.yml b/.github/workflows/doc-release.yml deleted file mode 100644 index 2574d105a8..0000000000 --- a/.github/workflows/doc-release.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Documentation version release - -on: - workflow_dispatch: - inputs: - create-branch: - description: "Branch name" - version: - description: "RELEASE_VERSION" - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Setup node 16.14 - uses: actions/setup-node@v2 - with: - node-version: 16.14 - - run: cd docs && yarn install && npm run docusaurus docs:version ${{ github.event.inputs.version }} - - - name: Create Pull Request - id: doc - uses: peter-evans/create-pull-request@v5 - with: - title: "Creating a new version folder ${{ github.event.version }}" - body: "Created a new version folder for version: ${{ github.event.inputs.version }}" - branch: ${{ github.event.inputs.create-branch }} - base: "develop" - token: ${{ secrets.GITHUB }} - delete-branch : true - labels: versioned-docs, automated pr - commit-message: added new version folder - - - diff --git a/.github/workflows/tooljet-docker-develop-build.yml b/.github/workflows/tooljet-docker-develop-build.yml deleted file mode 100644 index 7dab1342c3..0000000000 --- a/.github/workflows/tooljet-docker-develop-build.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Tooljet develop docker image build - -on: - push: - branches: - - develop - - workflow_dispatch: - inputs: - job-to-run: - description: Enter the job name (tooljet-develop-image) - options: ["tooljet-develop-image"] - required: true - -jobs: - tooljet-develop-image: - runs-on: ubuntu-latest - if: | - ${{ github.ref == 'refs/heads/develop' }} && - ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.job-to-run == 'tooljet-develop-image' }} - - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - ref: refs/heads/develop - - # Create Docker Buildx builder with platform configuration - - name: Set up Docker Buildx - run: | - mkdir -p ~/.docker/cli-plugins - curl -SL https://github.com/docker/buildx/releases/download/v0.11.0/buildx-v0.11.0.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx - chmod a+x ~/.docker/cli-plugins/docker-buildx - docker buildx create --name mybuilder --platform linux/arm64,linux/amd64,linux/amd64/v2,linux/riscv64,linux/ppc64le,linux/s390x,linux/386,linux/mips64le,linux/mips64,linux/arm/v7,linux/arm/v6 - docker buildx use mybuilder - - - name: Set DOCKER_CLI_EXPERIMENTAL - run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV - - - name: use mybuilder buildx - run: docker buildx use mybuilder - - - name: Docker Login - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Build and Push Docker image - uses: docker/build-push-action@v4 - with: - context: . - file: docker/production.Dockerfile - push: true - tags: tooljet/tooljet-ce:develop - platforms: linux/amd64 - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - - - name: Send Slack Notification on Failure - if: failure() - run: | - message="Job '${{ env.JOB_NAME }}' failed! tooljet/tooljet-ce:develop" - curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$message\"}" ${{ secrets.SLACK_WEBHOOK_URL_OPS_CHANNEL }} diff --git a/.github/workflows/update-lts-test-system.yml b/.github/workflows/update-lts-test-system.yml deleted file mode 100644 index c0757dcd5f..0000000000 --- a/.github/workflows/update-lts-test-system.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: LTS Test system deploy - -on: - workflow_run: - workflows: ["Tooljet release docker images build"] - types: - - completed - -jobs: - Build-and-update-image: - runs-on: ubuntu-22.04 - - steps: - - name: SSH into GCP VM instance - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.EC2_INSTANCE_IP }} - username: ${{ secrets.GCP_USERNAME }} - key: ${{ secrets.EC2_INSTANCE_SSH_KEY }} - script: | - ls -lah - - # Stop the Docker containers - sudo docker-compose down - - # Remove the existing tooljet/* images - sudo docker images -a | grep 'tooljet/' | awk '{print $3}' | xargs sudo docker rmi -f - - # Check remaining images - sudo docker images - - # Update docker-compose.yml with the new image for tooljet service - sed -i '/^[[:space:]]*tooljet:/,/^[[:space:]]*[^:]*$/ { /^[[:space:]]*image:[[:space:]]*tooljet\/tj-osv/s|\(image:[[:space:]]*\).*|\1tooljet/tj-osv:'"${{ env.SAFE_BRANCH_NAME }}"'| }' docker-compose.yaml - - # Start the Docker containers - cat docker-compose.yaml - sudo docker-compose up -d - - #View containers - sudo docker ps diff --git a/.github/workflows/update-test-system.yml b/.github/workflows/update-test-system.yml deleted file mode 100644 index 529f21a917..0000000000 --- a/.github/workflows/update-test-system.yml +++ /dev/null @@ -1,132 +0,0 @@ -name: Test system deploy - -on: - pull_request_target: - types: [labeled, unlabeled, closed] - - workflow_dispatch: - - -env: - PR_NUMBER: ${{ github.event.number }} - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} - -jobs: - Build-and-update-image: - runs-on: ubuntu-22.04 - - if: ${{ github.event.action == 'labeled' && github.event.label.name == 'test-system-deploy' }} - - steps: - - - name: Check authorization - run: | - allowed_user1=${{ secrets.ALLOWED_USER1_USERNAME }} - allowed_user2=${{ secrets.ALLOWED_USER2_USERNAME }} - allowed_user3=${{ secrets.ALLOWED_USER3_USERNAME }} - allowed_user4=${{ secrets.ALLOWED_USER4_USERNAME }} - allowed_user5=${{ secrets.ALLOWED_USER5_USERNAME }} - allowed_user6=${{ secrets.ALLOWED_USER6_USERNAME }} - allowed_user6=${{ secrets.ALLOWED_USER7_USERNAME }} - - if [[ "${{ github.actor }}" != "$allowed_user1" && \ - "${{ github.actor }}" != "$allowed_user2" && \ - "${{ github.actor }}" != "$allowed_user3" && \ - "${{ github.actor }}" != "$allowed_user4" && \ - "${{ github.actor }}" != "$allowed_user5" && \ - "${{ github.actor }}" != "$allowed_user6" && \ - "${{ github.actor }}" != "$allowed_user7" ]]; then - echo "User not authorized to trigger this workflow" - exit 1 - fi - - - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.ref }} - - # Create Docker Buildx builder with platform configuration - - name: Set up Docker Buildx - run: | - mkdir -p ~/.docker/cli-plugins - curl -SL https://github.com/docker/buildx/releases/download/v0.11.0/buildx-v0.11.0.linux-amd64 -o ~/.docker/cli-plugins/docker-buildx - chmod a+x ~/.docker/cli-plugins/docker-buildx - docker buildx create --name mybuilder --platform linux/arm64,linux/amd64,linux/amd64/v2,linux/riscv64,linux/ppc64le,linux/s390x,linux/386,linux/mips64le,linux/mips64,linux/arm/v7,linux/arm/v6 - docker buildx use mybuilder - - - name: Set DOCKER_CLI_EXPERIMENTAL - run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV - - - name: use mybuilder buildx - run: docker buildx use mybuilder - - - name: Docker Login - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Set SAFE_BRANCH_NAME - run: echo "SAFE_BRANCH_NAME=$(echo ${{ env.BRANCH_NAME }} | tr '/' '-')" >> $GITHUB_ENV - - - name: Build and Push Docker image - uses: docker/build-push-action@v4 - with: - context: . - file: docker/production.Dockerfile - push: true - tags: tooljet/tj-osv:${{ env.SAFE_BRANCH_NAME }} - platforms: linux/amd64 - env: - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - - - name: SSH into GCP VM instance - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.EC2_INSTANCE_IP }} - username: ${{ secrets.GCP_USERNAME }} - key: ${{ secrets.EC2_INSTANCE_SSH_KEY }} - script: | - ls -lah - - # Stop the Docker containers - sudo docker-compose down - - # Remove the existing tooljet/* images - sudo docker images -a | grep 'tooljet/' | awk '{print $3}' | xargs sudo docker rmi -f - - # Check remaining images - sudo docker images - - # Update docker-compose.yml with the new image for tooljet service - sed -i '/^[[:space:]]*tooljet:/,/^[[:space:]]*[^:]*$/ { /^[[:space:]]*image:[[:space:]]*tooljet\/tj-osv/s|\(image:[[:space:]]*\).*|\1tooljet/tj-osv:'"${{ env.SAFE_BRANCH_NAME }}"'| }' docker-compose.yaml - - # Start the Docker containers - cat docker-compose.yaml - sudo docker-compose up -d - - #View containers - sudo docker ps - - - uses: actions/github-script@v6 - with: - script: | - try { - await github.rest.issues.removeLabel({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - name: 'test-system-deploy' - }) - } catch (e) { - console.log(e) - } - - await github.rest.issues.addLabels({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - labels: ['test-system-deployed'] - }) From 538d311182992d3c298b9f1a65c38116203dad90 Mon Sep 17 00:00:00 2001 From: Parth <108089718+parthy007@users.noreply.github.com> Date: Mon, 3 Mar 2025 15:06:40 +0530 Subject: [PATCH 07/42] Fix: Plugins installation and test-connections bugs (#12088) * Add dependency injections to fix plugin install issue * Add id in testConnection URL param --- frontend/src/_services/datasource.service.js | 3 ++- .../components/DataSourceManager/TestConnection.jsx | 1 + server/src/modules/plugins/service.ts | 2 ++ server/src/modules/plugins/util.service.ts | 3 ++- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/frontend/src/_services/datasource.service.js b/frontend/src/_services/datasource.service.js index 7bf81c891e..9f192e5c4a 100644 --- a/frontend/src/_services/datasource.service.js +++ b/frontend/src/_services/datasource.service.js @@ -56,8 +56,9 @@ function deleteDataSource(id) { } function test(body) { + const id = body.dataSourceId; const requestOptions = { method: 'POST', headers: authHeader(), credentials: 'include', body: JSON.stringify(body) }; - return fetch(`${config.apiUrl}/data-sources/test-connection`, requestOptions).then(handleResponse); + return fetch(`${config.apiUrl}/data-sources/${id}/test-connection`, requestOptions).then(handleResponse); } function testSampleDb(body) { diff --git a/frontend/src/modules/dataSources/components/DataSourceManager/TestConnection.jsx b/frontend/src/modules/dataSources/components/DataSourceManager/TestConnection.jsx index fafbf77122..61b6fc391c 100644 --- a/frontend/src/modules/dataSources/components/DataSourceManager/TestConnection.jsx +++ b/frontend/src/modules/dataSources/components/DataSourceManager/TestConnection.jsx @@ -41,6 +41,7 @@ export const TestConnection = ({ options, plugin_id: pluginId, environment_id: environmentId, + dataSourceId: dataSourceId, }; const sampleDbTestConnection = { kind, diff --git a/server/src/modules/plugins/service.ts b/server/src/modules/plugins/service.ts index 0dbe7df500..7b6fd0782b 100644 --- a/server/src/modules/plugins/service.ts +++ b/server/src/modules/plugins/service.ts @@ -9,8 +9,10 @@ import { encode } from 'js-base64'; import { FilesRepository } from '@modules/files/repository'; import { IPluginsService } from './interfaces/IService'; import * as path from 'path'; +import { Injectable } from '@nestjs/common'; const fs = require('fs'); +@Injectable() export class PluginsService implements IPluginsService { constructor( protected readonly pluginsUtilService: PluginsUtilService, diff --git a/server/src/modules/plugins/util.service.ts b/server/src/modules/plugins/util.service.ts index df29684a0e..bbadc30758 100644 --- a/server/src/modules/plugins/util.service.ts +++ b/server/src/modules/plugins/util.service.ts @@ -11,9 +11,10 @@ import * as jszip from 'jszip'; import * as fs from 'fs'; import { CreateFileDto, UpdateFileDto } from '@modules/files/dto'; import { IPluginsUtilService } from './interfaces/IUtilService'; +import { Injectable } from '@nestjs/common'; const jszipInstance = new jszip(); - +@Injectable() export class PluginsUtilService implements IPluginsUtilService { constructor(protected readonly filesRepository: FilesRepository, protected readonly configService: ConfigService) {} async create( From fa9f82f7c4ffba07fc41645dd47c4523c06e1693 Mon Sep 17 00:00:00 2001 From: Parth <108089718+parthy007@users.noreply.github.com> Date: Mon, 3 Mar 2025 16:19:48 +0530 Subject: [PATCH 08/42] Add marketplace plugins to plugins.json (#12093) --- server/src/assets/marketplace/plugins.json | 67 +++++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/server/src/assets/marketplace/plugins.json b/server/src/assets/marketplace/plugins.json index ade3e47272..a312406724 100644 --- a/server/src/assets/marketplace/plugins.json +++ b/server/src/assets/marketplace/plugins.json @@ -50,8 +50,8 @@ "timestamp": "Wed, 17 Jan 2024 20:05:16 GMT" }, { - "name": "aws-lambda", - "description": "Plugin for aws-lambda", + "name": "AWS Lambda", + "description": "Plugin for AWS Lambda", "version": "1.0.0", "id": "aws-lambda", "author": "Tooljet", @@ -130,5 +130,68 @@ "author": "Tooljet", "timestamp": "Mon, 28 Oct 2024 08:08:28 GMT", "tags": ["AI"] + }, + { + "name": "Cohere", + "description": "Integrate with Cohere API to use its AI models for chat and text generation.", + "version": "1.0.0", + "id": "cohere", + "author": "Tooljet", + "timestamp": "Tue, 21 Jan 2025 05:09:30 GMT", + "tags": ["AI"] + }, + { + "name": "Mistral", + "description": "Integrate with Mistral API to use its AI models for chat completion capabilities", + "version": "1.0.0", + "id": "mistral_ai", + "author": "Tooljet", + "timestamp": "Tue, 21 Jan 2025 06:35:01 GMT", + "tags": ["AI"] + }, + { + "name": "Hugging Face", + "description": "Plugin for Hugging Face's Inference API to use listed AI models for text capabilities", + "version": "1.0.0", + "id": "hugging_face", + "author": "Tooljet", + "timestamp": "Thu, 23 Jan 2025 06:44:25 GMT", + "tags": ["AI"] + }, + { + "name": "Gemini", + "description": "Integrate with Gemini AI models for text generation.", + "version": "1.0.0", + "id": "gemini", + "author": "Tooljet", + "timestamp": "Fri, 17 Jan 2025 18:04:48 GMT", + "tags": ["AI"] + }, + { + "name": "Anthropic", + "description": "Integrate with Anthropic API to use Claude AI models.", + "version": "1.0.0", + "id": "anthropic", + "author": "Tooljet", + "timestamp": "Mon, 20 Jan 2025 08:04:46 GMT", + "tags": ["AI"] + }, + { + "name": "Qdrant", + "description": "Plugin for Qdrant APIs", + "version": "1.0.0", + "id": "qdrant", + "author": "Tooljet", + "timestamp": "Tue, 10 Dec 2024 02:11:32 GMT", + "tags": ["AI"] + }, + { + "name": "Weaviate DB", + "description": "Integrate with Weaviate DB to store and query vectors.", + "version": "1.0.0", + "id": "weaviate", + "author": "Tooljet", + "timestamp": "Tue, 21 Jan 2025 16:55:28 GMT", + "tags": ["AI"] } ] From 3202390dd9be429139b45d3e932d5853e063dee3 Mon Sep 17 00:00:00 2001 From: Anantshree Chandola Date: Mon, 3 Mar 2025 16:39:02 +0530 Subject: [PATCH 09/42] fix oauth (#12095) --- .vscode/launch.json | 18 ++++++++++++++++-- server/src/modules/login-configs/repository.ts | 4 ++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 29aec59b0b..fb142c19f3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,7 +11,21 @@ { "type": "node", "request": "launch", - "name": "Server", + "name": "Server (Dev Mode)", + "runtimeExecutable": "npm", + "runtimeArgs": [ + "run", + "start:dev" + ], + "sourceMaps": true, + "cwd": "${workspaceRoot}/server", + "console": "integratedTerminal", + "skipFiles": ["/**"] + }, + { + "type": "node", + "request": "launch", + "name": "Server (Original)", "args": [ "${workspaceFolder}/server/src/main.ts" ], @@ -49,7 +63,7 @@ "remoteRoot": "/app/server", "sourceMaps": true, "skipFiles": ["/**"] - }, + } } ] } diff --git a/server/src/modules/login-configs/repository.ts b/server/src/modules/login-configs/repository.ts index b8c2cc0e14..b088143173 100644 --- a/server/src/modules/login-configs/repository.ts +++ b/server/src/modules/login-configs/repository.ts @@ -1,5 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { DataSource, Repository } from 'typeorm'; +import { DataSource, IsNull, Repository } from 'typeorm'; import { SSOConfigs, SSOType } from '@entities/sso_config.entity'; @Injectable() @@ -12,7 +12,7 @@ export class SSOConfigsRepository extends Repository { } async findInstanceConfigs(): Promise { - return this.find({ where: { organizationId: null } }); + return this.find({ where: { organizationId: IsNull() } }); } async createOrUpdateSSOConfig(configData: Partial): Promise { From ffa92c419e51babcd52d325978200363ad853afb Mon Sep 17 00:00:00 2001 From: Adish M <44204658+adishM98@users.noreply.github.com> Date: Mon, 3 Mar 2025 18:26:28 +0530 Subject: [PATCH 10/42] Feature: Adding Modularisation for AWS Packer build (#12098) --- .github/workflows/packer-build.yml | 35 ++-- deploy/ec2/{ => ce}/.env | 0 deploy/ec2/{ => ce}/nest.service | 0 deploy/ec2/{ => ce}/postgrest.service | 0 deploy/ec2/{ => ce}/setup_app | 2 +- deploy/ec2/{ => ce}/setup_machine.sh | 2 +- .../ec2/{ => ce}/tooljet_ubuntu_focal.pkr.hcl | 0 deploy/ec2/{ => ce}/variables.pkr.hcl | 0 deploy/ec2/ee/.env | 68 +++++++ deploy/ec2/ee/nest.service | 17 ++ deploy/ec2/ee/postgrest.service | 16 ++ deploy/ec2/ee/redis-server.service | 45 +++++ deploy/ec2/ee/setup_app | 175 ++++++++++++++++++ deploy/ec2/ee/setup_machine.sh | 99 ++++++++++ deploy/ec2/ee/tooljet_ubuntu_focal.pkr.hcl | 77 ++++++++ deploy/ec2/ee/variables.pkr.hcl | 33 ++++ 16 files changed, 556 insertions(+), 13 deletions(-) rename deploy/ec2/{ => ce}/.env (100%) rename deploy/ec2/{ => ce}/nest.service (100%) rename deploy/ec2/{ => ce}/postgrest.service (100%) rename deploy/ec2/{ => ce}/setup_app (94%) rename deploy/ec2/{ => ce}/setup_machine.sh (99%) rename deploy/ec2/{ => ce}/tooljet_ubuntu_focal.pkr.hcl (100%) rename deploy/ec2/{ => ce}/variables.pkr.hcl (100%) create mode 100644 deploy/ec2/ee/.env create mode 100644 deploy/ec2/ee/nest.service create mode 100644 deploy/ec2/ee/postgrest.service create mode 100644 deploy/ec2/ee/redis-server.service create mode 100755 deploy/ec2/ee/setup_app create mode 100644 deploy/ec2/ee/setup_machine.sh create mode 100644 deploy/ec2/ee/tooljet_ubuntu_focal.pkr.hcl create mode 100644 deploy/ec2/ee/variables.pkr.hcl diff --git a/.github/workflows/packer-build.yml b/.github/workflows/packer-build.yml index 16a6b656c0..aa60c6444f 100644 --- a/.github/workflows/packer-build.yml +++ b/.github/workflows/packer-build.yml @@ -11,13 +11,16 @@ on: description: "RELEASE_VERSION" jobs: - packer: + packer-ee: runs-on: ubuntu-latest - name: packer + name: packer-ee steps: - - name: Checkout Repository - uses: actions/checkout@v3 + - name: Checkout code to lts-4.0 + if: contains(github.event.release.tag_name, '-ee-lts') + uses: actions/checkout@v2 + with: + ref: refs/heads/lts-4.0 - name: Setting tag if: "${{ github.event.inputs.version != '' }}" @@ -28,7 +31,7 @@ jobs: run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 + uses: aws-actions/configure-aws-credentials@v1-node16 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -40,7 +43,7 @@ jobs: with: command: init target: . - working_directory: deploy/ec2 + working_directory: deploy/ec2/ee # validate templates - name: Validate Template @@ -49,25 +52,35 @@ jobs: command: validate arguments: -syntax-only target: . - working_directory: deploy/ec2 + working_directory: deploy/ec2/ee + + # Echo RENDER_GITHUB_PAT + - name: Set PACKER_GITHUB_PAT + run: echo "PACKER_GITHUB_PAT=${{ secrets.PACKER_GITHUB_PAT}}" >> $GITHUB_ENV + + # Dynamically update setup_machine.sh with PAT + - name: Validate PAT + run: | + sed -i "s|git config --global url."https://x-access-token:CUSTOM_GITHUB_TOKEN@github.com/".insteadOf "https://github.com/"|git config --global url."https://x-access-token:${ secrets.CUSTOM_GITHUB_TOKEN }@github.com/".insteadOf "https://github.com/"|g" ./deploy/ec2/ee/setup_machine.sh # build artifact - name: Build Artifact uses: hashicorp/packer-github-actions@master with: command: build + #The the below argument is specific for building EE AMI image arguments: -color=false -on-error=abort -var ami_name=tooljet_${{ env.RELEASE_VERSION }}.ubuntu_focal target: . - working_directory: deploy/ec2 + working_directory: deploy/ec2/ee env: PACKER_LOG: 1 - name: Send Slack Notification run: | if [[ "${{ job.status }}" == "success" ]]; then - message="Job '${{ env.JOB_NAME }}' succeeded! AMI = tooljet_${{ env.RELEASE_VERSION }}.ubuntu_focal" + message="ToolJet enterprise AWS AMI published:\\n\`tooljet_${{ env.RELEASE_VERSION }}.ubuntu_focal\`" else - message="Job '${{ env.JOB_NAME }}' failed! AMI = tooljet_${{ env.RELEASE_VERSION }}.ubuntu_focal" + message="ToolJet enterprise AWS AMI release failed! \\n\`tooljet_${{ env.RELEASE_VERSION }}.ubuntu_focal\`" fi - curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$message\"}" ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + curl -X POST -H 'Content-type: application/json' --data "{\"text\":\"$message\"}" ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/deploy/ec2/.env b/deploy/ec2/ce/.env similarity index 100% rename from deploy/ec2/.env rename to deploy/ec2/ce/.env diff --git a/deploy/ec2/nest.service b/deploy/ec2/ce/nest.service similarity index 100% rename from deploy/ec2/nest.service rename to deploy/ec2/ce/nest.service diff --git a/deploy/ec2/postgrest.service b/deploy/ec2/ce/postgrest.service similarity index 100% rename from deploy/ec2/postgrest.service rename to deploy/ec2/ce/postgrest.service diff --git a/deploy/ec2/setup_app b/deploy/ec2/ce/setup_app similarity index 94% rename from deploy/ec2/setup_app rename to deploy/ec2/ce/setup_app index bc106fc4b4..b07a1299d5 100755 --- a/deploy/ec2/setup_app +++ b/deploy/ec2/ce/setup_app @@ -35,7 +35,7 @@ then fi fi -npm --prefix server run db:setup:prod +TOOLJET_EDTION=ce npm --prefix server run db:setup:prod if sudo systemctl start nest then diff --git a/deploy/ec2/setup_machine.sh b/deploy/ec2/ce/setup_machine.sh similarity index 99% rename from deploy/ec2/setup_machine.sh rename to deploy/ec2/ce/setup_machine.sh index b5c3f632df..bb65d1c11c 100644 --- a/deploy/ec2/setup_machine.sh +++ b/deploy/ec2/ce/setup_machine.sh @@ -78,4 +78,4 @@ npm install -g npm@9.8.1 # Building ToolJet app npm install -g @nestjs/cli -npm run build +TOOLJET_EDTION=ce npm run build diff --git a/deploy/ec2/tooljet_ubuntu_focal.pkr.hcl b/deploy/ec2/ce/tooljet_ubuntu_focal.pkr.hcl similarity index 100% rename from deploy/ec2/tooljet_ubuntu_focal.pkr.hcl rename to deploy/ec2/ce/tooljet_ubuntu_focal.pkr.hcl diff --git a/deploy/ec2/variables.pkr.hcl b/deploy/ec2/ce/variables.pkr.hcl similarity index 100% rename from deploy/ec2/variables.pkr.hcl rename to deploy/ec2/ce/variables.pkr.hcl diff --git a/deploy/ec2/ee/.env b/deploy/ec2/ee/.env new file mode 100644 index 0000000000..c28115183f --- /dev/null +++ b/deploy/ec2/ee/.env @@ -0,0 +1,68 @@ +# https://docs.tooljet.io/docs/setup/env-vars +TOOLJET_HOST=http://localhost +LOCKBOX_MASTER_KEY= +SECRET_KEY_BASE= +PG_USER= +PG_HOST= +PG_PASS= +PG_DB=tooljet_prod +ORM_LOGGING=true +NODE_ENV=production +DEPLOYMENT_PLATFORM=ec2 + +# ToolJet Database +TOOLJET_DB=tooljet_db +TOOLJET_DB_USER= +TOOLJET_DB_HOST= +TOOLJET_DB_PASS= +PGRST_HOST=localhost:3001 +PGRST_SERVER_PORT=3001 +PGRST_JWT_SECRET= +PGRST_DB_URI= +PGRST_DB_PRE_CONFIG=postgrest.pre_config + + +#Redis +REDIS_HOST=localhost +REDIS_PORT=6379 +REDIS_USER=default +REDIS_PASSWORD= + +# Checks every 24 hours to see if a new version of ToolJet is available +# (Enabled by default. Set 0 to disable) +CHECK_FOR_UPDATES= + +# Checks every 24 hours to update app telemetry data to ToolJet hub. +# (Telemetry is enabled by default. Set value to true to disable.) +# DISABLE_APP_TELEMETRY=false + +GOOGLE_CLIENT_ID= +GOOGLE_CLIENT_SECRET= + +# EMAIL CONFIGURATION +DEFAULT_FROM_EMAIL=hello@tooljet.io +SMTP_USERNAME= +SMTP_PASSWORD= +SMTP_DOMAIN= +SMTP_PORT= + +# DISABLE USER SIGNUPS (true or false). Default: true +DISABLE_SIGNUPS= + +# OBSERVABILITY +APM_VENDOR= +SENTRY_DNS= +SENTRY_DEBUG= + +# FEATURE TOGGLE +COMMENT_FEATURE_ENABLE=true +ENABLE_MULTIPLAYER_EDITING=true +ENABLE_MARKETPLACE_FEATURE=true + +#SSO +SSO_DISABLE_SIGNUP= +SSO_RESTRICTED_DOMAIN= +SSO_GOOGLE_OAUTH2_CLIENT_ID= +SSO_GIT_OAUTH2_CLIENT_ID= +SSO_GIT_OAUTH2_CLIENT_SECRET= +SSO_GIT_OAUTH2_HOST= diff --git a/deploy/ec2/ee/nest.service b/deploy/ec2/ee/nest.service new file mode 100644 index 0000000000..61a1127e2f --- /dev/null +++ b/deploy/ec2/ee/nest.service @@ -0,0 +1,17 @@ +[Unit] +Description=Nest Server +After=network.target + +[Service] +Type=simple +User=ubuntu + +WorkingDirectory=/home/ubuntu/app +Environment="NODE_ENV=production" +EnvironmentFile=/home/ubuntu/app/.env +RestartSec=1 +ExecStart=/usr/bin/npm --prefix /home/ubuntu/app run start:prod +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/deploy/ec2/ee/postgrest.service b/deploy/ec2/ee/postgrest.service new file mode 100644 index 0000000000..806c6c8ee1 --- /dev/null +++ b/deploy/ec2/ee/postgrest.service @@ -0,0 +1,16 @@ +[Unit] +Description=PostgREST Server +After=network.target + +[Service] +Type=simple +User=ubuntu + +WorkingDirectory=/bin +EnvironmentFile=/home/ubuntu/app/.env +RestartSec=1 +ExecStart=/bin/postgrest +Restart=always + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/deploy/ec2/ee/redis-server.service b/deploy/ec2/ee/redis-server.service new file mode 100644 index 0000000000..c1a83a7581 --- /dev/null +++ b/deploy/ec2/ee/redis-server.service @@ -0,0 +1,45 @@ +[Unit] +Description=Advanced key-value store +After=network.target +Documentation=http://redis.io/documentation, man:redis-server(1) + +[Service] +Type=forking +ExecStart=/usr/bin/redis-server /etc/redis/redis.conf +PIDFile=/run/redis/redis-server.pid +TimeoutStopSec=0 +Restart=always +User=redis +Group=redis +RuntimeDirectory=redis +RuntimeDirectoryMode=2755 + +UMask=007 +PrivateTmp=yes +LimitNOFILE=65535 +PrivateDevices=yes +ProtectHome=yes +ReadOnlyDirectories=/ +ReadWritePaths=-/var/lib/redis +ReadWritePaths=-/var/log/redis +ReadWritePaths=-/var/run/redis + +NoNewPrivileges=true +CapabilityBoundingSet=CAP_SETGID CAP_SETUID CAP_SYS_RESOURCE +MemoryDenyWriteExecute=true +ProtectKernelModules=true +ProtectKernelTunables=true +ProtectControlGroups=true +RestrictRealtime=true +RestrictNamespaces=true +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX + +# redis-server can write to its own config file when in cluster mode so we +# permit writing there by default. If you are not using this feature, it is +# recommended that you replace the following lines with "ProtectSystem=full". +ProtectSystem=true +ReadWriteDirectories=-/etc/redis + +[Install] +WantedBy=multi-user.target +Alias=redis-server.service \ No newline at end of file diff --git a/deploy/ec2/ee/setup_app b/deploy/ec2/ee/setup_app new file mode 100755 index 0000000000..3dad6ebeef --- /dev/null +++ b/deploy/ec2/ee/setup_app @@ -0,0 +1,175 @@ +#!/bin/bash + +# Load the .env file +source .env + +# Check if LOCKBOX_MASTER_KEY is present or empty +if [[ -z "$LOCKBOX_MASTER_KEY" ]]; then + # Generate LOCKBOX_MASTER_KEY + LOCKBOX_MASTER_KEY=$(openssl rand -hex 32) + + # Update .env file + awk -v key="$LOCKBOX_MASTER_KEY" ' + BEGIN { FS=OFS="=" } + /^LOCKBOX_MASTER_KEY=/ { $2=key; found=1 } + 1 + END { if (!found) print "LOCKBOX_MASTER_KEY="key } + ' .env > temp.env && mv temp.env .env + + echo "Generated a secure master key for the lockbox" +else + echo "The lockbox master key already exists." +fi + +# Check if SECRET_KEY_BASE is present or empty +if [[ -z "$SECRET_KEY_BASE" ]]; then + # Generate SECRET_KEY_BASE + SECRET_KEY_BASE=$(openssl rand -hex 64) + + # Update .env file + awk -v key="$SECRET_KEY_BASE" ' + BEGIN { FS=OFS="=" } + /^SECRET_KEY_BASE=/ { $2=key; found=1 } + 1 + END { if (!found) print "SECRET_KEY_BASE="key } + ' .env > temp.env && mv temp.env .env + + echo "Created a secret key for secure operations." +else + echo "The secret key base is already in place." +fi + +# Check if PGRST_JWT_SECRET is present or empty +if [[ -z "$PGRST_JWT_SECRET" ]]; then + # Generate PGRST_JWT_SECRET + PGRST_JWT_SECRET=$(openssl rand -hex 32) + + # Update .env file + awk -v key="$PGRST_JWT_SECRET" ' + BEGIN { FS=OFS="=" } + /^PGRST_JWT_SECRET=/ { $2=key; found=1 } + 1 + END { if (!found) print "PGRST_JWT_SECRET="key } + ' .env > temp.env && mv temp.env .env + + echo "Generated a unique secret for PGRST authentication." +else + echo "The PGRST JWT secret is already generated and in place." +fi + +# Function to generate a random password +generate_password() { + openssl rand -base64 12 | tr -d '/+' | cut -c1-16 +} + +# Check if PG_USER, PG_HOST, PG_PASS, PG_DB are present or empty +if [[ -z "$PG_USER" ]] || [[ -z "$PG_HOST" ]] || [[ -z "$PG_PASS" ]] || [[ -z "$PG_DB" ]]; then + # Prompt user for values + read -p "Enter PostgreSQL database username: " PG_USER + read -p "Enter PostgreSQL database hostname: " PG_HOST + read -p "Enter PostgreSQL database password: " PG_PASS + read -p "Enter PostgreSQL database name: " PG_DB + + # Update .env file + awk -v pg_user="$PG_USER" -v pg_host="$PG_HOST" -v pg_pass="$PG_PASS" -v pg_db="$PG_DB" ' + BEGIN { FS=OFS="=" } + /^PG_USER=/ { $2=pg_user; found=1 } + /^PG_HOST=/ { $2=pg_host; found=1 } + /^PG_PASS=/ { $2=pg_pass; found=1 } + /^PG_DB=/ { $2=pg_db; found=1 } + 1 + END { + if (!found) { + print "PG_USER="pg_user + print "PG_HOST="pg_host + print "PG_PASS="pg_pass + print "PG_DB="pg_db + } + } + ' .env > temp.env && mv temp.env .env + + echo "Successfully updated postgresql database values .env file" +fi + +# Copy values from PG to TOOLJET_DB +TOOLJET_DB_USER=$PG_USER +TOOLJET_DB_HOST=$PG_HOST +TOOLJET_DB_PASS=$PG_PASS + +# Update .env file for TOOLJET_DB +awk -v tj_user="$TOOLJET_DB_USER" -v tj_host="$TOOLJET_DB_HOST" -v tj_pass="$TOOLJET_DB_PASS" ' + BEGIN { FS=OFS="=" } + /^TOOLJET_DB_USER=/ { $2=tj_user; found=1 } + /^TOOLJET_DB_HOST=/ { $2=tj_host; found=1 } + /^TOOLJET_DB_PASS=/ { $2=tj_pass; found=1 } + 1 + END { if (!found) print "TOOLJET_DB_USER="tj_user ORS "TOOLJET_DB_HOST="tj_host ORS "TOOLJET_DB_PASS="tj_pass } +' .env > temp.env && mv temp.env .env + +echo "Successfully updated tooljet database values in the .env file" + +# Construct PGRST_DB_URI with user-provided values +PGRST_DB_URI="postgres://$PG_USER:$PG_PASS@$PG_HOST/tooljet_db" + +# Update .env file for PGRST_DB_URI +awk -v uri="$PGRST_DB_URI" ' + BEGIN { FS=OFS="=" } + /^PGRST_DB_URI=/ { $2=uri; found=1 } + 1 + END { if (!found) print "PGRST_DB_URI="uri } +' .env > temp.env && mv temp.env .env + +echo "Successfully updated PGRST database URI" + + +if [[ -z $PG_USER || -z $PG_PASS || -z $PG_HOST ]] +then + echo "Please set the required PG_USER, PG_PASS, and PG_HOST values within the .env file" + exit 1 +fi + +export $(grep -v '^#' .env | xargs) + +if psql -d postgresql://$PG_USER:$PG_PASS@$PG_HOST/postgres -c 'select now()' > /dev/null 2>&1 +then + echo "Successfully pinged the database!"; +else + echo "Can't connect to the database. Kindly check the credenials provided in the .env file!" + exit 1 +fi + +if sudo systemctl start redis-server && sudo systemctl enable redis-server +then + echo "Successfully started Redis!" +else + echo "Failed to start and enable Redis" +fi + +if sudo -E systemctl start openresty +then + echo "Successfully started reverse proxy!" +else + echo "Failed to start reverse proxy" + exit 1 +fi + +if sudo -E systemctl start postgrest +then + echo "Successfully started PostgREST server!" +else + echo "Failed to start PostgREST server" + exit 1 +fi + +TOOLJET_EDTION=ee npm --prefix server run db:setup:prod + +if sudo -E systemctl start nest +then + echo "The app will be served at ${TOOLJET_HOST}" +else + echo "Failed to start the server!" + exit 1 +fi + +sudo systemctl restart nest +sudo -E systemctl restart postgrest \ No newline at end of file diff --git a/deploy/ec2/ee/setup_machine.sh b/deploy/ec2/ee/setup_machine.sh new file mode 100644 index 0000000000..23aa5bf911 --- /dev/null +++ b/deploy/ec2/ee/setup_machine.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +set -e +# Setup prerequisite dependencies +sudo apt-get update +sudo apt-get -y install --no-install-recommends wget gnupg ca-certificates apt-utils git curl postgresql-client +curl https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash +export NVM_DIR="$HOME/.nvm" +[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" +nvm install 18.18.2 +sudo ln -s "$(which node)" /usr/bin/node +sudo ln -s "$(which npm)" /usr/bin/npm + +sudo npm i -g npm@9.8.1 + +# Setup openresty +wget -O - https://openresty.org/package/pubkey.gpg | sudo apt-key add - +echo "deb http://openresty.org/package/ubuntu bionic main" > openresty.list +sudo mv openresty.list /etc/apt/sources.list.d/ +sudo apt-get update +sudo apt-get -y install --no-install-recommends openresty +sudo apt-get install -y curl g++ gcc autoconf automake bison libc6-dev \ + libffi-dev libgdbm-dev libncurses5-dev libsqlite3-dev libtool \ + libyaml-dev make pkg-config sqlite3 zlib1g-dev libgmp-dev \ + libreadline-dev libssl-dev libmysqlclient-dev build-essential \ + freetds-dev libpq-dev +sudo apt-get install -y luarocks +sudo luarocks install lua-resty-auto-ssl +sudo mkdir /etc/resty-auto-ssl /var/log/openresty /etc/fallback-certs +sudo chown -R www-data:www-data /etc/resty-auto-ssl + +# Oracle db client library setup +sudo apt install -y libaio1 +curl -o instantclient-basiclite.zip https://download.oracle.com/otn_software/linux/instantclient/instantclient-basiclite-linuxx64.zip -SL && \ +curl -o instantclient-basiclite-11.zip https://tooljet-plugins-production.s3.us-east-2.amazonaws.com/marketplace-assets/oracledb/instantclients/instantclient-basiclite-linux.x64-11.2.0.4.0.zip -SL && \ + unzip instantclient-basiclite.zip && \ + unzip instantclient-basiclite-11.zip && \ + sudo mkdir -p /usr/lib/instantclient && sudo mv instantclient*/ /usr/lib/instantclient && \ + rm instantclient-basiclite.zip && \ + rm instantclient-basiclite-11.zip && \ + echo /usr/lib/instantclient/* | sudo tee /etc/ld.so.conf.d/oracle-instantclient.conf > /dev/null && sudo ldconfig +# Set the Instant Client library paths +export LD_LIBRARY_PATH="/usr/lib/instantclient/instantclient_11_2:/usr/lib/instantclient/instantclient_21_10${LD_LIBRARY_PATH}" + +# Gen fallback certs +sudo openssl rand -out /home/ubuntu/.rnd -hex 256 +sudo chown www-data:www-data /home/ubuntu/.rnd +sudo openssl req -new -newkey rsa:2048 -days 3650 -nodes -x509 \ + -subj '/CN=sni-support-required-for-valid-ssl' \ + -keyout /etc/fallback-certs/resty-auto-ssl-fallback.key \ + -out /etc/fallback-certs/resty-auto-ssl-fallback.crt + +# Setup nginx config +export SERVER_HOST="${SERVER_HOST:=localhost}" +export SERVER_USER="${SERVER_USER:=www-data}" +VARS_TO_SUBSTITUTE='$SERVER_HOST:$SERVER_USER' +envsubst "${VARS_TO_SUBSTITUTE}" < /tmp/nginx.conf > /tmp/nginx-substituted.conf +sudo cp /tmp/nginx-substituted.conf /usr/local/openresty/nginx/conf/nginx.conf + +# Download and setup postgrest binary +curl -OL https://github.com/PostgREST/postgrest/releases/download/v12.0.2/postgrest-v12.0.2-linux-static-x64.tar.xz +tar xJf postgrest-v12.0.2-linux-static-x64.tar.xz +sudo mv ./postgrest /bin/postgrest +sudo rm postgrest-v12.0.2-linux-static-x64.tar.xz + +# Add the Redis APT repository +sudo add-apt-repository ppa:redislabs/redis -y + +# Install redis +sudo apt-get update +sudo apt-get install redis-server -y + +# Setup app, postgrest and redis as systemd service +sudo cp /tmp/nest.service /lib/systemd/system/nest.service +sudo cp /tmp/postgrest.service /lib/systemd/system/postgrest.service +sudo cp /tmp/redis-server.service /lib/systemd/system/redis-server.service + +# Start and enable Redis service +sudo systemctl daemon-reload + +# Setup app directory +mkdir -p ~/app + +git config --global url."https://x-access-token:CUSTOM_GITHUB_TOKEN@github.com/".insteadOf "https://github.com/" + +#The below url will be edited dynamically when actions is triggered +git clone -b main https://github.com/ToolJet/ToolJet.git ~/app && cd ~/app +git submodule update --init --recursive +git submodule foreach 'git checkout main || true' + +mv /tmp/.env ~/app/.env +mv /tmp/setup_app ~/app/setup_app +sudo chmod +x ~/app/setup_app + +npm install -g npm@9.8.1 + +# Building ToolJet app +npm install -g @nestjs/cli +TOOLJET_EDTION=ee npm run build \ No newline at end of file diff --git a/deploy/ec2/ee/tooljet_ubuntu_focal.pkr.hcl b/deploy/ec2/ee/tooljet_ubuntu_focal.pkr.hcl new file mode 100644 index 0000000000..144803d55c --- /dev/null +++ b/deploy/ec2/ee/tooljet_ubuntu_focal.pkr.hcl @@ -0,0 +1,77 @@ +packer { + required_plugins { + amazon = { + version = ">= 0.0.1" + source = "github.com/hashicorp/amazon" + } + } +} + +source "amazon-ebs" "ubuntu" { + ami_name = "${var.ami_name}" + instance_type = "${var.instance_type}" + region = "${var.ami_region}" + ami_regions = "${var.ami_regions}" + ami_groups = "${var.ami_groups}" + + source_ami_filter { + filters = { + name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["099720109477"] + } + ssh_username = "ubuntu" + ssh_clear_authorized_keys = "true" + shutdown_behavior = "terminate" + force_delete_snapshot = "true" + + launch_block_device_mappings { + device_name = "/dev/sda1" + volume_size = 10 + delete_on_termination = true + } + +} + +build { + sources = [ + "source.amazon-ebs.ubuntu" + ] + + provisioner "file" { + source = "nest.service" + destination = "/tmp/nest.service" + } + + provisioner "file" { + source = "../../frontend/config/nginx.conf.template" + destination = "/tmp/nginx.conf" + } + + provisioner "file" { + source = ".env" + destination = "/tmp/.env" + } + + provisioner "file" { + source = "setup_app" + destination = "/tmp/setup_app" + } + + provisioner "file" { + source = "postgrest.service" + destination = "/tmp/postgrest.service" + } + + provisioner "file" { + source = "redis-server.service" + destination = "/tmp/redis-server.service" + } + + provisioner "shell" { + script = "setup_machine.sh" + } +} diff --git a/deploy/ec2/ee/variables.pkr.hcl b/deploy/ec2/ee/variables.pkr.hcl new file mode 100644 index 0000000000..39dcdfd3cd --- /dev/null +++ b/deploy/ec2/ee/variables.pkr.hcl @@ -0,0 +1,33 @@ +variable "ami_name" { + type = string +} + +variable "instance_type" { + type = string + default = "t2.medium" +} + +variable "ami_region" { + type = string + default = "us-west-2" +} + +variable "ami_groups" { + type = list(string) + default = ["all"] +} + +variable "ami_regions" { + type = list(string) + default = ["us-west-1","us-east-1", "us-east-2", "eu-central-1", "ap-northeast-1", "ca-central-1"] +} + +variable "PACKER_BUILDER_TYPE" { + type = string + default = "amazon-ebs" +} + +variable "PACKER_BUILD_NAME" { + type = string + default = "ubuntu" +} From 710798d00072e9b8869b2250403f5165b33a2320 Mon Sep 17 00:00:00 2001 From: Parth <108089718+parthy007@users.noreply.github.com> Date: Mon, 3 Mar 2025 18:44:36 +0530 Subject: [PATCH 11/42] Add AI plugins (#12097) --- marketplace/plugins/anthropic/.gitignore | 5 + marketplace/plugins/anthropic/README.md | 4 + .../plugins/anthropic/__tests__/index.js | 7 + marketplace/plugins/anthropic/lib/icon.svg | 4 + marketplace/plugins/anthropic/lib/index.ts | 93 +++ .../plugins/anthropic/lib/manifest.json | 34 + .../plugins/anthropic/lib/operations.json | 246 +++++++ .../plugins/anthropic/lib/query_operations.ts | 60 ++ marketplace/plugins/anthropic/lib/types.ts | 36 + marketplace/plugins/anthropic/package.json | 27 + marketplace/plugins/anthropic/tsconfig.json | 11 + marketplace/plugins/cohere/.gitignore | 5 + marketplace/plugins/cohere/README.md | 4 + marketplace/plugins/cohere/__tests__/index.js | 7 + marketplace/plugins/cohere/lib/icon.svg | 6 + marketplace/plugins/cohere/lib/index.ts | 90 +++ marketplace/plugins/cohere/lib/manifest.json | 34 + .../plugins/cohere/lib/operations.json | 651 ++++++++++++++++++ .../plugins/cohere/lib/query_operations.ts | 52 ++ marketplace/plugins/cohere/lib/types.ts | 15 + marketplace/plugins/cohere/package.json | 27 + marketplace/plugins/cohere/tsconfig.json | 11 + marketplace/plugins/gemini/.gitignore | 5 + marketplace/plugins/gemini/README.md | 4 + marketplace/plugins/gemini/__tests__/index.js | 7 + marketplace/plugins/gemini/lib/icon.svg | 10 + marketplace/plugins/gemini/lib/index.ts | 95 +++ marketplace/plugins/gemini/lib/manifest.json | 34 + .../plugins/gemini/lib/operations.json | 386 +++++++++++ .../plugins/gemini/lib/query_operations.ts | 82 +++ marketplace/plugins/gemini/lib/types.ts | 19 + marketplace/plugins/gemini/package.json | 27 + marketplace/plugins/gemini/tsconfig.json | 11 + marketplace/plugins/hugging_face/.gitignore | 5 + marketplace/plugins/hugging_face/README.md | 4 + .../plugins/hugging_face/__tests__/index.js | 7 + marketplace/plugins/hugging_face/lib/icon.svg | 11 + marketplace/plugins/hugging_face/lib/index.ts | 73 ++ .../plugins/hugging_face/lib/manifest.json | 54 ++ .../plugins/hugging_face/lib/operations.json | 90 +++ .../hugging_face/lib/query_operation.ts | 31 + marketplace/plugins/hugging_face/lib/types.ts | 14 + marketplace/plugins/hugging_face/package.json | 26 + .../plugins/hugging_face/tsconfig.json | 11 + marketplace/plugins/mistral_ai/.gitignore | 5 + marketplace/plugins/mistral_ai/README.md | 4 + .../plugins/mistral_ai/__tests__/index.js | 7 + marketplace/plugins/mistral_ai/lib/icon.svg | 32 + marketplace/plugins/mistral_ai/lib/index.ts | 111 +++ .../plugins/mistral_ai/lib/manifest.json | 35 + .../plugins/mistral_ai/lib/operations.json | 155 +++++ marketplace/plugins/mistral_ai/lib/types.ts | 22 + marketplace/plugins/mistral_ai/package.json | 27 + marketplace/plugins/mistral_ai/tsconfig.json | 11 + marketplace/plugins/qdrant/.gitignore | 5 + marketplace/plugins/qdrant/README.md | 4 + marketplace/plugins/qdrant/lib/icon.svg | 22 + marketplace/plugins/qdrant/lib/index.ts | 96 +++ marketplace/plugins/qdrant/lib/manifest.json | 47 ++ .../plugins/qdrant/lib/operations.json | 166 +++++ .../plugins/qdrant/lib/query_operations.ts | 85 +++ marketplace/plugins/qdrant/lib/types.ts | 26 + marketplace/plugins/qdrant/package.json | 27 + marketplace/plugins/qdrant/tsconfig.json | 11 + marketplace/plugins/weaviate/.gitignore | 5 + marketplace/plugins/weaviate/README.md | 4 + .../plugins/weaviate/__tests__/index.js | 7 + marketplace/plugins/weaviate/lib/icon.svg | 49 ++ marketplace/plugins/weaviate/lib/index.ts | 69 ++ .../plugins/weaviate/lib/manifest.json | 76 ++ .../plugins/weaviate/lib/operations.json | 506 ++++++++++++++ .../plugins/weaviate/lib/query_operations.ts | 239 +++++++ marketplace/plugins/weaviate/lib/types.ts | 64 ++ marketplace/plugins/weaviate/package.json | 27 + marketplace/plugins/weaviate/tsconfig.json | 11 + 75 files changed, 4390 insertions(+) create mode 100644 marketplace/plugins/anthropic/.gitignore create mode 100644 marketplace/plugins/anthropic/README.md create mode 100644 marketplace/plugins/anthropic/__tests__/index.js create mode 100644 marketplace/plugins/anthropic/lib/icon.svg create mode 100644 marketplace/plugins/anthropic/lib/index.ts create mode 100644 marketplace/plugins/anthropic/lib/manifest.json create mode 100644 marketplace/plugins/anthropic/lib/operations.json create mode 100644 marketplace/plugins/anthropic/lib/query_operations.ts create mode 100644 marketplace/plugins/anthropic/lib/types.ts create mode 100644 marketplace/plugins/anthropic/package.json create mode 100644 marketplace/plugins/anthropic/tsconfig.json create mode 100644 marketplace/plugins/cohere/.gitignore create mode 100644 marketplace/plugins/cohere/README.md create mode 100644 marketplace/plugins/cohere/__tests__/index.js create mode 100644 marketplace/plugins/cohere/lib/icon.svg create mode 100644 marketplace/plugins/cohere/lib/index.ts create mode 100644 marketplace/plugins/cohere/lib/manifest.json create mode 100644 marketplace/plugins/cohere/lib/operations.json create mode 100644 marketplace/plugins/cohere/lib/query_operations.ts create mode 100644 marketplace/plugins/cohere/lib/types.ts create mode 100644 marketplace/plugins/cohere/package.json create mode 100644 marketplace/plugins/cohere/tsconfig.json create mode 100644 marketplace/plugins/gemini/.gitignore create mode 100644 marketplace/plugins/gemini/README.md create mode 100644 marketplace/plugins/gemini/__tests__/index.js create mode 100644 marketplace/plugins/gemini/lib/icon.svg create mode 100644 marketplace/plugins/gemini/lib/index.ts create mode 100644 marketplace/plugins/gemini/lib/manifest.json create mode 100644 marketplace/plugins/gemini/lib/operations.json create mode 100644 marketplace/plugins/gemini/lib/query_operations.ts create mode 100644 marketplace/plugins/gemini/lib/types.ts create mode 100644 marketplace/plugins/gemini/package.json create mode 100644 marketplace/plugins/gemini/tsconfig.json create mode 100644 marketplace/plugins/hugging_face/.gitignore create mode 100644 marketplace/plugins/hugging_face/README.md create mode 100644 marketplace/plugins/hugging_face/__tests__/index.js create mode 100644 marketplace/plugins/hugging_face/lib/icon.svg create mode 100644 marketplace/plugins/hugging_face/lib/index.ts create mode 100644 marketplace/plugins/hugging_face/lib/manifest.json create mode 100644 marketplace/plugins/hugging_face/lib/operations.json create mode 100644 marketplace/plugins/hugging_face/lib/query_operation.ts create mode 100644 marketplace/plugins/hugging_face/lib/types.ts create mode 100644 marketplace/plugins/hugging_face/package.json create mode 100644 marketplace/plugins/hugging_face/tsconfig.json create mode 100644 marketplace/plugins/mistral_ai/.gitignore create mode 100644 marketplace/plugins/mistral_ai/README.md create mode 100644 marketplace/plugins/mistral_ai/__tests__/index.js create mode 100644 marketplace/plugins/mistral_ai/lib/icon.svg create mode 100644 marketplace/plugins/mistral_ai/lib/index.ts create mode 100644 marketplace/plugins/mistral_ai/lib/manifest.json create mode 100644 marketplace/plugins/mistral_ai/lib/operations.json create mode 100644 marketplace/plugins/mistral_ai/lib/types.ts create mode 100644 marketplace/plugins/mistral_ai/package.json create mode 100644 marketplace/plugins/mistral_ai/tsconfig.json create mode 100644 marketplace/plugins/qdrant/.gitignore create mode 100644 marketplace/plugins/qdrant/README.md create mode 100644 marketplace/plugins/qdrant/lib/icon.svg create mode 100644 marketplace/plugins/qdrant/lib/index.ts create mode 100644 marketplace/plugins/qdrant/lib/manifest.json create mode 100644 marketplace/plugins/qdrant/lib/operations.json create mode 100644 marketplace/plugins/qdrant/lib/query_operations.ts create mode 100644 marketplace/plugins/qdrant/lib/types.ts create mode 100644 marketplace/plugins/qdrant/package.json create mode 100644 marketplace/plugins/qdrant/tsconfig.json create mode 100644 marketplace/plugins/weaviate/.gitignore create mode 100644 marketplace/plugins/weaviate/README.md create mode 100644 marketplace/plugins/weaviate/__tests__/index.js create mode 100644 marketplace/plugins/weaviate/lib/icon.svg create mode 100644 marketplace/plugins/weaviate/lib/index.ts create mode 100644 marketplace/plugins/weaviate/lib/manifest.json create mode 100644 marketplace/plugins/weaviate/lib/operations.json create mode 100644 marketplace/plugins/weaviate/lib/query_operations.ts create mode 100644 marketplace/plugins/weaviate/lib/types.ts create mode 100644 marketplace/plugins/weaviate/package.json create mode 100644 marketplace/plugins/weaviate/tsconfig.json diff --git a/marketplace/plugins/anthropic/.gitignore b/marketplace/plugins/anthropic/.gitignore new file mode 100644 index 0000000000..23e6609462 --- /dev/null +++ b/marketplace/plugins/anthropic/.gitignore @@ -0,0 +1,5 @@ +node_modules +lib/*.d.* +lib/*.js +lib/*.js.map +dist/* \ No newline at end of file diff --git a/marketplace/plugins/anthropic/README.md b/marketplace/plugins/anthropic/README.md new file mode 100644 index 0000000000..5ac48868eb --- /dev/null +++ b/marketplace/plugins/anthropic/README.md @@ -0,0 +1,4 @@ + +# Anthropic + +Documentation on: https://docs.tooljet.com/docs/data-sources/anthropic \ No newline at end of file diff --git a/marketplace/plugins/anthropic/__tests__/index.js b/marketplace/plugins/anthropic/__tests__/index.js new file mode 100644 index 0000000000..90c405bc03 --- /dev/null +++ b/marketplace/plugins/anthropic/__tests__/index.js @@ -0,0 +1,7 @@ +'use strict'; + +const anthropic = require('../lib'); + +describe('anthropic', () => { + it.todo('needs tests'); +}); diff --git a/marketplace/plugins/anthropic/lib/icon.svg b/marketplace/plugins/anthropic/lib/icon.svg new file mode 100644 index 0000000000..db550ee3d7 --- /dev/null +++ b/marketplace/plugins/anthropic/lib/icon.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/marketplace/plugins/anthropic/lib/index.ts b/marketplace/plugins/anthropic/lib/index.ts new file mode 100644 index 0000000000..c9ffc33fa0 --- /dev/null +++ b/marketplace/plugins/anthropic/lib/index.ts @@ -0,0 +1,93 @@ +import { QueryError, QueryResult, QueryService, ConnectionTestResult } from '@tooljet-marketplace/common'; +import { SourceOptions, QueryOptions, Operation } from './types'; +import { Anthropic } from '@anthropic-ai/sdk'; +import { getChatCompletion, /*getVisionCompletion */} from './query_operations'; + +export default class AnthropicService implements QueryService { + async run(sourceOptions: SourceOptions, queryOptions: QueryOptions, dataSourceId: string): Promise { + const operation: Operation = queryOptions.operation; + const anthropicClient: any = await this.getConnection(sourceOptions); + let result = {}; + + try { + switch (operation) { + case Operation.Chat: + result = await getChatCompletion(anthropicClient, queryOptions); + break; + + /*case Operation.Vision: + result = await getVisionCompletion(anthropicClient, queryOptions); + break;*/ + + default: + throw new QueryError('Query could not be completed', 'Invalid operation', {}); + } + } catch (error: any) { + let errorMessage = 'Unknown error occured'; + let errorDetails: any = {}; + if (error) { + try { + errorMessage = error?.error?.error?.message || 'Unknown error'; + errorDetails = {requestId: error?.request_id, + errorType: error?.error?.error?.type, + statusCode: error?.status,} + } catch (parseError) { + console.error('Failed to parse Anthropic error response:', parseError); + } + } + throw new QueryError('Query could not be completed', errorMessage, errorDetails); + } + return { + status: 'ok', + data: result, + }; + } + + async testConnection(sourceOptions: SourceOptions): Promise { + const { apiKey } = sourceOptions; + + if (!apiKey) { + throw new QueryError('Connection could not be established', 'API key is missing', {}); + } + + const anthropicClient = new Anthropic({ apiKey }); + + try { + await anthropicClient.messages.create({ + model: 'claude-3-5-sonnet-20241022', + max_tokens: 1, + messages: [{ role: 'user', content: 'ping' }], + }); + + } catch (error: any) { + let errorMessage = 'Unknown error occured'; + let errorDetails: any = {}; + if (error) { + try { + errorMessage = error?.error?.error?.message || 'Unknown error'; + errorDetails = {requestId: error?.request_id, + errorType: error?.error?.error?.type, + statusCode: error?.status,} + } catch (parseError) { + console.error('Failed to parse Anthropic error response:', parseError); + } + } + throw new QueryError('Query could not be completed', errorMessage, errorDetails); + } + return { + status: 'ok', + }; + } + + async getConnection(sourceOptions: SourceOptions): Promise { + const { apiKey } = sourceOptions; + + if (!apiKey) { + throw new QueryError('Connection could not be established', 'API key is missing', {}); + } + + const anthropicClient = new Anthropic({ apiKey }); + + return anthropicClient; + } +} \ No newline at end of file diff --git a/marketplace/plugins/anthropic/lib/manifest.json b/marketplace/plugins/anthropic/lib/manifest.json new file mode 100644 index 0000000000..178df091ca --- /dev/null +++ b/marketplace/plugins/anthropic/lib/manifest.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json", + "title": "Anthropic datasource", + "description": "A schema defining Anthropic datasource", + "type": "api", + "source": { + "name": "Anthropic", + "kind": "anthropic", + "exposedVariables": { + "isLoading": false, + "data": {}, + "rawData": {} + }, + "options": { + "apiKey": { + "type": "string", + "encrypted": true + } + } + }, + "defaults": {}, + "properties": { + "apiKey": { + "label": "API key", + "key": "apiKey", + "type": "password", + "description": "Enter your Anthropic API Key", + "encrypted": true + } + }, + "required": [ + "apiKey" + ] +} \ No newline at end of file diff --git a/marketplace/plugins/anthropic/lib/operations.json b/marketplace/plugins/anthropic/lib/operations.json new file mode 100644 index 0000000000..92f6812731 --- /dev/null +++ b/marketplace/plugins/anthropic/lib/operations.json @@ -0,0 +1,246 @@ +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/operations.schema.json", + "title": "Anthropic datasource", + "description": "A schema defining Anthropic datasource", + "type": "api", + "defaults": { + "operation": "chat", + "model": "claude-3-5-sonnet-20241022" + }, + "properties": { + "operation": { + "label": "Operation", + "key": "operation", + "type": "dropdown-component-flip", + "description": "Select an operation", + "list": [ + { "value": "chat", "name": "Chat" } + ] + }, + "chat": { + "model": { + "label": "Model", + "key": "model", + "type": "dropdown-component-flip", + "description": "Select Claude Model", + "list": [ + { "value": "claude-3-5-sonnet-20241022", "name": "claude-3-5-sonnet-20241022"}, + { "value": "claude-3-5-haiku-20241022", "name": "claude-3-5-haiku-20241022" }, + { "value": "claude-3-opus-20240229", "name": "claude-3-opus-20240229" }, + { "value": "claude-3-sonnet-20240229", "name": "claude-3-sonnet-20240229" }, + { "value": "claude-3-haiku-20240307", "name": "claude-3-haiku-20240307" } + ] + }, + "claude-3-5-sonnet-20241022": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate messages and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company", + "height": "150px", + "mandatory": false, + "tooltip":"Defines role, context and/or role of the model to evaluate messages and send response" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Enter messages", + "placeholder": "[\n\t{ \"role\": \"user\", \"content\": \"Hello Claude!\" },\n\t{ \"role\": \"assistant\", \"content\": \"Hello! How can I assist you today?\" },\n\t{ \"role\": \"user\", \"content\": \"Give a report on stock performances of top 10 Fortune 500 companies\" }\n]", + "height": "150px", + "mandatory": true, + "tooltip": "Message object with role as \"assistant\" should always have a prefix and suffix message object with role as \"user\" in this array" + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Defines randomness of response. Takes value between 0 and 1. Default is 1.", + "placeholder": "0.5", + "height": "150px", + "mandatory": false, + "tooltip":"Defines randomness of response. Takes value between 0 and 1. Default is 1." + }, + "max_size": { + "label": "Max size", + "key": "max_size", + "type": "codehinter", + "description": "Maximum tokens used in response", + "placeholder": "256", + "height": "150px", + "mandatory": true, + "tooltip":"Maximum tokens used in response" + } + }, + "claude-3-5-haiku-20241022": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate messages and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company", + "height": "150px", + "mandatory": false, + "tooltip":"Defines role, context and/or role of the model to evaluate messages and send response" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Enter messages", + "placeholder": "[\n\t{ \"role\": \"user\", \"content\": \"Hello Claude!\" },\n\t{ \"role\": \"assistant\", \"content\": \"Hello! How can I assist you today?\" },\n\t{ \"role\": \"user\", \"content\": \"Give a report on stock performances of top 10 Fortune 500 companies\" }\n]", + "height": "150px", + "mandatory": true, + "tooltip": "Message object with role as \"assistant\" should always have a prefix and suffix message object with role as \"user\" in this array" + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Defines randomness of response. Takes value between 0 and 1. Default is 1.", + "placeholder": "0.5", + "height": "150px", + "mandatory": false, + "tooltip":"Defines randomness of response. Takes value between 0 and 1. Default is 1." + }, + "max_size": { + "label": "Max size", + "key": "max_size", + "type": "codehinter", + "description": "Maximum tokens used in response", + "placeholder": "256", + "height": "150px", + "mandatory": true, + "tooltip":"Maximum tokens used in response" + } + }, + "claude-3-opus-20240229": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate messages and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company", + "height": "150px", + "mandatory": false, + "tooltip":"Defines role, context and/or role of the model to evaluate messages and send response" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Enter messages", + "placeholder": "[\n\t{ \"role\": \"user\", \"content\": \"Hello Claude!\" },\n\t{ \"role\": \"assistant\", \"content\": \"Hello! How can I assist you today?\" },\n\t{ \"role\": \"user\", \"content\": \"Give a report on stock performances of top 10 Fortune 500 companies\" }\n]", + "height": "150px", + "mandatory": true, + "tooltip": "Message object with role as \"assistant\" should always have a prefix and suffix message object with role as \"user\" in this array" + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Defines randomness of response. Takes value between 0 and 1. Default is 1.", + "placeholder": "0.5", + "height": "150px", + "mandatory": false, + "tooltip":"Defines randomness of response. Takes value between 0 and 1. Default is 1." + }, + "max_size": { + "label": "Max size", + "key": "max_size", + "type": "codehinter", + "description": "Maximum tokens used in response", + "placeholder": "256", + "height": "150px", + "mandatory": true, + "tooltip":"Maximum tokens used in response" + } + }, + "claude-3-sonnet-20240229": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate messages and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company", + "height": "150px", + "mandatory": false, + "tooltip":"Defines role, context and/or role of the model to evaluate messages and send response" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Enter messages", + "placeholder": "[\n\t{ \"role\": \"user\", \"content\": \"Hello Claude!\" },\n\t{ \"role\": \"assistant\", \"content\": \"Hello! How can I assist you today?\" },\n\t{ \"role\": \"user\", \"content\": \"Give a report on stock performances of top 10 Fortune 500 companies\" }\n]", + "height": "150px", + "mandatory": true, + "tooltip": "Message object with role as \"assistant\" should always have a prefix and suffix message object with role as \"user\" in this array" + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Defines randomness of response. Takes value between 0 and 1. Default is 1.", + "placeholder": "0.5", + "height": "150px", + "mandatory": false, + "tooltip":"Defines randomness of response. Takes value between 0 and 1. Default is 1." + }, + "max_size": { + "label": "Max size", + "key": "max_size", + "type": "codehinter", + "description": "Maximum tokens used in response", + "placeholder": "256", + "height": "150px", + "mandatory": true, + "tooltip":"Maximum tokens used in response" + } + }, + "claude-3-haiku-20240307": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate messages and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company", + "height": "150px", + "mandatory": false, + "tooltip":"Defines role, context and/or role of the model to evaluate messages and send response" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Enter messages", + "placeholder": "[\n\t{ \"role\": \"user\", \"content\": \"Hello Claude!\" },\n\t{ \"role\": \"assistant\", \"content\": \"Hello! How can I assist you today?\" },\n\t{ \"role\": \"user\", \"content\": \"Give a report on stock performances of top 10 Fortune 500 companies\" }\n]", + "height": "150px", + "mandatory": true, + "tooltip": "Message object with role as \"assistant\" should always have a prefix and suffix message object with role as \"user\" in this array" + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Defines randomness of response. Takes value between 0 and 1. Default is 1.", + "placeholder": "0.5", + "height": "150px", + "mandatory": false, + "tooltip":"Defines randomness of response. Takes value between 0 and 1. Default is 1." + }, + "max_size": { + "label": "Max size", + "key": "max_size", + "type": "codehinter", + "description": "Maximum tokens used in response", + "placeholder": "256", + "height": "150px", + "mandatory": true, + "tooltip":"Maximum tokens used in response" + } + } + } + } +} \ No newline at end of file diff --git a/marketplace/plugins/anthropic/lib/query_operations.ts b/marketplace/plugins/anthropic/lib/query_operations.ts new file mode 100644 index 0000000000..79f8ab0643 --- /dev/null +++ b/marketplace/plugins/anthropic/lib/query_operations.ts @@ -0,0 +1,60 @@ +import { QueryOptions } from './types'; +import Anthropic from '@anthropic-ai/sdk'; + +const getMaxSize = (max_size: number | string | undefined): number => { + const size = typeof max_size === 'string' ? parseInt(max_size) : max_size; + return isNaN(size) ? 256 : Math.max(1, Math.min(2048, size)); +}; + +const getTemperature = (temperature: number | string | undefined): number => { + const temp = typeof temperature === 'string' ? parseFloat(temperature) : temperature; + return isNaN(temp) ? 0.5 : Math.max(0, Math.min(1, temp)); +}; + +export async function getChatCompletion( + anthropicClient: Anthropic, + options: QueryOptions +): Promise { + const { model, system_prompt, message, temperature, max_size } = options; + //try { + const messages = JSON.parse(message) + const response: any = await anthropicClient.messages.create({ + model: model || 'claude-3-5-sonnet-20241022', + system: system_prompt || '', + messages: messages, + max_tokens: getMaxSize(max_size), + temperature: getTemperature(temperature), + }); + + return response.content; //|| (response.choices && response.choices[0]?.text) || 'No output received'; + } /*catch (error) { + throw new Error(error?.message || 'An unexpected error occurred'); + } +}*/ + +/*export async function getVisionCompletion( + anthropicClient: Anthropic, + options: QueryOptions +): Promise { + const { model, system_prompt, message, temperature, max_size } = options; + + try { + const response = await anthropicClient.vision.create({ + model: model || 'claude-3-5-sonnet-20241022', + prompt: system_prompt || '', + messages: message || [], + temperature: getTemperature(temperature), + max_tokens: getMaxSize(max_size), + }); + + return response.completion; + } catch (error: any) { + console.error('Error in Anthropic vision completion:', error); + + return { + error: error?.message, + statusCode: error?.response?.status, + }; + } +} +*/ \ No newline at end of file diff --git a/marketplace/plugins/anthropic/lib/types.ts b/marketplace/plugins/anthropic/lib/types.ts new file mode 100644 index 0000000000..601fce6a7b --- /dev/null +++ b/marketplace/plugins/anthropic/lib/types.ts @@ -0,0 +1,36 @@ + +export type SourceOptions = { + apiKey: string; +}; + +export type QueryOptions = { + model?: string; + operation: Operation; + system_prompt?: string; + message?: string; + temperature?: number | string; + max_size?: number | string; +}; + +//for vision operation +/*export type MessageParam = { + role: "user" | "assistant"; + content: string | Content[]; +}; + +export type Content = { + type: "text" | "image"; + text?: string; + source?: ImageSource; +}; + +export type ImageSource = { + type: "base64"; + media_type: "image/jpeg"; + data: string; +};*/ + +export enum Operation { + Chat = "chat", + Vision = "vision" +} diff --git a/marketplace/plugins/anthropic/package.json b/marketplace/plugins/anthropic/package.json new file mode 100644 index 0000000000..7f2669b544 --- /dev/null +++ b/marketplace/plugins/anthropic/package.json @@ -0,0 +1,27 @@ +{ + "name": "@tooljet-marketplace/anthropic", + "version": "1.0.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "directories": { + "lib": "lib", + "test": "__tests__" + }, + "files": [ + "lib" + ], + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1", + "build": "ncc build lib/index.ts -o dist", + "watch": "ncc build lib/index.ts -o dist --watch" + }, + "homepage": "https://github.com/tooljet/tooljet#readme", + "dependencies": { + "@anthropic-ai/sdk": "^0.32.1", + "@tooljet-marketplace/common": "^1.0.0" + }, + "devDependencies": { + "@vercel/ncc": "^0.34.0", + "typescript": "^4.7.4" + } +} diff --git a/marketplace/plugins/anthropic/tsconfig.json b/marketplace/plugins/anthropic/tsconfig.json new file mode 100644 index 0000000000..a18a801b14 --- /dev/null +++ b/marketplace/plugins/anthropic/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "lib" + }, + "exclude": [ + "node_modules", + "dist" + ] +} \ No newline at end of file diff --git a/marketplace/plugins/cohere/.gitignore b/marketplace/plugins/cohere/.gitignore new file mode 100644 index 0000000000..23e6609462 --- /dev/null +++ b/marketplace/plugins/cohere/.gitignore @@ -0,0 +1,5 @@ +node_modules +lib/*.d.* +lib/*.js +lib/*.js.map +dist/* \ No newline at end of file diff --git a/marketplace/plugins/cohere/README.md b/marketplace/plugins/cohere/README.md new file mode 100644 index 0000000000..023a8dd8b2 --- /dev/null +++ b/marketplace/plugins/cohere/README.md @@ -0,0 +1,4 @@ + +# Cohere + +Documentation on: https://docs.tooljet.com/docs/data-sources/cohere \ No newline at end of file diff --git a/marketplace/plugins/cohere/__tests__/index.js b/marketplace/plugins/cohere/__tests__/index.js new file mode 100644 index 0000000000..c7299ef071 --- /dev/null +++ b/marketplace/plugins/cohere/__tests__/index.js @@ -0,0 +1,7 @@ +'use strict'; + +const cohere = require('../lib'); + +describe('cohere', () => { + it.todo('needs tests'); +}); diff --git a/marketplace/plugins/cohere/lib/icon.svg b/marketplace/plugins/cohere/lib/icon.svg new file mode 100644 index 0000000000..ea0f6b4aef --- /dev/null +++ b/marketplace/plugins/cohere/lib/icon.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/marketplace/plugins/cohere/lib/index.ts b/marketplace/plugins/cohere/lib/index.ts new file mode 100644 index 0000000000..da85c9bdfa --- /dev/null +++ b/marketplace/plugins/cohere/lib/index.ts @@ -0,0 +1,90 @@ +import { QueryError, QueryResult, QueryService, ConnectionTestResult } from '@tooljet-marketplace/common'; +import { SourceOptions, QueryOptions, Operation } from './types'; +import { textGeneration, chat } from './query_operations'; +import { CohereClientV2 } from 'cohere-ai'; + +export default class CohereService implements QueryService { + async run(sourceOptions: SourceOptions, queryOptions: QueryOptions, dataSourceId: string): Promise { + const operation = queryOptions.operation; + const cohere = await this.getConnection(sourceOptions); + let result = {}; + + try { + switch (operation) { + case Operation.TextGeneration: + result = await textGeneration(cohere, queryOptions); + break; + case Operation.Chat: + result = await chat(cohere, queryOptions); + break; + default: + throw new QueryError('Query could not be completed', 'Invalid operation', {}); + } + } catch (error: any) { + console.error('Error in Cohere query:', error); + + let errorMessage = 'Unknown error occurred'; + let errorDetails: any = {}; + + if (error && typeof error === 'object') { + try { + errorMessage = error?.body?.message || 'Unknown error'; + errorDetails = { + requestId: error?.req?.id || 'N/A', + errorType: error?.error?.type || error?.type|| error?.error?.details?.error || 'Unknown Type', + statusCode: error?.statusCode || error?.res?.statusCode || 'Unknown status', + }; + } catch (parseError: any) { + console.error('Failed to parse error response:', parseError); + } + } + + throw new QueryError('Query could not be completed', errorMessage, errorDetails); + } + + return { + status: 'ok', + data: result, + }; + } + + async testConnection(sourceOptions: SourceOptions): Promise { + const { apiKey } = sourceOptions; + if (!apiKey) { + throw new QueryError('Connection could not be established', 'API key is missing', {}); + } + const cohere = await this.getConnection(sourceOptions); + + try { + await cohere.chat({ + model: 'command-r-plus-08-2024', + messages: [ + { + role: 'user', + content: 'hello world!', + }, + ], + }); + } catch (error: any) { + const errorMessage = error.message || 'Unknown error occurred'; + throw new QueryError('Connection could not be established', errorMessage, { + statusCode: error.response?.status || 500, + }); + } + return { + status: 'ok', + }; + } + + async getConnection(sourceOptions: SourceOptions): Promise { + const { apiKey } = sourceOptions; + + if (!apiKey) { + throw new Error('API key missing: No API key provided in source options.'); + } + + const cohere = new CohereClientV2({token: apiKey}); + + return cohere; + } +} diff --git a/marketplace/plugins/cohere/lib/manifest.json b/marketplace/plugins/cohere/lib/manifest.json new file mode 100644 index 0000000000..64f343b882 --- /dev/null +++ b/marketplace/plugins/cohere/lib/manifest.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json", + "title": "Cohere datasource", + "description": "A schema defining Cohere datasource", + "type": "api", + "source": { + "name": "Cohere", + "kind": "cohere", + "exposedVariables": { + "isLoading": false, + "data": {}, + "rawData": {} + }, + "options": { + "apiKey": { + "type": "string", + "encrypted": true + } + } + }, + "defaults": {}, + "properties": { + "apiKey": { + "label": "API key", + "key": "apiKey", + "type": "password", + "description": "Enter your Cohere API Key", + "encrypted": true + } + }, + "required": [ + "apiKey" + ] +} \ No newline at end of file diff --git a/marketplace/plugins/cohere/lib/operations.json b/marketplace/plugins/cohere/lib/operations.json new file mode 100644 index 0000000000..af63629441 --- /dev/null +++ b/marketplace/plugins/cohere/lib/operations.json @@ -0,0 +1,651 @@ +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/operations.schema.json", + "title": "Cohere Datasource", + "description": "A schema defining Cohere datasource", + "type": "api", + "defaults": { + "operation": "text_generation", + "model": "command-r-plus" + }, + "properties": { + "operation": { + "label": "Operation", + "key": "operation", + "type": "dropdown-component-flip", + "description": "Select the operation to perform", + "list": [ + { "value": "text_generation", "name": "Text Generation" }, + { "value": "chat", "name": "Chat" } + ] + }, + "chat": { + "model": { + "label": "Model", + "key": "model", + "type": "dropdown-component-flip", + "description": "Select the Cohere model", + "mandatory": true, + "list": [ + { "value": "command-r7b-12-2024", "name": "command-r7b-12-2024" }, + { "value": "command-r-plus-08-2024", "name": "command-r-plus-08-2024" }, + { "value": "command-r-plus-04-2024", "name": "command-r-plus-04-2024" }, + { "value": "command-r-plus", "name": "command-r-plus" }, + { "value": "command-r-08-2024", "name": "command-r-08-2024" }, + { "value": "command-r-03-2024", "name": "command-r-03-2024" }, + { "value": "command-r", "name": "command-r" }, + { "value": "command", "name": "command" }, + { "value": "command-nightly", "name": "command-nightly" }, + { "value": "command-light", "name": "command-light" }, + { "value": "command-light-nightly", "name": "command-light-nightly" }, + { "value": "c4ai-aya-expanse-8b", "name": "c4ai-aya-expanse-8b" }, + { "value": "c4ai-aya-expanse-32b", "name": "c4ai-aya-expanse-32b" } + ] + }, + "command-r7b-12-2024":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + } + }, + "command-r-plus-08-2024":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-r-plus-04-2024":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-r-plus":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-r-08-2024":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-r-03-2024":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"NONE\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-r":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"NONE\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"NONE\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-nightly":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-light":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"NONE\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-light-nightly":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"NONE\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "c4ai-aya-expanse-8b":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"NONE\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "c4ai-aya-expanse-32b":{ + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Input the conversation history", + "mandatory": true, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are an SEO specialist content writer\"\n}, {\n \"role\": \"user\",\n \"content\": \"Write a title for a blog post about API design. Only output the title text.\"\n}, {\n \"role\": \"assistant\",\n \"content\": \"Designing Perfect APIs\"\n}]" + }, + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"NONE\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + } + }, + "text_generation": { + "model": { + "label": "Model", + "key": "model", + "type": "dropdown-component-flip", + "description": "Select the Cohere model", + "mandatory": true, + "list": [ + { "value": "command-r-plus-08-2024", "name": "command-r-plus-08-2024" }, + { "value": "command-r-plus-04-2024", "name": "command-r-plus-04-2024" }, + { "value": "command-r-plus", "name": "command-r-plus" }, + { "value": "command-r-08-2024", "name": "command-r-08-2024" }, + { "value": "command-r-03-2024", "name": "command-r-03-2024" }, + { "value": "command", "name": "command" }, + { "value": "command-nightly", "name": "command-nightly" }, + { "value": "command-light", "name": "command-light" }, + { "value": "command-light-nightly", "name": "command-light-nightly" }, + { "value": "c4ai-aya-expanse-8b", "name": "c4ai-aya-expanse-8b" }, + { "value": "c4ai-aya-expanse-32b", "name": "c4ai-aya-expanse-32b" } + ] + }, + "command-r-plus-08-2024":{ + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-r-plus-04-2024":{ + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-r-plus":{ + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-r-08-2024":{ + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-r-03-2024":{ + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command":{ + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-nightly":{ + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-light":{ + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "command-light-nightly":{ + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "c4ai-aya-expanse-8b":{ + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + }, + "c4ai-aya-expanse-32b":{ + "message": { + "label": "Message", + "key": "message", + "type": "codehinter", + "description": "Input the message for text generation", + "mandatory": true, + "placeholder": "Another one about generative AI.", + "tooltip": "Next user prompt in the chat" + }, + "advanced_parameters": { + "label": "Advanced parameters", + "key": "advanced_parameters", + "type": "codehinter", + "description": "Optional advanced parameters", + "mandatory": false, + "placeholder":"{\n \"response_format\": {\"type\": \"text\"},\n \"temperature\": 0.3,\n \"max_tokens\": 256,\n \"seed\": 3,\n \"p\": 0.3,\n \"k\": 1,\n \"frequency_penalty\": 0.3,\n \"presence_penalty\": 0.3,\n \"citation_options\": {\"mode\": \"fast\"},\n \"safety_mode\": \"CONTEXTUAL\",\n \"stop_sequences\": [\"spam\", \"fraud\"]\n}" + + } + } + } + } +} \ No newline at end of file diff --git a/marketplace/plugins/cohere/lib/query_operations.ts b/marketplace/plugins/cohere/lib/query_operations.ts new file mode 100644 index 0000000000..e10d8a4e9e --- /dev/null +++ b/marketplace/plugins/cohere/lib/query_operations.ts @@ -0,0 +1,52 @@ +import { CohereClientV2 } from 'cohere-ai'; +import { QueryOptions } from './types'; + +export async function textGeneration(cohere: CohereClientV2, options: QueryOptions) { + const { model, message, advanced_parameters } = options; + + if (!model || !message) { + return { error: 'Model and message are required for text generation.', statusCode: 400 }; + } + + let advancedParams = {}; + if (advanced_parameters) { + advancedParams = JSON.parse(advanced_parameters); + } + + const response = await cohere.generate({ + model: model, + prompt: message, + ...advancedParams, + }); + + return response; +} + +export async function chat(cohere: CohereClientV2, options: QueryOptions) { + const { model, message, advanced_parameters, history } = options; + + if (!model || !history || !message) { + throw new Error('Model, history, and message are required for chat.'); + } + + let parsedHistory = []; + parsedHistory = JSON.parse(history); + + parsedHistory.push({ + role: 'user', + content: message, + }); + + let advancedParams = {}; + if (advanced_parameters) { + advancedParams = JSON.parse(advanced_parameters); + } + + const response = await cohere.chat({ + model, + messages: parsedHistory, + ...advancedParams, + }); + + return response; +} diff --git a/marketplace/plugins/cohere/lib/types.ts b/marketplace/plugins/cohere/lib/types.ts new file mode 100644 index 0000000000..497a00a01f --- /dev/null +++ b/marketplace/plugins/cohere/lib/types.ts @@ -0,0 +1,15 @@ +export type SourceOptions = { + apiKey: string; +}; +export type QueryOptions = { + model?: string; + operation: Operation; + history?: string; + message?: string; + advanced_parameters?: string; +}; + +export enum Operation { + TextGeneration = 'text_generation', + Chat = 'chat', +} \ No newline at end of file diff --git a/marketplace/plugins/cohere/package.json b/marketplace/plugins/cohere/package.json new file mode 100644 index 0000000000..89e7a48c7c --- /dev/null +++ b/marketplace/plugins/cohere/package.json @@ -0,0 +1,27 @@ +{ + "name": "@tooljet-marketplace/cohere", + "version": "1.0.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "directories": { + "lib": "lib", + "test": "__tests__" + }, + "files": [ + "lib" + ], + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1", + "build": "ncc build lib/index.ts -o dist", + "watch": "ncc build lib/index.ts -o dist --watch" + }, + "homepage": "https://github.com/tooljet/tooljet#readme", + "dependencies": { + "@tooljet-marketplace/common": "^1.0.0", + "cohere-ai": "^7.15.4" + }, + "devDependencies": { + "@vercel/ncc": "^0.34.0", + "typescript": "^4.7.4" + } +} diff --git a/marketplace/plugins/cohere/tsconfig.json b/marketplace/plugins/cohere/tsconfig.json new file mode 100644 index 0000000000..a18a801b14 --- /dev/null +++ b/marketplace/plugins/cohere/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "lib" + }, + "exclude": [ + "node_modules", + "dist" + ] +} \ No newline at end of file diff --git a/marketplace/plugins/gemini/.gitignore b/marketplace/plugins/gemini/.gitignore new file mode 100644 index 0000000000..23e6609462 --- /dev/null +++ b/marketplace/plugins/gemini/.gitignore @@ -0,0 +1,5 @@ +node_modules +lib/*.d.* +lib/*.js +lib/*.js.map +dist/* \ No newline at end of file diff --git a/marketplace/plugins/gemini/README.md b/marketplace/plugins/gemini/README.md new file mode 100644 index 0000000000..16905e7829 --- /dev/null +++ b/marketplace/plugins/gemini/README.md @@ -0,0 +1,4 @@ + +# Gemini + +Documentation on: https://docs.tooljet.com/docs/data-sources/gemini \ No newline at end of file diff --git a/marketplace/plugins/gemini/__tests__/index.js b/marketplace/plugins/gemini/__tests__/index.js new file mode 100644 index 0000000000..8266a89a15 --- /dev/null +++ b/marketplace/plugins/gemini/__tests__/index.js @@ -0,0 +1,7 @@ +'use strict'; + +const gemini = require('../lib'); + +describe('gemini', () => { + it.todo('needs tests'); +}); diff --git a/marketplace/plugins/gemini/lib/icon.svg b/marketplace/plugins/gemini/lib/icon.svg new file mode 100644 index 0000000000..d2492c2e99 --- /dev/null +++ b/marketplace/plugins/gemini/lib/icon.svg @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/marketplace/plugins/gemini/lib/index.ts b/marketplace/plugins/gemini/lib/index.ts new file mode 100644 index 0000000000..b64f830373 --- /dev/null +++ b/marketplace/plugins/gemini/lib/index.ts @@ -0,0 +1,95 @@ +import { QueryError, QueryResult, QueryService, ConnectionTestResult } from '@tooljet-marketplace/common'; +import { SourceOptions, QueryOptions, Operation } from './types'; +import { GoogleGenerativeAI } from '@google/generative-ai'; +import { generateText, chat } from './query_operations'; + +export default class GeminiService implements QueryService { + async run(sourceOptions: SourceOptions, queryOptions: QueryOptions, dataSourceId: string): Promise { + const operation: Operation = queryOptions.operation; + const geminiClient = await this.getConnection(sourceOptions); + + let result: object | object[]; + + try { + switch (operation) { + case Operation.TextGeneration: + result = { text: await generateText(geminiClient, queryOptions) }; + break; + + case Operation.Chat: + result = { text: await chat(geminiClient, queryOptions) }; + break; + + default: + throw new QueryError('Query could not be completed', 'Invalid operation', {}); + } + } catch (error: any) { + console.error('Error in Gemini query:', error); + + let errorMessage = 'Unknown error occurred'; + let errorDetails: any = {}; + + if (error && typeof error === 'object') { + try { + errorMessage = error.message || error.response?.data?.error?.message || 'Unknown error'; + errorDetails = { + requestId: error.response?.data?.requestId || error.code, + errorType: error.response?.data?.error?.type || 'Unknown type', + statusCode: error.response?.status || error.status, + }; + } catch (parseError: any) { + console.error('Failed to parse Gemini error response:', parseError); + } + } + + throw new QueryError('Query could not be completed', errorMessage, errorDetails); + } + + return { + status: 'ok', + data: result, + }; + } + + async testConnection(sourceOptions: SourceOptions): Promise { + const { apiKey } = sourceOptions; + + if (!apiKey) { + throw new QueryError('Connection could not be established', 'API key is missing', {}); + } + + const options: QueryOptions = { + operation: Operation.TextGeneration, + model: 'models/gemini-1.5-flash', + system_prompt: 'Test system prompt', + prompt: 'This is a test prompt to generate some text.', + max_tokens: 100, + temperature: 0.7, + }; + + const geminiClient = new GoogleGenerativeAI(apiKey); + + try { + await generateText(geminiClient, options); + } catch (error: any) { + const errorMessage = error.message || 'Unknown error occurred'; + throw new QueryError('Connection could not be established', errorMessage, { + statusCode: error.response?.status || 500, + }); + } + + return { + status: 'ok', + }; + } + + async getConnection(sourceOptions: SourceOptions): Promise { + const { apiKey } = sourceOptions; + + if (!apiKey) { + throw new QueryError('Connection could not be established', 'API key is missing', {}); + } + + return new GoogleGenerativeAI(apiKey); + } +} \ No newline at end of file diff --git a/marketplace/plugins/gemini/lib/manifest.json b/marketplace/plugins/gemini/lib/manifest.json new file mode 100644 index 0000000000..fa05141aee --- /dev/null +++ b/marketplace/plugins/gemini/lib/manifest.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json", + "title": "Gemini datasource", + "description": "A schema defining Gemini datasource", + "type": "api", + "source": { + "name": "Gemini", + "kind": "gemini", + "exposedVariables": { + "isLoading": false, + "data": {}, + "rawData": {} + }, + "options": { + "apiKey": { + "type": "string", + "encrypted": true + } + } + }, + "defaults": {}, + "properties": { + "apiKey": { + "label": "API key", + "key": "apiKey", + "type": "password", + "description": "Enter your Gemini API Key", + "encrypted": true + } + }, + "required": [ + "apiKey" + ] +} \ No newline at end of file diff --git a/marketplace/plugins/gemini/lib/operations.json b/marketplace/plugins/gemini/lib/operations.json new file mode 100644 index 0000000000..7d3ab0ee2d --- /dev/null +++ b/marketplace/plugins/gemini/lib/operations.json @@ -0,0 +1,386 @@ +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/operations.schema.json", + "title": "Gemini datasource", + "description": "A schema defining Gemini datasource", + "type": "api", + "defaults": { + "operation": "text_generation", + "model": "models/gemini-1.5-flash" + }, + "properties": { + "operation": { + "label": "Operation", + "key": "operation", + "type": "dropdown-component-flip", + "description": "Select an operation", + "list": [ + { "value": "text_generation", "name": "Text Generation" }, + { "value": "chat", "name": "Chat" } + ] + }, + "text_generation": { + "model": { + "label": "Model", + "key": "model", + "type": "dropdown-component-flip", + "description": "Select Gemini Model", + "list": [ + { "value": "models/gemini-1.5-flash", "name": "Gemini 1.5 Flash" }, + { "value": "models/gemini-1.5-flash-8b", "name": "Gemini 1.5 Flash-8B" }, + { "value": "models/gemini-1.5-pro", "name": "Gemini 1.5 Pro" }, + { "value": "models/gemini-2.0-flash-exp", "name": "Gemini 2.0 Flash" } + ] + }, + "models/gemini-1.5-flash": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate prompts and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company ", + "mandatory": false, + "tooltip": "Defines role, context and/or role of the model to evaluate prompts and send response" + }, + "prompt": { + "label": "Prompt", + "key": "prompt", + "type": "codehinter", + "description": "Enter your prompt", + "placeholder": "Give a client-friendly explanation of the tax implications of different investment vehicles.", + "mandatory": true + }, + "max_tokens": { + "label": "Max tokens", + "key": "max_tokens", + "type": "codehinter", + "description": "Controls the length of the generated text.", + "placeholder": "1000", + "mandatory": false, + "tooltip": "Controls the length of the generated text." + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Controls the randomness/creativity of the generated text", + "placeholder": "0.1", + "mandatory": false, + "tooltip": "Controls the randomness/creativity of the generated text" + } + }, + "models/gemini-1.5-flash-8b": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate prompts and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company ", + "mandatory": false, + "tooltip": "Defines role, context and/or role of the model to evaluate prompts and send response" + }, + "prompt": { + "label": "Prompt", + "key": "prompt", + "type": "codehinter", + "description": "Enter your prompt", + "placeholder": "Give a client-friendly explanation of the tax implications of different investment vehicles.", + "mandatory": true + }, + "max_tokens": { + "label": "Max tokens", + "key": "max_tokens", + "type": "codehinter", + "description": "Controls the length of the generated text.", + "placeholder": "1000", + "mandatory": false, + "tooltip": "Controls the length of the generated text." + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Controls the randomness/creativity of the generated text", + "placeholder": "0.1", + "mandatory": false, + "tooltip": "Controls the randomness/creativity of the generated text" + } + }, + "models/gemini-1.5-pro": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate prompts and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company ", + "mandatory": false, + "tooltip": "Defines role, context and/or role of the model to evaluate prompts and send response" + }, + "prompt": { + "label": "Prompt", + "key": "prompt", + "type": "codehinter", + "description": "Enter your prompt", + "placeholder": "Give a client-friendly explanation of the tax implications of different investment vehicles.", + "mandatory": true + }, + "max_tokens": { + "label": "Max tokens", + "key": "max_tokens", + "type": "codehinter", + "description": "Controls the length of the generated text.", + "placeholder": "1000", + "mandatory": false, + "tooltip": "Controls the length of the generated text." + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Controls the randomness/creativity of the generated text", + "placeholder": "0.1", + "mandatory": false, + "tooltip": "Controls the randomness/creativity of the generated text" + } + }, + "models/gemini-2.0-flash-exp": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate prompts and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company ", + "mandatory": false, + "tooltip": "Defines role, context and/or role of the model to evaluate prompts and send response" + }, + "prompt": { + "label": "Prompt", + "key": "prompt", + "type": "codehinter", + "description": "Enter your prompt", + "placeholder": "Give a client-friendly explanation of the tax implications of different investment vehicles.", + "mandatory": true + }, + "max_tokens": { + "label": "Max tokens", + "key": "max_tokens", + "type": "codehinter", + "description": "Controls the length of the generated text.", + "placeholder": "1000", + "mandatory": false, + "tooltip": "Controls the length of the generated text." + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Controls the randomness/creativity of the generated text", + "placeholder": "0.1", + "mandatory": false, + "tooltip": "Controls the randomness/creativity of the generated text" + } + } + }, + "chat": { + "model": { + "label": "Model", + "key": "model", + "type": "dropdown-component-flip", + "description": "Select Gemini Model", + "list": [ + { "value": "models/gemini-1.5-flash", "name": "Gemini 1.5 Flash" }, + { "value": "models/gemini-1.5-flash-8b", "name": "Gemini 1.5 Flash-8B" }, + { "value": "models/gemini-1.5-pro", "name": "Gemini 1.5 Pro" }, + { "value": "models/gemini-2.0-flash-exp", "name": "Gemini 2.0 Flash" } + ] + }, + "models/gemini-1.5-flash": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate prompts and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company ", + "mandatory": false, + "tooltip": "Defines role, context and/or role of the model to evaluate prompts and send response" + }, + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Record of the conversation between the user and the model", + "placeholder": "[\n {\n \"role\": \"user\",\n \"parts\": [{\"text\": \"Hello\"}]\n },\n {\n \"role\": \"model\",\n \"parts\": [{\"text\": \"Great to meet you. What would you like to know?\"}]\n }\n]", + "mandatory": false, + "tooltip": "Record of the conversation between the user and the model" + }, + "user_prompt": { + "label": "User prompt", + "key": "user_prompt", + "type": "codehinter", + "description": "User response/reply to previous chat", + "placeholder": "Explain how AI works.", + "mandatory": true, + "tooltip":"User response/reply to previous chat" + }, + "max_tokens": { + "label": "Max tokens", + "key": "max_tokens", + "type": "codehinter", + "description": "Controls the length of the generated text.", + "placeholder": "1000", + "mandatory": false, + "tooltip":"Controls the length of the generated text." + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Controls the randomness/creativity of the generated text", + "placeholder": "0.1", + "mandatory": false, + "tooltip":"Controls the randomness/creativity of the generated text" + } + }, + "models/gemini-1.5-flash-8b": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate prompts and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company ", + "mandatory": false, + "tooltip": "Defines role, context and/or role of the model to evaluate prompts and send response" + }, + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Record of the conversation between the user and the model", + "placeholder": "[\n {\n \"role\": \"user\",\n \"parts\": [{\"text\": \"Hello\"}]\n },\n {\n \"role\": \"model\",\n \"parts\": [{\"text\": \"Great to meet you. What would you like to know?\"}]\n }\n]", + "mandatory": false, + "tooltip": "Record of the conversation between the user and the model" + }, + "user_prompt": { + "label": "User prompt", + "key": "user_prompt", + "type": "codehinter", + "description": "User response/reply to previous chat", + "placeholder": "Explain how AI works.", + "mandatory": true, + "tooltip":"User response/reply to previous chat" + }, + "max_tokens": { + "label": "Max tokens", + "key": "max_tokens", + "type": "codehinter", + "description": "Controls the length of the generated text.", + "placeholder": "1000", + "mandatory": false, + "tooltip":"Controls the length of the generated text." + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Controls the randomness/creativity of the generated text", + "placeholder": "0.1", + "mandatory": false, + "tooltip":"Controls the randomness/creativity of the generated text" + } + }, + "models/gemini-1.5-pro": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate prompts and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company ", + "mandatory": false, + "tooltip": "Defines role, context and/or role of the model to evaluate prompts and send response" + }, + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Record of the conversation between the user and the model", + "placeholder": "[\n {\n \"role\": \"user\",\n \"parts\": [{\"text\": \"Hello\"}]\n },\n {\n \"role\": \"model\",\n \"parts\": [{\"text\": \"Great to meet you. What would you like to know?\"}]\n }\n]", + "mandatory": false, + "tooltip": "Record of the conversation between the user and the model" + }, + "user_prompt": { + "label": "User prompt", + "key": "user_prompt", + "type": "codehinter", + "description": "User response/reply to previous chat", + "placeholder": "Explain how AI works.", + "mandatory": true, + "tooltip":"User response/reply to previous chat" + }, + "max_tokens": { + "label": "Max tokens", + "key": "max_tokens", + "type": "codehinter", + "description": "Controls the length of the generated text.", + "placeholder": "1000", + "mandatory": false, + "tooltip":"Controls the length of the generated text." + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Controls the randomness/creativity of the generated text", + "placeholder": "0.1", + "mandatory": false, + "tooltip":"Controls the randomness/creativity of the generated text" + } + }, + "models/gemini-2.0-flash-exp": { + "system_prompt": { + "label": "System prompt", + "key": "system_prompt", + "type": "codehinter", + "description": "Defines role, context and/or role of the model to evaluate prompts and send response", + "placeholder": "You are a Financial advisor working in a Fortune 500 company ", + "mandatory": false, + "tooltip": "Defines role, context and/or role of the model to evaluate prompts and send response" + }, + "history": { + "label": "History", + "key": "history", + "type": "codehinter", + "description": "Record of the conversation between the user and the model", + "placeholder": "[\n {\n \"role\": \"user\",\n \"parts\": [{\"text\": \"Hello\"}]\n },\n {\n \"role\": \"model\",\n \"parts\": [{\"text\": \"Great to meet you. What would you like to know?\"}]\n }\n]", + "mandatory": false, + "tooltip": "Record of the conversation between the user and the model" + }, + "user_prompt": { + "label": "User prompt", + "key": "user_prompt", + "type": "codehinter", + "description": "User response/reply to previous chat", + "placeholder": "Explain how AI works.", + "mandatory": true, + "tooltip":"User response/reply to previous chat" + }, + "max_tokens": { + "label": "Max tokens", + "key": "max_tokens", + "type": "codehinter", + "description": "Controls the length of the generated text.", + "placeholder": "1000", + "mandatory": false, + "tooltip":"Controls the length of the generated text." + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "description": "Controls the randomness/creativity of the generated text", + "placeholder": "0.1", + "mandatory": false, + "tooltip":"Controls the randomness/creativity of the generated text" + } + } + } + } +} \ No newline at end of file diff --git a/marketplace/plugins/gemini/lib/query_operations.ts b/marketplace/plugins/gemini/lib/query_operations.ts new file mode 100644 index 0000000000..523f298f98 --- /dev/null +++ b/marketplace/plugins/gemini/lib/query_operations.ts @@ -0,0 +1,82 @@ +import { GoogleGenerativeAI } from '@google/generative-ai'; +import { QueryOptions } from './types'; + +const getMaxTokens = (max_tokens: number | string | undefined): number => { + const tokens = typeof max_tokens === 'string' ? parseInt(max_tokens) : max_tokens; + return isNaN(tokens) ? 1000 : Math.max(1, Math.min(4096, tokens)); +}; + +const getTemperature = (temperature: number | string | undefined): number => { + const temp = typeof temperature === 'string' ? parseFloat(temperature) : temperature; + return isNaN(temp) ? 0.1 : Math.max(0, Math.min(1, temp)); +}; + +export async function generateText( + geminiClient: GoogleGenerativeAI, + options: QueryOptions +): Promise { + const { model, system_prompt, prompt, max_tokens, temperature } = options; + + if (!prompt) { + return { error: 'Prompt is required for text generation.', statusCode: 400 }; + } + + const generativeModel = geminiClient.getGenerativeModel({ + model: model || 'models/gemini-1.5-flash', + systemInstruction: system_prompt, + }); + + //try { + const response = await generativeModel.generateContent({ + contents: [ + { + role: 'user', + parts: [{ text: prompt }], + }, + ], + generationConfig: { + maxOutputTokens: getMaxTokens(max_tokens), + temperature: getTemperature(temperature), + }, + }); + + return response.response.text() || 'No output received'; + } //catch (error) { + //throw new Error(error?.message || 'An unexpected error occurred'); + //} +//} + +export async function chat( + geminiClient: GoogleGenerativeAI, + options: QueryOptions +): Promise { + const { model, system_prompt, history, user_prompt, max_tokens, temperature } = options; + + if (!user_prompt) { + return { error: 'User prompt is required for chat.', statusCode: 400 }; + } + + const generativeModel = geminiClient.getGenerativeModel({ + model: model || 'models/gemini-1.5-flash', + systemInstruction: system_prompt, + }); + + //try { + let histories = []; + if (history) { + histories = JSON.parse(history); + } + const chat = await generativeModel.startChat({ + history: histories, + generationConfig: { + maxOutputTokens: getMaxTokens(max_tokens), + temperature: getTemperature(temperature), + }, + }); + const response = await chat.sendMessage(user_prompt); + + return response.response.text() || 'No output received'; + } //catch (error) { + //throw new Error(error?.message || 'An unexpected error occurred'); + //} +//} \ No newline at end of file diff --git a/marketplace/plugins/gemini/lib/types.ts b/marketplace/plugins/gemini/lib/types.ts new file mode 100644 index 0000000000..a93a485928 --- /dev/null +++ b/marketplace/plugins/gemini/lib/types.ts @@ -0,0 +1,19 @@ +export type SourceOptions = { + apiKey: string; +}; + +export type QueryOptions = { + operation: Operation; + model: string; + system_prompt?: string; + prompt?: string; + user_prompt?: string; + history?: string; + max_tokens?: number | string; + temperature?: number | string; +}; + +export enum Operation { + TextGeneration = "text_generation", + Chat = "chat", +} \ No newline at end of file diff --git a/marketplace/plugins/gemini/package.json b/marketplace/plugins/gemini/package.json new file mode 100644 index 0000000000..4647a9f4d3 --- /dev/null +++ b/marketplace/plugins/gemini/package.json @@ -0,0 +1,27 @@ +{ + "name": "@tooljet-marketplace/gemini", + "version": "1.0.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "directories": { + "lib": "lib", + "test": "__tests__" + }, + "files": [ + "lib" + ], + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1", + "build": "ncc build lib/index.ts -o dist", + "watch": "ncc build lib/index.ts -o dist --watch" + }, + "homepage": "https://github.com/tooljet/tooljet#readme", + "dependencies": { + "@google/generative-ai": "^0.21.0", + "@tooljet-marketplace/common": "^1.0.0" + }, + "devDependencies": { + "@vercel/ncc": "^0.34.0", + "typescript": "^4.7.4" + } +} diff --git a/marketplace/plugins/gemini/tsconfig.json b/marketplace/plugins/gemini/tsconfig.json new file mode 100644 index 0000000000..a18a801b14 --- /dev/null +++ b/marketplace/plugins/gemini/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "lib" + }, + "exclude": [ + "node_modules", + "dist" + ] +} \ No newline at end of file diff --git a/marketplace/plugins/hugging_face/.gitignore b/marketplace/plugins/hugging_face/.gitignore new file mode 100644 index 0000000000..23e6609462 --- /dev/null +++ b/marketplace/plugins/hugging_face/.gitignore @@ -0,0 +1,5 @@ +node_modules +lib/*.d.* +lib/*.js +lib/*.js.map +dist/* \ No newline at end of file diff --git a/marketplace/plugins/hugging_face/README.md b/marketplace/plugins/hugging_face/README.md new file mode 100644 index 0000000000..df65ae5fec --- /dev/null +++ b/marketplace/plugins/hugging_face/README.md @@ -0,0 +1,4 @@ + +# Huggin Face + +Documentation on: https://docs.tooljet.com/docs/data-sources/huggingface \ No newline at end of file diff --git a/marketplace/plugins/hugging_face/__tests__/index.js b/marketplace/plugins/hugging_face/__tests__/index.js new file mode 100644 index 0000000000..1900218600 --- /dev/null +++ b/marketplace/plugins/hugging_face/__tests__/index.js @@ -0,0 +1,7 @@ +'use strict'; + +const huggingface = require('../lib'); + +describe('huggingface', () => { + it.todo('needs tests'); +}); diff --git a/marketplace/plugins/hugging_face/lib/icon.svg b/marketplace/plugins/hugging_face/lib/icon.svg new file mode 100644 index 0000000000..ea1d3bf570 --- /dev/null +++ b/marketplace/plugins/hugging_face/lib/icon.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/marketplace/plugins/hugging_face/lib/index.ts b/marketplace/plugins/hugging_face/lib/index.ts new file mode 100644 index 0000000000..fb5c1b80e1 --- /dev/null +++ b/marketplace/plugins/hugging_face/lib/index.ts @@ -0,0 +1,73 @@ +import { QueryError, QueryResult, QueryService, ConnectionTestResult } from '@tooljet-marketplace/common'; +import { SourceOptions, QueryOptions } from './types'; +import { summarisation_operation, text_generation_operation } from './query_operation'; + +export default class Huggingface implements QueryService { + private readonly API_URL = 'https://api-inference.huggingface.co/models/'; + async run(sourceOptions: SourceOptions, queryOptions: QueryOptions, dataSourceId: string): Promise { + const { personal_access_token, use_cache, wait_for_model } = sourceOptions; + const { operation } = queryOptions; + let result = {}; + + try { + const headers = { + Authorization: `Bearer ${personal_access_token}`, + 'Content-Type': 'application/json', + 'x-use-cache': String(use_cache ?? true), + 'x-wait-for-model': String(wait_for_model ?? false), + }; + + switch (operation) { + case 'text_generation': { + const response = await text_generation_operation(this.API_URL, queryOptions, headers); + result = { + text: response[0]?.generated_text || '', + }; + break; + } + case 'summarisation': { + const response = await summarisation_operation(this.API_URL, queryOptions, headers); + result = { + text: response[0]?.summary_text || '', + }; + break; + } + } + } catch (error) { + throw new QueryError('Query could not be completed', error.message, {}); + } + + return { + status: 'ok', + data: result, + }; + } + + async testConnection(sourceOptions: SourceOptions): Promise { + const { personal_access_token } = sourceOptions; + + try { + const response = await fetch('https://huggingface.co/api/whoami-v2', { + method: 'GET', + headers: { + Authorization: `Bearer ${personal_access_token}`, + 'Content-Type': 'application/json', + }, + }); + + if (!response.ok) { + throw new Error(`API responded with status ${response.status}`); + } + + return { + status: 'ok', + }; + } catch (error) { + throw new QueryError( + 'Connection test failed', + 'Could not establish connection to Hugging Face API. Please check your credentials.', + {} + ); + } + } +} diff --git a/marketplace/plugins/hugging_face/lib/manifest.json b/marketplace/plugins/hugging_face/lib/manifest.json new file mode 100644 index 0000000000..fdf908315e --- /dev/null +++ b/marketplace/plugins/hugging_face/lib/manifest.json @@ -0,0 +1,54 @@ + +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json", + "title": "Hugging Face datasource", + "description": "A schema defining Hugging Face datasource", + "type": "api", + "source": { + "name": "Hugging Face", + "kind": "hugging_face", + "exposedVariables": { + "isLoading": false, + "data": {}, + "rawData": {} + }, + "options": { + "personal_access_token": { + "type": "string", + "encrypted": true + } + } + }, + "defaults": { + "personal_access_token": { + "value": "" + }, + "use_cache":{ + "value": true + }, + "wait_for_model":{ + "value": false + } + }, + "properties": { + "personal_access_token": { + "label": "Personal access token", + "key": "personal_access_token", + "type": "textarea", + "encrypted": true + }, + "use_cache": { + "key": "use_cache", + "type": "toggle", + "text":"Use cache", + "subtext":"Turn on cache layer on the inference API to speed up requests" + }, + "wait_for_model": { + "key": "wait_for_model", + "type": "toggle", + "text":"Wait for model", + "subtext":"Turn on to wait for model if it is not ready instead of receiving 503" + } + }, + "required": [] +} \ No newline at end of file diff --git a/marketplace/plugins/hugging_face/lib/operations.json b/marketplace/plugins/hugging_face/lib/operations.json new file mode 100644 index 0000000000..1ab8c742ff --- /dev/null +++ b/marketplace/plugins/hugging_face/lib/operations.json @@ -0,0 +1,90 @@ + +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/operations.schema.json", + "title": "Hugging Face datasource", + "description": "A schema defining Hugging Face datasource", + "type": "api", + "defaults": { + "operation":"text_generation", + "model": "google/gemma-2-2b-it", + "model_summarisation": "facebook/bart-large-cnn" + }, + "properties": { + "operation": { + "label": "Operation", + "key": "operation", + "type": "dropdown-component-flip", + "description": "Single select dropdown for huggingface operations", + "list": [ + { "value": "text_generation", "name": "Text Generation" }, + { "value": "summarisation", "name": "Summarisation" } + ] + }, + "text_generation":{ + "model":{ + "label": "Model", + "key": "model", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "google/gemma-2-2b-it", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins" + }, + "input":{ + "label": "Input", + "key": "input", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "Can you elaborate this: {{components.textInput1.value}}", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins" + }, + "operation_parameters":{ + "label": "Operation parameters", + "key": "operation_parameters", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "{\n “max_new_tokens”: 1000,\n “temperature”: 0.1\n}", + "width": "320px", + "height": "80px", + "className": "codehinter-plugins", + "tooltip":"Model response configurations. Note: These parameters might change based on model being used." + } + }, + "summarisation":{ + "model_summarisation":{ + "label": "Model", + "key": "model_summarisation", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "facebook/bart-large-cnn", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins" + }, + "input_summarisation":{ + "label": "Input", + "key": "input_summarisation", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct.", + "width": "320px", + "height": "192px", + "className": "codehinter-plugins" + }, + "operation_parameters_summarisation":{ + "label": "Operation parameters", + "key": "operation_parameters_summarisation", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "{\n \"clean_up_tokenization_spaces\": true,\n \"truncation\": \"do_not_truncate\",\n \"temperature\": 0.7\n \"max_length\": 130,\n \"min_length\": 30,\n \"do_sample\": false,\n}", + "width": "320px", + "height": "80px", + "className": "codehinter-plugins", + "tooltip":"Model response configurations. Note: These parameters might change based on model being used." + } + } + } +} \ No newline at end of file diff --git a/marketplace/plugins/hugging_face/lib/query_operation.ts b/marketplace/plugins/hugging_face/lib/query_operation.ts new file mode 100644 index 0000000000..2864fda638 --- /dev/null +++ b/marketplace/plugins/hugging_face/lib/query_operation.ts @@ -0,0 +1,31 @@ +export async function text_generation_operation(api_url, queryOptions, headers) { + const { model, input, operation_parameters } = queryOptions; + const response = await fetch(`${api_url}${model}`, { + method: 'POST', + headers, + body: JSON.stringify({ + inputs: input, + ...(operation_parameters ? { parameters: JSON.parse(operation_parameters) } : {}), + }), + }); + if (!response.ok) { + throw new Error('Text generation operation failed'); + } + return await response.json(); +} + +export async function summarisation_operation(api_url, queryOptions, headers) { + const { model_summarisation, input_summarisation, operation_parameters_summarisation } = queryOptions; + const response = await fetch(`${api_url}${model_summarisation}`, { + method: 'POST', + headers, + body: JSON.stringify({ + inputs: input_summarisation, + ...(operation_parameters_summarisation ? { parameters: JSON.parse(operation_parameters_summarisation) } : {}), + }), + }); + if (!response.ok) { + throw new Error('Summarisation operation failed'); + } + return await response.json(); +} diff --git a/marketplace/plugins/hugging_face/lib/types.ts b/marketplace/plugins/hugging_face/lib/types.ts new file mode 100644 index 0000000000..f32813b919 --- /dev/null +++ b/marketplace/plugins/hugging_face/lib/types.ts @@ -0,0 +1,14 @@ +export type SourceOptions = { + personal_access_token: string; + use_cache: boolean; + wait_for_model: boolean; +}; +export type QueryOptions = { + operation: string; + model: string; + input: string; + operation_parameters: string; + model_summarisation: string; + input_summarisation: string; + operation_parameters_summarisation: string; +}; diff --git a/marketplace/plugins/hugging_face/package.json b/marketplace/plugins/hugging_face/package.json new file mode 100644 index 0000000000..940acc18a4 --- /dev/null +++ b/marketplace/plugins/hugging_face/package.json @@ -0,0 +1,26 @@ +{ + "name": "@tooljet-marketplace/huggingface", + "version": "1.0.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "directories": { + "lib": "lib", + "test": "__tests__" + }, + "files": [ + "lib" + ], + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1", + "build": "ncc build lib/index.ts -o dist", + "watch": "ncc build lib/index.ts -o dist --watch" + }, + "homepage": "https://github.com/tooljet/tooljet#readme", + "dependencies": { + "@tooljet-marketplace/common": "^1.0.0" + }, + "devDependencies": { + "typescript": "^4.7.4", + "@vercel/ncc": "^0.34.0" + } +} diff --git a/marketplace/plugins/hugging_face/tsconfig.json b/marketplace/plugins/hugging_face/tsconfig.json new file mode 100644 index 0000000000..a18a801b14 --- /dev/null +++ b/marketplace/plugins/hugging_face/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "lib" + }, + "exclude": [ + "node_modules", + "dist" + ] +} \ No newline at end of file diff --git a/marketplace/plugins/mistral_ai/.gitignore b/marketplace/plugins/mistral_ai/.gitignore new file mode 100644 index 0000000000..23e6609462 --- /dev/null +++ b/marketplace/plugins/mistral_ai/.gitignore @@ -0,0 +1,5 @@ +node_modules +lib/*.d.* +lib/*.js +lib/*.js.map +dist/* \ No newline at end of file diff --git a/marketplace/plugins/mistral_ai/README.md b/marketplace/plugins/mistral_ai/README.md new file mode 100644 index 0000000000..b94b704572 --- /dev/null +++ b/marketplace/plugins/mistral_ai/README.md @@ -0,0 +1,4 @@ + +# Mistral + +Documentation on: https://docs.tooljet.com/docs/data-sources/mistral \ No newline at end of file diff --git a/marketplace/plugins/mistral_ai/__tests__/index.js b/marketplace/plugins/mistral_ai/__tests__/index.js new file mode 100644 index 0000000000..ce33ba827f --- /dev/null +++ b/marketplace/plugins/mistral_ai/__tests__/index.js @@ -0,0 +1,7 @@ +'use strict'; + +const mistral = require('../lib'); + +describe('mistral', () => { + it.todo('needs tests'); +}); diff --git a/marketplace/plugins/mistral_ai/lib/icon.svg b/marketplace/plugins/mistral_ai/lib/icon.svg new file mode 100644 index 0000000000..5dbd09383b --- /dev/null +++ b/marketplace/plugins/mistral_ai/lib/icon.svg @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/marketplace/plugins/mistral_ai/lib/index.ts b/marketplace/plugins/mistral_ai/lib/index.ts new file mode 100644 index 0000000000..6fafe000a7 --- /dev/null +++ b/marketplace/plugins/mistral_ai/lib/index.ts @@ -0,0 +1,111 @@ +import { QueryError, QueryResult, QueryService, ConnectionTestResult } from '@tooljet-marketplace/common'; +import { SourceOptions, QueryOptions } from './types'; +import { Mistral as MistralClient } from '@mistralai/mistralai'; + +export default class MistralService implements QueryService { + async run(sourceOptions: SourceOptions, queryOptions: QueryOptions, dataSourceId: string): Promise { + const client = await this.getConnection(sourceOptions); + const { operation } = queryOptions; + let result = {}; + + try { + switch (operation) { + case 'text_generation': { + const { + model, + messages, + max_tokens, + temperature, + top_p, + stop_tokens, + random_seed, + response_format, + presence_penalty, + frequency_penalty, + completions, + safe_prompt, + } = queryOptions; + + const parsedMessages = typeof messages === 'string' ? JSON.parse(messages) : messages; + + const chatRequest = { + model: model, + messages: parsedMessages, + ...(max_tokens && { max_tokens: parseInt(max_tokens) }), + ...(temperature && { temperature: parseFloat(temperature) }), + ...(top_p && { top_p: parseFloat(top_p) }), + ...(stop_tokens && { stop: JSON.parse(stop_tokens) }), + ...(random_seed && { random_seed: parseInt(random_seed) }), + ...(response_format && { response_format: { type: response_format } }), + ...(presence_penalty && { presence_penalty: parseFloat(presence_penalty) }), + ...(frequency_penalty && { frequency_penalty: parseFloat(frequency_penalty) }), + ...(completions && { n: parseInt(completions) }), + ...(safe_prompt.value !== undefined && { safe_prompt: Boolean(safe_prompt.value) }), + }; + + result = await client.chat.complete(chatRequest); + break; + } + } + } catch (error) { + let errorMessage = 'An unknown error occurred'; + let mainError = null; + + if (error) { + if (error.message.includes('Input validation failed')) { + if (error.cause?.issues?.[0]?.message) { + errorMessage = error.cause.issues[0].message; + } else { + errorMessage = 'Invalid input parameters'; + } + } else { + errorMessage = error.message; + } + mainError = error; + } + + const errorDetails = { + errorType: mainError?.name || 'Error', + raw: error, + }; + throw new QueryError('Query could not be completed', errorMessage, errorDetails); + } + return { + status: 'ok', + data: result, + }; + } + + async testConnection(sourceOptions: SourceOptions): Promise { + try { + const client = await this.getConnection(sourceOptions); + const chatResponse = await client.chat.complete({ + model: 'mistral-large-latest', + messages: [{ role: 'user', content: 'What is the best French cheese?' }], + }); + if (chatResponse.choices[0].message.content) { + return { + status: 'ok', + message: 'Connection established successfully', + }; + } + return { + status: 'failed', + message: 'Could not get a valid response from Mistral API', + }; + } catch (error) { + return { + status: 'failed', + message: error.message || 'Could not connect to Mistral API', + }; + } + } + + async getConnection(sourceOptions: SourceOptions) { + const { api_key } = sourceOptions; + return new MistralClient({ + apiKey: api_key, + timeoutMs: 60000, + }); + } +} diff --git a/marketplace/plugins/mistral_ai/lib/manifest.json b/marketplace/plugins/mistral_ai/lib/manifest.json new file mode 100644 index 0000000000..84015e0326 --- /dev/null +++ b/marketplace/plugins/mistral_ai/lib/manifest.json @@ -0,0 +1,35 @@ +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json", + "title": "Mistral datasource", + "description": "A schema defining Mistral datasource", + "type": "api", + "source": { + "name": "Mistral", + "kind": "mistral_ai", + "exposedVariables": { + "isLoading": false, + "data": {}, + "rawData": {} + }, + "options": { + "api_key": { + "type": "string", + "encrypted": true + } + } + }, + "defaults": { + "api_key": { + "value": "" + } + }, + "properties": { + "api_key": { + "label": "API key", + "key": "api_key", + "type": "textarea", + "encrypted": true + } + }, + "required": [] +} \ No newline at end of file diff --git a/marketplace/plugins/mistral_ai/lib/operations.json b/marketplace/plugins/mistral_ai/lib/operations.json new file mode 100644 index 0000000000..c4c96cdb17 --- /dev/null +++ b/marketplace/plugins/mistral_ai/lib/operations.json @@ -0,0 +1,155 @@ +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/operations.schema.json", + "title": "Mistral datasource", + "description": "A schema defining Mistral datasource", + "type": "api", + "defaults": { + "operation": "text_generation", + "model": "mistral-large-latest", + "response_format": "text", + "safe_prompt": false + }, + "properties": { + "operation": { + "label": "Operation", + "key": "operation", + "type": "dropdown-component-flip", + "description": "Single select dropdown for operation", + "list": [ + { "value": "text_generation", "name": "Text Generation" } + ] + }, + "text_generation": { + "model": { + "label": "Model", + "key": "model", + "type": "dropdown", + "description": "Select mistral model", + "list": [ + { "value": "mistral-small-latest", "name": "mistral-small-latest" }, + { "value": "mistral-large-latest", "name": "mistral-large-latest" }, + { "value": "ministral-3b-latest", "name": "ministral-3b-latest" }, + { "value": "ministral-8b-latest", "name": "ministral-8b-latest" }, + { "value": "open-mistral-nemo", "name": "open-mistral-nemo" } + ] + }, + "messages": { + "label": "Messages", + "key": "messages", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "[{\n \"role\": \"system\",\n \"content\": \"You are a financial advisor of a fortune 500 company\"\n},\n{\n \"role\": \"user\",\n \"content\": \"Can you help me with tax benefits for the employee\"\n},\n{\n \"role\": \"assistant\",\n \"content\": \"Sure! Please help me with more specific details of your employment and salary structure.\"\n},\n{\n \"role\": \"user\",\n \"content\": \"I am a salaried employee with 220,000 USD yearly compensation\"\n}]", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "tooltip": "Array of messages between system, assistant and user." + }, + "max_tokens": { + "label": "Max tokens", + "key": "max_tokens", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "512", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "tooltip": "Maximum tokens used in response" + }, + "temperature": { + "label": "Temperature", + "key": "temperature", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "0.0", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "tooltip": "Defines randomness of response. Takes value between 0 and 1. Default is 1" + }, + "top_p": { + "label": "Top P", + "key": "top_p", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "1", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "tooltip": "Nucleus sampling, where the model considers the results of the tokens with top_p probability mass" + }, + "stop_tokens": { + "label": "Stop tokens(s)", + "key": "stop_tokens", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "[]", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "tooltip": "Stop generation if this token (string or array of string) is detected" + }, + "random_seed": { + "label": "Random seed", + "key": "random_seed", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "42", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "tooltip": "Used for random sampling" + }, + "response_format": { + "label": "Response format", + "key": "response_format", + "type": "dropdown", + "description": "Select response format", + "list": [ + { "value": "json_object", "name": "JSON object" }, + { "value": "text", "name": "Text" } + ], + "tooltip": "Format of model's response" + }, + "presence_penalty": { + "label": "Presence penalty", + "key": "presence_penalty", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "0", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "tooltip": "Determines how much the model penalizes the repetition of words or phrases" + }, + "frequency_penalty": { + "label": "Frequency penalty", + "key": "frequency_penalty", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "0", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "tooltip": "Determines how much the model penalizes the repetition of words based on their frequency in the generated text" + }, + "completions": { + "label": "Completions (N)", + "key": "completions", + "type": "codehinter", + "lineNumbers": false, + "placeholder": "1", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "tooltip": "Number of completions to return for each request" + }, + "safe_prompt": { + "type": "toggle", + "label": "Safe prompt", + "key": "safe_prompt", + "text": "Enable", + "tooltip": "Whether to inject a safety prompt before all conversations" + } + } + } +} \ No newline at end of file diff --git a/marketplace/plugins/mistral_ai/lib/types.ts b/marketplace/plugins/mistral_ai/lib/types.ts new file mode 100644 index 0000000000..b76afd5bb6 --- /dev/null +++ b/marketplace/plugins/mistral_ai/lib/types.ts @@ -0,0 +1,22 @@ +export type SourceOptions = { + api_key: string; +}; +export type QueryOptions = { + operation: string; + model: string; + messages: string; + max_tokens: string; + temperature: string; + top_p: string; + stop_tokens: string; + random_seed: string; + response_format: string; + presence_penalty: string; + frequency_penalty: string; + completions: string; + safe_prompt: Value; +}; + +type Value = { + value: string; +}; diff --git a/marketplace/plugins/mistral_ai/package.json b/marketplace/plugins/mistral_ai/package.json new file mode 100644 index 0000000000..e373a03f51 --- /dev/null +++ b/marketplace/plugins/mistral_ai/package.json @@ -0,0 +1,27 @@ +{ + "name": "@tooljet-marketplace/mistral", + "version": "1.0.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "directories": { + "lib": "lib", + "test": "__tests__" + }, + "files": [ + "lib" + ], + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1", + "build": "ncc build lib/index.ts -o dist", + "watch": "ncc build lib/index.ts -o dist --watch" + }, + "homepage": "https://github.com/tooljet/tooljet#readme", + "dependencies": { + "@mistralai/mistralai": "^1.4.0", + "@tooljet-marketplace/common": "^1.0.0" + }, + "devDependencies": { + "@vercel/ncc": "^0.34.0", + "typescript": "^4.7.4" + } +} diff --git a/marketplace/plugins/mistral_ai/tsconfig.json b/marketplace/plugins/mistral_ai/tsconfig.json new file mode 100644 index 0000000000..a18a801b14 --- /dev/null +++ b/marketplace/plugins/mistral_ai/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "lib" + }, + "exclude": [ + "node_modules", + "dist" + ] +} \ No newline at end of file diff --git a/marketplace/plugins/qdrant/.gitignore b/marketplace/plugins/qdrant/.gitignore new file mode 100644 index 0000000000..23e6609462 --- /dev/null +++ b/marketplace/plugins/qdrant/.gitignore @@ -0,0 +1,5 @@ +node_modules +lib/*.d.* +lib/*.js +lib/*.js.map +dist/* \ No newline at end of file diff --git a/marketplace/plugins/qdrant/README.md b/marketplace/plugins/qdrant/README.md new file mode 100644 index 0000000000..2e4c5d3fb1 --- /dev/null +++ b/marketplace/plugins/qdrant/README.md @@ -0,0 +1,4 @@ + +# Qdrant + +Documentation on: https://docs.tooljet.com/docs/data-sources/qdrant \ No newline at end of file diff --git a/marketplace/plugins/qdrant/lib/icon.svg b/marketplace/plugins/qdrant/lib/icon.svg new file mode 100644 index 0000000000..45ca3b0afa --- /dev/null +++ b/marketplace/plugins/qdrant/lib/icon.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/marketplace/plugins/qdrant/lib/index.ts b/marketplace/plugins/qdrant/lib/index.ts new file mode 100644 index 0000000000..17e9f6a124 --- /dev/null +++ b/marketplace/plugins/qdrant/lib/index.ts @@ -0,0 +1,96 @@ +import { QueryError, QueryResult, QueryService, ConnectionTestResult } from '@tooljet-marketplace/common'; +import { SourceOptions, QueryOptions, Operation } from './types'; +import { QdrantClient } from '@qdrant/js-client-rest'; +import { + deletePoints, + getCollectionInfo, + getPoints, + listCollections, + queryPoints, + upsertPoints, +} from './query_operations'; + +export default class Qdrant implements QueryService { + async run(sourceOptions: SourceOptions, queryOptions: QueryOptions, dataSourceId: string): Promise { + const operation = queryOptions.operation; + const qdrant = await this.getConnection(sourceOptions); + let result = {}; + + try { + switch (operation) { + case Operation.GetCollectionInfo: + result = await getCollectionInfo(qdrant, queryOptions); + break; + case Operation.ListCollections: + result = await listCollections(qdrant, queryOptions); + break; + case Operation.GetPoints: + result = await getPoints(qdrant, queryOptions); + break; + case Operation.UpsertPoints: + result = await upsertPoints(qdrant, queryOptions); + break; + case Operation.DeletePoints: + result = await deletePoints(qdrant, queryOptions); + break; + case Operation.QueryPoints: + result = await queryPoints(qdrant, queryOptions); + break; + default: + throw new QueryError('Unsupported Operation', operation + ' is not supported.', {}); + } + } catch (error) { + let errorMessage = 'An unknown error occurred'; + let errorDetails = {}; + + if (error instanceof Error) { + errorMessage = error.message || errorMessage; + errorDetails = { + name: error.name, + code: (error as any).code || null, + codeName: (error as any).codeName || null, + keyPattern: (error as any).keyPattern || null, + keyValue: (error as any).keyValue || null, + }; + } + + throw new QueryError('Query could not be completed', errorMessage, errorDetails); + } + + return { + status: 'ok', + data: result, + }; + } + async testConnection(sourceOptions: SourceOptions): Promise { + const qdrant = await this.getConnection(sourceOptions); + + try { + await qdrant.getCollections(); + console.log('Connection successful.'); + return { status: 'ok' }; + } catch (error) { + console.error('Connection could not be established:', error.message); + throw new QueryError('Connection could not be established', error?.message, {}); + } + } + + async getConnection(sourceOptions: SourceOptions): Promise { + const { apiKey, url } = sourceOptions; + + if (!url) { + throw new QueryError('URL missing', 'No Qdrant URL provided in source options', {}); + } + + return new QdrantClient({ + url, + apiKey, + }); + } + + private parseJSON(json?: string): object { + if (!json) return {}; + + return JSON.parse(json); + } +} diff --git a/marketplace/plugins/qdrant/lib/manifest.json b/marketplace/plugins/qdrant/lib/manifest.json new file mode 100644 index 0000000000..aac27fd8ab --- /dev/null +++ b/marketplace/plugins/qdrant/lib/manifest.json @@ -0,0 +1,47 @@ +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json", + "title": "Qdrant datasource", + "description": "A schema defining Qdrant datasource", + "type": "api", + "source": { + "name": "Qdrant", + "kind": "qdrant", + "exposedVariables": { + "isLoading": false, + "data": {}, + "rawData": {} + }, + "options": { + "url": { + "type": "string", + "encrypted": false + }, + "apiKey": { + "type": "string", + "encrypted": true + } + } + }, + "defaults": {}, + "properties": { + "url": { + "label": "Qdrant URL", + "key": "url", + "type": "text", + "description": "Enter your Qdrant URL.", + "helpText": "REST URL to authenticate the requests of the Qdrant instance.", + "encrypted": true + }, + "apiKey": { + "label": "API Key", + "key": "apiKey", + "type": "password", + "description": "Enter your Qdrant API key", + "helpText": "An API key to authenticate the requests.", + "encrypted": true + } + }, + "required": [ + "url" + ] +} \ No newline at end of file diff --git a/marketplace/plugins/qdrant/lib/operations.json b/marketplace/plugins/qdrant/lib/operations.json new file mode 100644 index 0000000000..8d7fc70627 --- /dev/null +++ b/marketplace/plugins/qdrant/lib/operations.json @@ -0,0 +1,166 @@ +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/operations.schema.json", + "title": "Qdrant Datasource", + "description": "A schema defining Qdrant datasource", + "type": "api", + "defaults": { + "operation": "get_collection_info" + }, + "properties": { + "operation": { + "label": "Operation", + "key": "operation", + "type": "dropdown-component-flip", + "description": "Select an operation", + "list": [ + { + "value": "get_collection_info", + "name": "Get Collection Info" + }, + { + "value": "list_collections", + "name": "List Collections" + }, + { + "value": "get_points", + "name": "Get Points" + }, + { + "value": "upsert_points", + "name": "Upsert Points" + }, + { + "value": "delete_points", + "name": "Delete Points" + }, + { + "value": "query_points", + "name": "Query Points" + } + ] + }, + "get_collection_info": { + "collectionName": { + "label": "Collection Name", + "key": "collectionName", + "type": "codehinter", + "description": "Enter the collection name.", + "placeholder": "example-collection", + "height": "36px" + } + }, + "list_collections": {}, + "get_points": { + "collectionName": { + "label": "Collection Name", + "key": "collectionName", + "type": "codehinter", + "description": "Enter the collection name.", + "placeholder": "example-collection", + "height": "36px" + }, + "ids": { + "label": "IDs", + "key": "ids", + "type": "codehinter", + "description": "Enter point IDs as JSON array", + "placeholder": "[\"792defdc-e3f9-49aa-b7b2-ddebcdbdc2a6\", 23]", + "height": "36px" + } + }, + "upsert_points": { + "collectionName": { + "label": "Collection Name", + "key": "collectionName", + "type": "codehinter", + "description": "Enter the collection name.", + "placeholder": "example-collection", + "height": "36px" + }, + "points": { + "label": "Points", + "key": "points", + "type": "codehinter", + "description": "Enter points as JSON array", + "placeholder": "[{\"id\":1,\"payload\":{\"color\":\"red\"},\"vector\":[0.9,0.1,0.1]}]", + "height": "36px" + } + }, + "delete_points": { + "collectionName": { + "label": "Collection Name", + "key": "collectionName", + "type": "codehinter", + "description": "Enter the collection name.", + "placeholder": "example-collection", + "height": "36px" + }, + "ids": { + "label": "IDs", + "key": "id", + "type": "codehinter", + "description": "Enter point IDs as JSON array", + "placeholder": "[\"845d6b75-38f6-4af7-bffa-b1fd46131ab5\", 23]", + "height": "36px" + }, + "filter": { + "label": "Filter", + "key": "filter", + "type": "codehinter", + "description": "Enter a filter query in JSON format", + "placeholder": "{\"must\":[{\"key\":\"color\",\"match\":{\"value\":\"red\"}}]}", + "height": "150px" + } + }, + "query_points": { + "collectionName": { + "label": "Collection Name", + "key": "collectionName", + "type": "codehinter", + "description": "Enter the collection name.", + "placeholder": "example-collection", + "height": "36px" + }, + "limit": { + "label": "Limit", + "key": "limit", + "type": "codehinter", + "description": "Enter the number of results to return.", + "placeholder": "3", + "height": "36px" + }, + "filter": { + "label": "Filter", + "key": "filter", + "type": "codehinter", + "description": "Enter a filter query in JSON format.", + "placeholder": "{\"must\":[{\"key\":\"color\",\"match\":{\"value\":\"red\"}}]}", + "height": "150px" + }, + "withVectors": { + "label": "With Vectors", + "key": "withVectors", + "type": "codehinter", + "description": "Whether to return vector values.", + "placeholder": "true (false by default)", + "height": "36px" + }, + "withPayload": { + "label": "Include metadata", + "key": "withPayload", + "type": "codehinter", + "description": "Whether to return payload values.", + "placeholder": "true (false by default)", + "height": "36px" + }, + "query": { + "label": "Query", + "key": "query", + "type": "codehinter", + "description": "Enter the query in JSON.", + "placeholder": "[0.2, 0.1, 0.9, 0.79]", + "height": "36px" + } + } + } +} \ No newline at end of file diff --git a/marketplace/plugins/qdrant/lib/query_operations.ts b/marketplace/plugins/qdrant/lib/query_operations.ts new file mode 100644 index 0000000000..02999a3906 --- /dev/null +++ b/marketplace/plugins/qdrant/lib/query_operations.ts @@ -0,0 +1,85 @@ +import { QueryOptions } from './types'; +import { QdrantClient } from '@qdrant/js-client-rest'; + +export async function getCollectionInfo(qdrant: QdrantClient, options: QueryOptions): Promise { + const { collectionName } = options; + + if (!collectionName) { + throw new Error('Collection name is required'); + } + + return await qdrant.getCollection(collectionName); +} + +export async function listCollections(qdrant: QdrantClient, options: QueryOptions): Promise { + console.log('here'); + const collections = (await qdrant.getCollections()).collections; + console.log({ collections }); + return collections.map((c) => c.name); +} + +export async function getPoints(qdrant: QdrantClient, options: QueryOptions): Promise { + const { collectionName, ids } = options; + + if (!collectionName) { + throw new Error('Collection name is required'); + } + + const points = await qdrant.retrieve(collectionName, { + ids: JSON.parse(ids), + with_payload: true, + with_vector: true, + }); + + return points; +} + +export async function upsertPoints(qdrant: QdrantClient, options: QueryOptions): Promise { + const { collectionName, points } = options; + + if (!collectionName) { + throw new Error('Collection name is required'); + } + + const response = await qdrant.upsert(collectionName, { + points: JSON.parse(points), + wait: true, + }); + return response.status; +} + +export async function deletePoints(qdrant: QdrantClient, options: QueryOptions): Promise { + const { collectionName, ids, filter } = options; + + if (!collectionName) { + throw new Error('Collection name is required'); + } + + const response = await qdrant.delete(collectionName, { + filter: filter ? JSON.parse(filter) : undefined, + points: ids ? JSON.parse(ids) : undefined, + wait: true, + }); + + return response.status; +} + +export async function queryPoints(qdrant: QdrantClient, options: QueryOptions): Promise { + const { collectionName, query, filter, limit, withPayload, withVectors } = options; + + console.log('OPTIONS ARE ' + JSON.stringify(options)); + + if (!collectionName) { + throw new Error('Collection name is required'); + } + + const response = await qdrant.query(collectionName, { + query: query ? JSON.parse(query) : null, + filter: filter ? JSON.parse(filter) : {}, + limit: limit ? JSON.parse(limit) : 10, + with_payload: withPayload.toLowerCase() === 'true', + with_vector: withVectors.toLowerCase() == 'true', + }); + + return response.points; +} diff --git a/marketplace/plugins/qdrant/lib/types.ts b/marketplace/plugins/qdrant/lib/types.ts new file mode 100644 index 0000000000..5923463f0f --- /dev/null +++ b/marketplace/plugins/qdrant/lib/types.ts @@ -0,0 +1,26 @@ +export type SourceOptions = { + apiKey: string; + url: string; +}; + +export type QueryOptions = { + operation: Operation; + collectionName: string; + ids?: string; + points?: string; + id?: string; + filter?: string; + limit?: string; + withPayload?: string; + withVectors?: string; + query?: string; +}; + +export enum Operation { + GetCollectionInfo = 'get_collection_info', + ListCollections = 'list_collections', + GetPoints = 'get_points', + UpsertPoints = 'upsert_points', + DeletePoints = 'delete_points', + QueryPoints = 'query_points', +} diff --git a/marketplace/plugins/qdrant/package.json b/marketplace/plugins/qdrant/package.json new file mode 100644 index 0000000000..3dd45dadbf --- /dev/null +++ b/marketplace/plugins/qdrant/package.json @@ -0,0 +1,27 @@ +{ + "name": "@tooljet-marketplace/qdrant", + "version": "1.0.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "directories": { + "lib": "lib", + "test": "__tests__" + }, + "files": [ + "lib" + ], + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1", + "build": "ncc build lib/index.ts -o dist", + "watch": "ncc build lib/index.ts -o dist --watch" + }, + "homepage": "https://github.com/tooljet/tooljet#readme", + "dependencies": { + "@qdrant/js-client-rest": "^1.12.0", + "@tooljet-marketplace/common": "^1.0.0" + }, + "devDependencies": { + "@vercel/ncc": "^0.38.3", + "typescript": "^4.7.4" + } +} diff --git a/marketplace/plugins/qdrant/tsconfig.json b/marketplace/plugins/qdrant/tsconfig.json new file mode 100644 index 0000000000..a18a801b14 --- /dev/null +++ b/marketplace/plugins/qdrant/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "lib" + }, + "exclude": [ + "node_modules", + "dist" + ] +} \ No newline at end of file diff --git a/marketplace/plugins/weaviate/.gitignore b/marketplace/plugins/weaviate/.gitignore new file mode 100644 index 0000000000..23e6609462 --- /dev/null +++ b/marketplace/plugins/weaviate/.gitignore @@ -0,0 +1,5 @@ +node_modules +lib/*.d.* +lib/*.js +lib/*.js.map +dist/* \ No newline at end of file diff --git a/marketplace/plugins/weaviate/README.md b/marketplace/plugins/weaviate/README.md new file mode 100644 index 0000000000..af717a5163 --- /dev/null +++ b/marketplace/plugins/weaviate/README.md @@ -0,0 +1,4 @@ + +# Weaviate + +Documentation on: https://docs.tooljet.com/docs/data-sources/weaviate \ No newline at end of file diff --git a/marketplace/plugins/weaviate/__tests__/index.js b/marketplace/plugins/weaviate/__tests__/index.js new file mode 100644 index 0000000000..c776089aa9 --- /dev/null +++ b/marketplace/plugins/weaviate/__tests__/index.js @@ -0,0 +1,7 @@ +'use strict'; + +const weaviate = require('../lib'); + +describe('weaviate', () => { + it.todo('needs tests'); +}); diff --git a/marketplace/plugins/weaviate/lib/icon.svg b/marketplace/plugins/weaviate/lib/icon.svg new file mode 100644 index 0000000000..f2149fe0ea --- /dev/null +++ b/marketplace/plugins/weaviate/lib/icon.svg @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/marketplace/plugins/weaviate/lib/index.ts b/marketplace/plugins/weaviate/lib/index.ts new file mode 100644 index 0000000000..ea810d92d4 --- /dev/null +++ b/marketplace/plugins/weaviate/lib/index.ts @@ -0,0 +1,69 @@ +import { QueryService, QueryResult, ConnectionTestResult, QueryError } from '@tooljet-marketplace/common'; +import { SourceOptions, QueryOptions } from './types'; +import { getSchema, collectionOperation, objectsOperation } from './query_operations'; +export default class Weaviate implements QueryService { + async run(sourceOptions: SourceOptions, queryOptions: QueryOptions, dataSourceId: string): Promise { + let result = {}; + try { + const headers = {}; + let BASE_URL; + if (sourceOptions.connection_type === 'local') { + BASE_URL = `http://${sourceOptions.host}:${sourceOptions.port}`; + } else { + BASE_URL = `${sourceOptions.instanceUrl}`; + headers['Authorization'] = `Bearer ${sourceOptions.apiKey}`; + } + switch (queryOptions.data_type) { + case 'schema': { + result = await getSchema(queryOptions, BASE_URL, headers); + break; + } + case 'collection': { + result = await collectionOperation(queryOptions, BASE_URL, headers); + break; + } + case 'objects': { + result = await objectsOperation(queryOptions, BASE_URL, headers); + break; + } + default: + throw new QueryError('Invalid operation', 'Operation not supported', {}); + } + } catch (error) { + const errorMessage = error.message || 'An unknown error occurred'; + const errorDetails: any = {}; + + if (error && error instanceof Error) { + const weaviateError = error as any; + errorDetails.message = weaviateError.message; + } + + throw new QueryError('Query could not be completed', errorMessage, errorDetails); + } + + return { status: 'ok', data: result }; + } + + async testConnection(sourceOptions: SourceOptions): Promise { + try { + let response; + if (sourceOptions.connection_type === 'local') { + response = await fetch(`http://${sourceOptions.host}:${sourceOptions.port}/v1/schema`, { + method: 'GET', + }); + } else { + response = await fetch(`${sourceOptions.instanceUrl}/v1/schema`, { + method: 'GET', + headers: { + Authorization: `Bearer ${sourceOptions.apiKey}`, + }, + }); + } + if (response.ok) { + return { status: 'ok' }; + } + } catch (error) { + throw new QueryError('Connection failed', error?.message, {}); + } + } +} diff --git a/marketplace/plugins/weaviate/lib/manifest.json b/marketplace/plugins/weaviate/lib/manifest.json new file mode 100644 index 0000000000..e1320d4119 --- /dev/null +++ b/marketplace/plugins/weaviate/lib/manifest.json @@ -0,0 +1,76 @@ +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/manifest.schema.json", + "title": "Weaviate Plugin", + "description": "A schema defining Weaviate datasource", + "type": "api", + "source": { + "name": "Weaviate", + "kind": "weaviate", + "exposedVariables": { + "isLoading": false, + "data": {}, + "rawData": {} + }, + "options": { + "instanceUrl": { + "type": "string" + }, + "apiKey": { + "type": "string", + "encrypted": true + } + } + }, + "defaults": { + "connection_type": { + "value": "cloud" + } + }, + "properties": { + "connection_type": { + "label": "Connection type", + "key": "connection_type", + "type": "dropdown-component-flip", + "description": "Single select dropdown for connection type", + "list": [ + { "value": "cloud", "name": "Cloud" }, + { "value": "local", "name": "Local" } + ] + }, + "cloud": { + "instanceUrl": { + "label": "Instance URLs", + "key": "instanceUrl", + "type": "text", + "description": "Enter your Weaviate instance URL", + "placeholder": "https://your-weaviate-instance.com" + }, + "apiKey": { + "label": "API Key", + "key": "apiKey", + "type": "password", + "description": "Enter your Weaviate API Key", + "helpText": "For generating API Key, visit: Weaviate Documentation", + "encrypted": true + } + }, + "local": { + "host": { + "label": "Host", + "key": "host", + "type": "text", + "description": "Enter host" + }, + "port": { + "label": "Port", + "key": "port", + "type": "text", + "description": "Enter port" + } + } + }, + "required": [ + "instanceUrl", + "apiKey" + ] +} \ No newline at end of file diff --git a/marketplace/plugins/weaviate/lib/operations.json b/marketplace/plugins/weaviate/lib/operations.json new file mode 100644 index 0000000000..16e6315dc1 --- /dev/null +++ b/marketplace/plugins/weaviate/lib/operations.json @@ -0,0 +1,506 @@ + +{ + "$schema": "https://raw.githubusercontent.com/ToolJet/ToolJet/develop/plugins/schemas/operations.schema.json", + "title": "Weaviate datasource", + "description": "A schema defining Weaviate datasource", + "type": "database", + "defaults": { + "data_type": "schema", + "operation_schema": "get_schema", + "operation_collection": "get_collection", + "operation_objects": "list_objects" + }, + "properties": { + "data_type": { + "label": "Data type", + "key": "data_type", + "type": "dropdown-component-flip", + "description": "Single select dropdown for data type", + "list": [ + { + "value": "schema", + "name": "Schema" + }, + { + "value": "collection", + "name": "Collection" + }, + { + "value": "objects", + "name": "Objects" + } + ] + }, + "schema": { + "operation_schema": { + "label": "Operation", + "key": "operation_schema", + "type": "dropdown-component-flip", + "description": "Single select dropdown for operation", + "list": [ + { + "value": "get_schema", + "name": "Get database schema" + } + ] + }, + "get_schema": { + "consistency": { + "label": "Consistency", + "key": "consistency", + "type": "toggle", + "value": "true", + "tooltip":"Enable consistency" + } + } + }, + "collection": { + "operation_collection": { + "label": "Operation", + "key": "operation_collection", + "type": "dropdown-component-flip", + "description": "Single select dropdown for operation", + "list": [ + { + "value": "get_collection", + "name": "Get Collection" + }, + { + "value": "create_collection", + "name": "Create Collection" + }, + { + "value": "delete_collection", + "name": "Delete Collection" + } + ] + }, + "get_collection": { + "collectionName": { + "label": "Collection Name", + "key": "collectionName", + "type": "codehinter", + "lineNumbers": false, + "description": "Enter the name of the collection", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "Enter the name of the collection", + "tooltip":"Enter the name of the collection" + }, + "consistency": { + "label": "Consistency", + "key": "consistency", + "type": "toggle", + "value": "true", + "tooltip":"Enable consistency" + } + }, + "create_collection": { + "collectionName": { + "label": "Collection Name", + "key": "collectionName", + "type": "codehinter", + "lineNumbers": false, + "description": "Enter the name of the collection", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "Enter the name of the collection" + }, + "vectorizer": { + "label": "Vectorizer", + "key": "vectorizer", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "{}" + }, + "vector_index_type": { + "label": "Vector index type", + "key": "vector_index_type", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "Value" + }, + "vector_index_config": { + "label": "Vector index config", + "key": "vector_index_config", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "{}" + }, + "sharding_config": { + "label": "Sharding config", + "key": "sharding_config", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "{}" + }, + "factor": { + "label": "Factor", + "key": "factor", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "1" + }, + "async_enabled": { + "label": "Async enabled", + "key": "async_enabled", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "true" + }, + "deletion_strategy": { + "label": "Deletion strategy", + "key": "deletion_strategy", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "NoAutomatedResolution" + }, + "clean_up_interval_seconds": { + "label": "Cleanup interval seconds", + "key": "clean_up_interval_seconds", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "1" + }, + "bm_25": { + "label": "Bm 25", + "key": "bm_25", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "{“k1”:1,”b”:1}" + }, + "stop_words": { + "label": "Stop words", + "key": "stop_words", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "{“preset”:””,”additions”:[“”],”removals”:[“”]}" + }, + "index_time_stamps": { + "label": "Index time stamps", + "key": "index_time_stamps", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "true" + }, + "index_null_state": { + "label": "Index null state", + "key": "index_null_state", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "true" + }, + "index_property_length": { + "label": "Index property length", + "key": "index_property_length", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "true" + }, + "module_config": { + "label": "Module config", + "key": "module_config", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "{}" + }, + "description": { + "label": "Description", + "key": "description", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "{}" + }, + "consistency": { + "label": "Consistency", + "key": "consistency", + "type": "toggle", + "value": "true", + "tooltip":"Enable consistency" + }, + "properties": { + "label": "Properties", + "key": "properties", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "256px", + "className": "codehinter-plugins", + "placeholder": "[\n\t{\n\t\t\"dataType\": [\"\"],\n\t\t\"description\": \"\",\n\t\t\"moduleConfig\": {},\n\t\t\"name\": \"\",\n\t\t\"indexInverted\": true,\n\t\t\"indexFilterable\": true,\n\t\t\"indexSearchable\": true,\n\t\t\"indexRangeFilters\": true,\n\t\t\"tokenization\": \"word\",\n\t\t\"nestedProperties\": []\n\t}\n]" + } + }, + "delete_collection": { + "collectionName": { + "label": "Collection Name", + "key": "collectionName", + "type": "codehinter", + "lineNumbers": false, + "description": "Enter the name of the collection", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "Enter the name of the collection" + } + } + }, + "objects": { + "operation_objects": { + "label": "Operation", + "key": "operation_objects", + "type": "dropdown-component-flip", + "description": "Single select dropdown for operation", + "list": [ + { + "value": "list_objects", + "name": "List Objects" + }, + { + "value": "create_object", + "name": "Create Object" + }, + { + "value": "get_object_by_id", + "name": "Get Object By Id" + }, + { + "value": "delete_object_by_id", + "name": "Delete Object By Id" + } + ] + }, + "get_object_by_id": { + "collectionName_get_object": { + "label": "Collection Name", + "key": "collectionName_get_object", + "type": "codehinter", + "lineNumbers": false, + "description": "Enter the name of the collection", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "Enter the name of the collection" + }, + "objectId_get_object": { + "label": "Object ID", + "key": "objectId_get_object", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "Enter the ID of the object" + } + }, + "delete_object_by_id": { + "collectionName_delete_object": { + "label": "Collection Name", + "key": "collectionName_delete_object", + "type": "codehinter", + "lineNumbers": false, + "description": "Enter the name of the collection", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "Enter the name of the collection" + }, + "objectId_delete_object": { + "label": "Object ID", + "key": "objectId_delete_object", + "type": "codehinter", + "lineNumbers": false, + "description": "Enter the ID of the object", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "Enter the ID of the object" + } + }, + "list_objects": { + "collectionName": { + "label": "Collection Name", + "key": "collectionName", + "type": "codehinter", + "lineNumbers": false, + "description": "Enter the name of the collection", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "Enter the name of the collection" + }, + "include_vectors": { + "label": "Include vectors", + "key": "include_vectors", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "true or ['title','review_body']" + }, + "after": { + "label": "After", + "key": "after", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "{}" + }, + "offset": { + "label": "Offset", + "key": "offset", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "{}" + }, + "limit": { + "label": "Limit", + "key": "limit", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "10" + }, + "include": { + "label": "Include", + "key": "include", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "value1, value2" + }, + "sort": { + "label": "Sort", + "key": "sort", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "value1, value2" + }, + "order": { + "label": "Order", + "key": "order", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "value1, value2" + }, + "tenant": { + "label": "Tenant", + "key": "tenant", + "type": "codehinter", + "lineNumbers": false, + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "value" + } + }, + "create_object": { + "collectionName_create_object": { + "label": "Collection Name", + "key": "collectionName_create_object", + "type": "codehinter", + "lineNumbers": false, + "description": "Enter the name of the collection", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "ArticleAuthor" + }, + "object_uuid": { + "label": "Object uuid", + "key": "object_uuid", + "type": "codehinter", + "lineNumbers": false, + "description": "Enter the name of the collection", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "ed89d9e7-4c9d-4a6a-8d20-095cb0026f54" + }, + "properties_create_object": { + "label": "Properties", + "key": "properties_create_object", + "type": "codehinter", + "lineNumbers": false, + "description": "Enter the name of the collection", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "{\n 'question': 'This vector DB is OSS & supports automatic property type inference on import',\n 'newProperty': 123\n}" + }, + "vectors_create_object": { + "label": "Vectors", + "key": "vectors_create_object", + "type": "codehinter", + "lineNumbers": false, + "description": "Enter the name of the collection", + "width": "320px", + "height": "36px", + "className": "codehinter-plugins", + "placeholder": "{\n title: Array(1536).fill(0.12345),\n review_body: Array(1536).fill(0.31313),\n title_country: Array(1536).fill(0.05050)\n}" + } + } + } + } +} diff --git a/marketplace/plugins/weaviate/lib/query_operations.ts b/marketplace/plugins/weaviate/lib/query_operations.ts new file mode 100644 index 0000000000..851ebaf5ba --- /dev/null +++ b/marketplace/plugins/weaviate/lib/query_operations.ts @@ -0,0 +1,239 @@ +/* eslint-disable no-useless-catch */ +import { CollectionOperation, ObjectsOperation, QueryOptions } from './types'; + +export async function getSchema(queryOptions: QueryOptions, BASE_URL, headers) { + if (queryOptions.consistency) { + headers.consistency = queryOptions.consistency; + } + try { + const response = await fetch(`${BASE_URL}/v1/schema`, { + method: 'GET', + headers: headers, + }); + if (!response.ok) { + throw new Error(`HTTP error in fetching schema! status: ${response.status}`); + } + return await response.json(); + } catch (error) { + throw error; + } +} + +export async function collectionOperation(queryOptions: QueryOptions, BASE_URL, headers) { + const SCHEMA_URL = `${BASE_URL}/v1/schema`; + switch (queryOptions.operation_collection) { + case CollectionOperation.get_collection: { + return await getCollection(queryOptions.collectionName, queryOptions.consistency, SCHEMA_URL, headers); + } + case CollectionOperation.create_collection: { + return await createCollection(queryOptions, SCHEMA_URL, headers); + } + case CollectionOperation.delete_collection: { + return await deleteCollection(queryOptions.collectionName, SCHEMA_URL, headers); + } + } +} + +export async function getCollection(collectionName: string, consistency: boolean, SCHEMA_URL, headers) { + try { + if (consistency) { + headers.consistency = consistency; + } + const response = await fetch(`${SCHEMA_URL}/${collectionName}`, { + method: 'GET', + headers: headers, + }); + if (!response.ok) { + throw new Error(`HTTP error in getting collection! status: ${response.status}`); + } + return await response.json(); + } catch (error) { + throw error; + } +} + +export async function createCollection(queryOptions: QueryOptions, SCHEMA_URL, headers) { + const collectionConfig = { + class: queryOptions.collectionName, + vectorizer: queryOptions.vectorizer, + vectorIndexType: queryOptions.vector_index_type, + vectorIndexConfig: JSON.parse(queryOptions.vector_index_config), + shardingConfig: JSON.parse(queryOptions.sharding_config), + replicationConfig: { + factor: Number(queryOptions.factor), + asyncEnabled: Boolean(queryOptions.async_enabled), + deletionStrategy: queryOptions.deletion_strategy, + }, + invertedIndexConfig: { + cleanupIntervalSeconds: Number(queryOptions.clean_up_interval_seconds), + bm25: JSON.parse(queryOptions.bm_25), + stopwords: JSON.parse(queryOptions.stop_words), + indexTimestamps: Boolean(queryOptions.index_time_stamps), + indexNullState: Boolean(queryOptions.index_null_state), + indexPropertyLength: Boolean(queryOptions.index_property_length), + }, + moduleConfig: JSON.parse(queryOptions.module_config), + description: queryOptions.description, + properties: JSON.parse(queryOptions.properties), + }; + + try { + const response = await fetch(SCHEMA_URL, { + method: 'POST', + headers: { + ...headers, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(collectionConfig), + }); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + return await response.json(); + } catch (error) { + throw error; + } +} + +export async function deleteCollection(collectionName: string, SCHEMA_URL, headers) { + try { + const response = await fetch(`${SCHEMA_URL}/${collectionName}`, { + method: 'DELETE', + headers: { + ...headers, + 'Content-Type': 'application/json', + }, + }); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + return true; + } catch (error) { + throw error; + } +} + +export async function objectsOperation(queryOptions: QueryOptions, BASE_URL, headers) { + const OBJECT_URL = `${BASE_URL}/v1/objects`; + switch (queryOptions.operation_objects) { + case ObjectsOperation.create_object: { + return await createObject(queryOptions, OBJECT_URL, headers); + } + case ObjectsOperation.get_object_by_id: { + return await getObjectById(queryOptions, OBJECT_URL, headers); + } + case ObjectsOperation.list_objects: { + return await listObjects(queryOptions, OBJECT_URL, headers); + } + case ObjectsOperation.delete_object_by_id: { + return await deleteObjectById(queryOptions, OBJECT_URL, headers); + } + } +} + +export async function listObjects(queryOptions: QueryOptions, OBJECT_URL, headers) { + if (!queryOptions.collectionName) throw new Error('Collection name is required'); + const params = new URLSearchParams(); + params.append('class', queryOptions.collectionName); + const paramMapping = { + include_vectors: (val: string) => val === 'true' || JSON.parse(val), + after: String, + offset: Number, + limit: Number, + include: (val: string) => val.split(','), + sort: (val: string) => val.split(','), + order: (val: string) => val.split(','), + tenant: String, + }; + try { + Object.entries(paramMapping).forEach(([key, converter]) => { + if (queryOptions[key] && queryOptions[key] !== '{}' && queryOptions[key] !== '') { + const value = converter(queryOptions[key]); + if (value) params.append(key, String(value)); + } + }); + const response = await fetch(`${OBJECT_URL}?${params.toString()}`, { + method: 'GET', + headers: headers, + }); + if (!response.ok) { + throw new Error(`HTTP error in listing objects! status: ${response.status}`); + } + return await response.json(); + } catch (error) { + throw error; + } +} + +export async function createObject(queryOptions: QueryOptions, OBJECT_URL, headers) { + try { + const requestBody = { + class: queryOptions.collectionName_create_object, + properties: JSON.parse(queryOptions.properties_create_object), + id: queryOptions.object_uuid, + }; + + if (queryOptions.vectors_create_object) { + if (Array.isArray(queryOptions.vectors_create_object)) { + requestBody['vector'] = queryOptions.vectors_create_object; + } else { + requestBody['vectors'] = queryOptions.vectors_create_object; + } + } + + const response = await fetch(`${OBJECT_URL}`, { + method: 'POST', + headers: { + ...headers, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(requestBody), + }); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + return await response.json(); + } catch (error) { + throw error; + } +} + +export async function getObjectById(queryOptions: QueryOptions, OBJECT_URL, headers) { + const { collectionName_get_object: collection, objectId_get_object: uuid } = queryOptions; + try { + const response = await fetch(`${OBJECT_URL}/${collection}/${uuid}`, { + method: 'GET', + headers: headers, + }); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + return await response.json(); + } catch (error) { + throw error; + } +} + +export async function deleteObjectById(queryOptions: QueryOptions, OBJECT_URL, headers) { + const { collectionName_delete_object: collection, objectId_delete_object: uuid } = queryOptions; + try { + const response = await fetch(`${OBJECT_URL}/${collection}/${uuid}`, { + method: 'DELETE', + headers: { + ...headers, + 'Content-Type': 'application/json', + }, + }); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + return true; + } catch (error) { + throw error; + } +} diff --git a/marketplace/plugins/weaviate/lib/types.ts b/marketplace/plugins/weaviate/lib/types.ts new file mode 100644 index 0000000000..fa1484de88 --- /dev/null +++ b/marketplace/plugins/weaviate/lib/types.ts @@ -0,0 +1,64 @@ +export interface SourceOptions { + instanceUrl: string; + apiKey: string; + connection_type?: string; + host?: string; + port?: string; +} + +export interface QueryOptions { + data_type: string; + operation_schema: SchemaOperation; + operation_collection: CollectionOperation; + operation_objects: ObjectsOperation; + collectionName?: string; + consistency?: boolean; + objectId?: string; + properties?: string; + vectorizer?: string; + vector_index_type?: string; + vector_index_config?: string; + sharding_config?: string; + factor?: string; + async_enabled?: string; + clean_up_interval_seconds?: string; + bm_25?: string; + deletion_strategy?: string; + stop_words?: string; + index_time_stamps?: string; + index_null_state?: string; + index_property_length?: string; + module_config?: string; + description?: string; + include_vectors?: string; + after?: string; + offset?: string; + limit?: string; + include?: string; + sort?: string; + tenant?: string; + collectionName_create_object?: string; + object_uuid?: string; + properties_create_object?: string; + vectors_create_object?: string; + references?: string; + collectionName_delete_object?: string; + objectId_delete_object?: string; + collectionName_get_object?: string; + objectId_get_object?: string; +} + +export enum SchemaOperation { + get_schema = 'get_schema', +} +export enum CollectionOperation { + get_collection = 'get_collection', + create_collection = 'create_collection', + delete_collection = 'delete_collection', +} +export enum ObjectsOperation { + list_objects = 'list_objects', + create_object = 'create_object', + get_object_by_id = 'get_object_by_id', + delete_object_by_id = 'delete_object_by_id', +} diff --git a/marketplace/plugins/weaviate/package.json b/marketplace/plugins/weaviate/package.json new file mode 100644 index 0000000000..09865b989b --- /dev/null +++ b/marketplace/plugins/weaviate/package.json @@ -0,0 +1,27 @@ +{ + "name": "@tooljet-marketplace/weaviate", + "version": "1.0.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "directories": { + "lib": "lib", + "test": "__tests__" + }, + "files": [ + "lib" + ], + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1", + "build": "ncc build lib/index.ts -o dist", + "watch": "ncc build lib/index.ts -o dist --watch" + }, + "homepage": "https://github.com/tooljet/tooljet#readme", + "dependencies": { + "@tooljet-marketplace/common": "^1.0.0", + "got": "^11.8.6" + }, + "devDependencies": { + "@vercel/ncc": "^0.34.0", + "typescript": "^4.7.4" + } +} diff --git a/marketplace/plugins/weaviate/tsconfig.json b/marketplace/plugins/weaviate/tsconfig.json new file mode 100644 index 0000000000..a18a801b14 --- /dev/null +++ b/marketplace/plugins/weaviate/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "lib" + }, + "exclude": [ + "node_modules", + "dist" + ] +} \ No newline at end of file From 2ef356938cb8b23c8acc88492e001efd91dd1acd Mon Sep 17 00:00:00 2001 From: Anantshree Chandola Date: Mon, 3 Mar 2025 19:50:19 +0530 Subject: [PATCH 12/42] fix invite (#12096) --- server/src/modules/onboarding/controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/modules/onboarding/controller.ts b/server/src/modules/onboarding/controller.ts index b584cdd63d..42252d9476 100644 --- a/server/src/modules/onboarding/controller.ts +++ b/server/src/modules/onboarding/controller.ts @@ -29,7 +29,7 @@ export class OnboardingController implements IOnboardingController { constructor(protected onboardingService: OnboardingService) {} @InitFeature(FEATURE_KEY.ACTIVATE_ACCOUNT) - @UseGuards(SignupDisableGuard, FirstUserSignupDisableGuard, FeatureAbilityGuard) + @UseGuards(FirstUserSignupDisableGuard, FeatureAbilityGuard) @Post('activate-account-with-token') async activateAccountWithToken( @Body() activateAccountDto: ActivateAccountWithTokenDto, From c5a06423ef9fed9517ac29bdf128ebdbd39e5952 Mon Sep 17 00:00:00 2001 From: Adish M <44204658+adishM98@users.noreply.github.com> Date: Tue, 4 Mar 2025 11:11:59 +0530 Subject: [PATCH 13/42] fix ee/server (#12101) --- server/ee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/ee b/server/ee index 88dcd4e30b..615fba88c5 160000 --- a/server/ee +++ b/server/ee @@ -1 +1 @@ -Subproject commit 88dcd4e30b5e78df1b873189c8416dd26edcaa1c +Subproject commit 615fba88c5a631ea5d2af76470ffdff234fba001 From 3828c93e23536a38e4aec945639a6ba6688dfbd1 Mon Sep 17 00:00:00 2001 From: Muhsin Shah C P Date: Tue, 4 Mar 2025 20:18:30 +0530 Subject: [PATCH 14/42] Copying reset password changes (#12108) --- server/src/modules/auth/oauth/service.ts | 5 +++++ server/src/modules/auth/service.ts | 1 + 2 files changed, 6 insertions(+) diff --git a/server/src/modules/auth/oauth/service.ts b/server/src/modules/auth/oauth/service.ts index 8c05d07a35..b289f64628 100644 --- a/server/src/modules/auth/oauth/service.ts +++ b/server/src/modules/auth/oauth/service.ts @@ -320,6 +320,11 @@ export class OauthService implements IOAuthService { organizationDetails = await this.organizationRepository.fetchOrganization(userDetails.defaultOrganizationId); } + // Clear forgot password token + if (userDetails.forgotPasswordToken) { + await this.userRepository.updateOne(userDetails.id, { forgotPasswordToken: null }, manager); + } + return await this.sessionUtilService.generateLoginResultPayload( response, userDetails, diff --git a/server/src/modules/auth/service.ts b/server/src/modules/auth/service.ts index 7470d6f39d..5175197ceb 100644 --- a/server/src/modules/auth/service.ts +++ b/server/src/modules/auth/service.ts @@ -109,6 +109,7 @@ export class AuthService implements IAuthService { const updateData = { ...(shouldUpdateDefaultOrgId && { defaultOrganizationId: organization?.id }), passwordRetryCount: 0, + forgotPasswordToken: null, }; await this.userRepository.updateOne(user.id, updateData, manager); From c2c0018eb73e31e1bf7216c171f1cc2febebdac3 Mon Sep 17 00:00:00 2001 From: Nakul Nagargade Date: Tue, 4 Feb 2025 16:12:47 +0530 Subject: [PATCH 15/42] Fix page handle being wrong when fetching from exposed values in Viewer on initial reload --- frontend/src/AppBuilder/_hooks/useAppData.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frontend/src/AppBuilder/_hooks/useAppData.js b/frontend/src/AppBuilder/_hooks/useAppData.js index a8fac45c48..1b664cfa21 100644 --- a/frontend/src/AppBuilder/_hooks/useAppData.js +++ b/frontend/src/AppBuilder/_hooks/useAppData.js @@ -329,6 +329,11 @@ const useAppData = (appId, moduleId, darkMode, mode = 'edit', { environmentId, v setCurrentPageHandle(startingPage.handle); updateFeatureAccess(); setCurrentPageId(startingPage.id, moduleId); + setResolvedPageConstants({ + id: startingPage?.id, + handle: startingPage?.handle, + name: startingPage?.name, + }); setComponentNameIdMapping(moduleId); updateEventsField('events', appData.events); setCurrentVersionId(appData.editing_version?.id || appData.current_version_id); From 20f03d6e1142a2fa3c7c021ad457ee7cece13c25 Mon Sep 17 00:00:00 2001 From: Nakul Nagargade Date: Wed, 5 Feb 2025 19:21:15 +0530 Subject: [PATCH 16/42] Fix alignment issues arising due to label width setting for input components --- .../WidgetManager/widgets/listview.js | 8 ++- .../Editor/WidgetManager/configs/listview.js | 8 ++- frontend/src/_styles/table-component.scss | 55 ++++++++++++++++++- .../apps/services/widget-config/listview.js | 2 +- 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/frontend/src/AppBuilder/WidgetManager/widgets/listview.js b/frontend/src/AppBuilder/WidgetManager/widgets/listview.js index a813bb5a0b..fec2e812b4 100644 --- a/frontend/src/AppBuilder/WidgetManager/widgets/listview.js +++ b/frontend/src/AppBuilder/WidgetManager/widgets/listview.js @@ -48,8 +48,12 @@ export const listviewConfig = { data: { type: 'code', displayName: 'List data', - validation: { - schema: { type: 'array', element: { type: 'object' } }, + schema: { + type: 'union', + schemas: [ + { type: 'array', element: { type: 'object' } }, + { type: 'array', element: { type: 'string' } }, + ], defaultValue: "[{text: 'Sample text 1'}]", }, }, diff --git a/frontend/src/Editor/WidgetManager/configs/listview.js b/frontend/src/Editor/WidgetManager/configs/listview.js index a813bb5a0b..25da8f73c6 100644 --- a/frontend/src/Editor/WidgetManager/configs/listview.js +++ b/frontend/src/Editor/WidgetManager/configs/listview.js @@ -49,7 +49,13 @@ export const listviewConfig = { type: 'code', displayName: 'List data', validation: { - schema: { type: 'array', element: { type: 'object' } }, + schema: { + type: 'union', + schemas: [ + { type: 'array', element: { type: 'object' } }, + { type: 'array', element: { type: 'string' } }, + ], + }, defaultValue: "[{text: 'Sample text 1'}]", }, }, diff --git a/frontend/src/_styles/table-component.scss b/frontend/src/_styles/table-component.scss index 5e0edd5aff..100cb450cc 100644 --- a/frontend/src/_styles/table-component.scss +++ b/frontend/src/_styles/table-component.scss @@ -1467,5 +1467,56 @@ } .tj-table-tag-col-readonly { - margin-left: -2px !important; //this -ve margin offset for the margin given to each tags in overall column width -} \ No newline at end of file + margin-left: -2px !important; //this -ve margin offset for the margin given to each tags in overall column width +} + +.jet-data-table { + .table-bordered { + th, + td { + border-bottom: 1px solid var(--interactive-overlay-border-pressed) !important; + border-right: 1px solid var(--interactive-overlay-border-pressed) !important; + + &:first-child { + border-left: none !important; + } + + &:last-child { + border-right: none !important; + } + } + + thead th { + border-top: none !important; + + &:first-child { + border-left: none !important; + } + + &:last-child { + border-right: none !important; + } + } + } + + .table-striped { + tbody { + div[data-index]:nth-child(odd) { + background-color: transparent !important; + } + + div[data-index]:nth-child(even) { + background-color: var(--slate2) !important; + } + } + } +} + +// hide scrollbar on touch devices + @media (hover: none) and (pointer: coarse) { + .jet-data-table::-webkit-scrollbar { + width: 0; + height: 0; + background: transparent; + } + } diff --git a/server/src/modules/apps/services/widget-config/listview.js b/server/src/modules/apps/services/widget-config/listview.js index a813bb5a0b..d710babbf0 100644 --- a/server/src/modules/apps/services/widget-config/listview.js +++ b/server/src/modules/apps/services/widget-config/listview.js @@ -49,7 +49,7 @@ export const listviewConfig = { type: 'code', displayName: 'List data', validation: { - schema: { type: 'array', element: { type: 'object' } }, + schema: { type: 'union', schemas: [{ type: 'array', element: { type: 'object' } },{ type: 'array', element: { type: 'string' } }] }, defaultValue: "[{text: 'Sample text 1'}]", }, }, From cbfe197a28b1f4d545325ad8d91b1fcd5fe962d7 Mon Sep 17 00:00:00 2001 From: Nakul Nagargade Date: Thu, 6 Feb 2025 11:03:51 +0530 Subject: [PATCH 17/42] fi --- frontend/src/AppBuilder/Widgets/Container.jsx | 5 +---- frontend/src/_styles/components.scss | 9 +++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/frontend/src/AppBuilder/Widgets/Container.jsx b/frontend/src/AppBuilder/Widgets/Container.jsx index 3ccedd869b..16623280a5 100644 --- a/frontend/src/AppBuilder/Widgets/Container.jsx +++ b/frontend/src/AppBuilder/Widgets/Container.jsx @@ -45,7 +45,6 @@ export const Container = ({ border: `1px solid ${borderColor}`, height, display: isVisible ? 'flex' : 'none', - overflow: 'hidden auto', position: 'relative', boxShadow, }; @@ -66,9 +65,7 @@ export const Container = ({ return (