diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8dfb1e3715..cdd0be3fc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,7 +123,9 @@ jobs: ${{ runner.os }}- - run: apt update && apt install -y postgresql - run: npm i -g npm@7.20.0 - - run: npm --prefix plugins ci && npm --prefix plugins run build:packages && npm --prefix plugins run build:server + - run: npm --prefix plugins ci + - run: npm --prefix plugins run create:client && npm --prefix plugins run create:server + - run: npm --prefix plugins run build:packages && npm --prefix plugins run build:server - run: npm --prefix server ci - run: npm --prefix server run db:create - run: npm --prefix server run db:migrate @@ -160,7 +162,9 @@ jobs: ${{ runner.os }}- - run: apt update && apt install -y postgresql - run: npm i -g npm@7.20.0 - - run: npm --prefix plugins ci && npm --prefix plugins run build:packages && npm --prefix plugins run build:server + - run: npm --prefix plugins ci + - run: npm --prefix plugins run create:client && npm --prefix plugins run create:server + - run: npm --prefix plugins run build:packages && npm --prefix plugins run build:server - run: npm --prefix server ci - run: npm --prefix server run db:create - run: npm --prefix server run db:migrate diff --git a/docker-compose.yaml b/docker-compose.yaml index 7f7854181f..b43c674593 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,16 @@ version: "3" services: + plugins: + build: + context: ./ + dockerfile: ./docker/plugins.Dockerfile.dev + image: tooljet-plugins:development + platform: linux/x86_64 + volumes: + - plugins_vol:/app/plugins + command: npm run --prefix plugins start + client: build: context: ./ @@ -9,9 +19,7 @@ services: platform: linux/x86_64 volumes: - ./frontend:/app/frontend:delegated - - ./plugins/:/app/plugins - - /app/plugins/dist/ - - /app/plugins/node_modules/ + - plugins_vol:/app/plugins - /app/frontend/node_modules/ ports: - 8082:8082 @@ -30,12 +38,10 @@ services: - postgres volumes: - ./server:/app/server:delegated - - ./plugins/:/app/plugins - - /app/plugins/dist/ - - /app/plugins/node_modules/ + - plugins_vol:/app/plugins - /app/server/node_modules/ - - ".env:/app/.env" - - ".env.test:/app/.env.test" + - ./.env:/app/.env + - ./.env.test:/app/.env.test ports: - 3000:3000 environment: @@ -55,3 +61,8 @@ services: volumes: postgres: + plugins_vol: + driver_opts: + type: none + device: ${PWD}/plugins + o: bind diff --git a/docker/client.Dockerfile.dev b/docker/client.Dockerfile.dev index ddd9d7ff91..b998b39d42 100644 --- a/docker/client.Dockerfile.dev +++ b/docker/client.Dockerfile.dev @@ -16,12 +16,6 @@ ENV PATH /app/node_modules/.bin:$PATH # Fix for heap limit allocation issue ENV NODE_OPTIONS="--max-old-space-size=4096" -# Build plugins -COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/ -RUN npm --prefix plugins install -COPY ./plugins/ ./plugins/ -RUN npm run build:plugins - # install app dependencies COPY ./frontend/package.json ./frontend/package-lock.json ./frontend/ RUN npm --prefix frontend install diff --git a/docker/plugins.Dockerfile.dev b/docker/plugins.Dockerfile.dev new file mode 100644 index 0000000000..4391d949f1 --- /dev/null +++ b/docker/plugins.Dockerfile.dev @@ -0,0 +1,17 @@ +# pull official base image +FROM node:14.17.3-alpine AS builder + +RUN npm i -g npm@7.20.0 + +# set working directory +WORKDIR /app + +COPY ./package.json ./package.json + +# Fix for heap limit allocation issue +ENV NODE_OPTIONS="--max-old-space-size=4096" + +# Build plugins +COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/ +RUN npm --prefix plugins install +COPY ./plugins/ ./plugins/ \ No newline at end of file diff --git a/docker/server.Dockerfile.dev b/docker/server.Dockerfile.dev index 7149265cf7..dfdf50d57a 100644 --- a/docker/server.Dockerfile.dev +++ b/docker/server.Dockerfile.dev @@ -11,12 +11,6 @@ WORKDIR /app COPY ./package.json ./package.json -# Build plugins -COPY ./plugins/package.json ./plugins/package-lock.json ./plugins/ -RUN npm --prefix plugins install -COPY ./plugins/ ./plugins/ -RUN npm run build:plugins - # install app dependencies COPY ./server/package.json ./server/package-lock.json ./server/ RUN npm --prefix server install diff --git a/docs/docs/contributing-guide/setup/docker.md b/docs/docs/contributing-guide/setup/docker.md index ceddd96b6a..a4082bb0ec 100644 --- a/docs/docs/contributing-guide/setup/docker.md +++ b/docs/docs/contributing-guide/setup/docker.md @@ -80,6 +80,7 @@ docker-compose version 1.26.2, build eefe0d31 ```bash docker-compose build + docker-compose run --rm plugins npm run build:plugins ``` 5. Run ToolJet @@ -91,7 +92,7 @@ docker-compose version 1.26.2, build eefe0d31 6. ToolJet server is built using NestJS and the data such as application definitions are persisted on a postgres database. You can run the below command to seed the database. ```bash - docker-compose run --rm server npm run db:seed + docker-compose exec server npm run db:seed ``` 7. ToolJet should now be served locally at `http://localhost:8082`. You can login using the default user created. diff --git a/package.json b/package.json index 3e169e7497..8784112878 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ }, "scripts": { "prebuild:plugins": "npm --prefix plugins install", - "build:plugins": "npm --prefix plugins run build:packages && npm --prefix plugins run build:server", + "build:plugins": "npm --prefix plugins run build", "prebuild:frontend": "npm --prefix frontend install", "build:frontend": "NODE_ENV=production npm --prefix frontend run build", "prebuild:server": "npm --prefix server install", diff --git a/plugins/package-lock.json b/plugins/package-lock.json index c663b6ff4e..91e3e0bce7 100644 --- a/plugins/package-lock.json +++ b/plugins/package-lock.json @@ -5284,9 +5284,6 @@ "version": "4.0.0", "license": "MIT", "peer": true, - "dependencies": { - "graceful-fs": "^4.1.6" - }, "optionalDependencies": { "graceful-fs": "^4.1.6" } @@ -6474,8 +6471,7 @@ "esprima": "^4.0.1", "estraverse": "^5.2.0", "esutils": "^2.0.2", - "optionator": "^0.8.1", - "source-map": "~0.6.1" + "optionator": "^0.8.1" }, "bin": { "escodegen": "bin/escodegen.js", @@ -7337,7 +7333,6 @@ "minimist": "^1.2.5", "neo-async": "^2.6.0", "source-map": "^0.6.1", - "uglify-js": "^3.1.4", "wordwrap": "^1.0.0" }, "bin": { @@ -8485,7 +8480,6 @@ "@types/node": "*", "anymatch": "^3.0.3", "fb-watchman": "^2.0.0", - "fsevents": "^2.3.2", "graceful-fs": "^4.2.4", "jest-regex-util": "^27.4.0", "jest-serializer": "^27.4.0", @@ -8996,7 +8990,6 @@ "dev": true, "license": "MIT", "dependencies": { - "graceful-fs": "^4.1.6", "universalify": "^2.0.0" }, "optionalDependencies": { @@ -9982,7 +9975,6 @@ "dev": true, "license": "MIT", "dependencies": { - "encoding": "^0.1.12", "minipass": "^3.1.0", "minipass-sized": "^1.0.3", "minizlib": "^2.0.0" @@ -10084,8 +10076,7 @@ "dependencies": { "bson": "^4.6.0", "denque": "^2.0.1", - "mongodb-connection-string-url": "^2.3.2", - "saslprep": "^1.0.3" + "mongodb-connection-string-url": "^2.3.2" }, "engines": { "node": ">=12.9.0" @@ -13372,7 +13363,6 @@ "version": "1.1.5", "license": "MIT", "dependencies": { - "@zxing/text-encoding": "0.9.0", "util": "^0.12.3" }, "optionalDependencies": { diff --git a/plugins/package.json b/plugins/package.json index fb99cc37f7..d2424b23b7 100644 --- a/plugins/package.json +++ b/plugins/package.json @@ -5,12 +5,12 @@ "types": "dist/server.d.ts", "main": "dist/server.js", "scripts": { - "clean": "rimraf dist/", "create:client": "node create-client-entry", "create:server": "node create-server-entry", - "prebuild:server": "npm run clean", - "build:server": "tsc -b", - "prebuild:packages": "npm run clean:packages && npm run create:client && npm run create:server", + "build:server": "rimraf dist/ && tsc -b", + "start": "tsc -b --watch", + "build": "npm run create:client && npm run create:server && npm run build:packages && npm run build:server", + "prebuild:packages": "npm run clean:packages", "build:packages": "npx lerna run build --stream", "clean:packages": "npx lerna run clean --parallel" }, diff --git a/plugins/tsconfig.json b/plugins/tsconfig.json index 08219cc824..06a361f85d 100644 --- a/plugins/tsconfig.json +++ b/plugins/tsconfig.json @@ -18,5 +18,5 @@ "composite": true, "resolveJsonModule": true }, - "exclude": ["packages/*/lib/*.json", "packages/*/__tests__/*"] + "exclude": ["packages/*/lib/*.json", "packages/*/__tests__/*", "dist"] } diff --git a/server/package-lock.json b/server/package-lock.json index 8aa4737e07..96d0e3d48d 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -89,32 +89,31 @@ "name": "@tooljet/plugins", "version": "0.0.1", "dependencies": { - "airtable": "file:packages/airtable", - "common": "file:packages/common", - "dynamodb": "file:packages/dynamodb", - "elasticsearch": "file:packages/elasticsearch", - "firestore": "file:packages/firestore", - "gcs": "file:packages/gcs", - "googlesheets": "file:packages/googlesheets", - "graphql": "file:packages/graphql", - "lerna": "^4.0.0", - "minioapi": "file:packages/minioapi", - "mongo": "file:packages/mongo", - "mssql": "file:packages/mssql", - "mysql": "file:packages/mysql", - "postgresql": "file:packages/postgresql", - "redis": "file:packages/redis", - "restapi": "file:packages/restapi", - "s3": "file:packages/s3", - "sendgrid": "file:packages/sendgrid", - "slack": "file:packages/slack", - "stripe": "file:packages/stripe", - "twilioapi": "file:packages/twilioapi", - "typesenseapi": "file:packages/typesenseapi" + "@tooljet-plugins/airtable": "file:packages/airtable", + "@tooljet-plugins/common": "file:packages/common", + "@tooljet-plugins/dynamodb": "file:packages/dynamodb", + "@tooljet-plugins/elasticsearch": "file:packages/elasticsearch", + "@tooljet-plugins/firestore": "file:packages/firestore", + "@tooljet-plugins/gcs": "file:packages/gcs", + "@tooljet-plugins/googlesheets": "file:packages/googlesheets", + "@tooljet-plugins/graphql": "file:packages/graphql", + "@tooljet-plugins/minioapi": "file:packages/minioapi", + "@tooljet-plugins/mongo": "file:packages/mongo", + "@tooljet-plugins/mssql": "file:packages/mssql", + "@tooljet-plugins/mysql": "file:packages/mysql", + "@tooljet-plugins/postgresql": "file:packages/postgresql", + "@tooljet-plugins/redis": "file:packages/redis", + "@tooljet-plugins/restapi": "file:packages/restapi", + "@tooljet-plugins/s3": "file:packages/s3", + "@tooljet-plugins/sendgrid": "file:packages/sendgrid", + "@tooljet-plugins/slack": "file:packages/slack", + "@tooljet-plugins/stripe": "file:packages/stripe", + "@tooljet-plugins/twilioapi": "file:packages/twilioapi", + "@tooljet-plugins/typesenseapi": "file:packages/typesenseapi" }, "devDependencies": { "jest": "^27.4.5", - "react": "^17.0.2", + "lerna": "^4.0.0", "rimraf": "^3.0.2", "ts-jest": "^27.1.2", "typescript": "^4.5.4" @@ -3987,6 +3986,17 @@ "node-int64": "^0.4.0" } }, + "node_modules/bson": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.6.1.tgz", + "integrity": "sha512-I1LQ7Hz5zgwR4QquilLNZwbhPw0Apx7i7X9kGMBTsqPdml/03Q9NBtD9nt/19ahjlphktQImrnderxqpzeVDjw==", + "dependencies": { + "buffer": "^5.6.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/buffer": { "version": "5.7.1", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", @@ -14248,33 +14258,32 @@ "@tooljet/plugins": { "version": "file:../plugins", "requires": { - "airtable": "file:packages/airtable", - "common": "file:packages/common", - "dynamodb": "file:packages/dynamodb", - "elasticsearch": "file:packages/elasticsearch", - "firestore": "file:packages/firestore", - "gcs": "file:packages/gcs", - "googlesheets": "file:packages/googlesheets", - "graphql": "file:packages/graphql", + "@tooljet-plugins/airtable": "file:packages/airtable", + "@tooljet-plugins/common": "file:packages/common", + "@tooljet-plugins/dynamodb": "file:packages/dynamodb", + "@tooljet-plugins/elasticsearch": "file:packages/elasticsearch", + "@tooljet-plugins/firestore": "file:packages/firestore", + "@tooljet-plugins/gcs": "file:packages/gcs", + "@tooljet-plugins/googlesheets": "file:packages/googlesheets", + "@tooljet-plugins/graphql": "file:packages/graphql", + "@tooljet-plugins/minioapi": "file:packages/minioapi", + "@tooljet-plugins/mongo": "file:packages/mongo", + "@tooljet-plugins/mssql": "file:packages/mssql", + "@tooljet-plugins/mysql": "file:packages/mysql", + "@tooljet-plugins/postgresql": "file:packages/postgresql", + "@tooljet-plugins/redis": "file:packages/redis", + "@tooljet-plugins/restapi": "file:packages/restapi", + "@tooljet-plugins/s3": "file:packages/s3", + "@tooljet-plugins/sendgrid": "file:packages/sendgrid", + "@tooljet-plugins/slack": "file:packages/slack", + "@tooljet-plugins/stripe": "file:packages/stripe", + "@tooljet-plugins/twilioapi": "file:packages/twilioapi", + "@tooljet-plugins/typesenseapi": "file:packages/typesenseapi", "jest": "^27.4.5", "lerna": "^4.0.0", - "minioapi": "file:packages/minioapi", - "mongo": "file:packages/mongo", - "mssql": "file:packages/mssql", - "mysql": "file:packages/mysql", - "postgresql": "file:packages/postgresql", - "react": "^17.0.2", - "redis": "file:packages/redis", - "restapi": "file:packages/restapi", "rimraf": "^3.0.2", - "s3": "file:packages/s3", - "sendgrid": "file:packages/sendgrid", - "slack": "file:packages/slack", - "stripe": "file:packages/stripe", "ts-jest": "^27.1.2", - "twilioapi": "file:packages/twilioapi", - "typescript": "^4.5.4", - "typesenseapi": "file:packages/typesenseapi" + "typescript": "^4.5.4" } }, "@tootallnate/once": { @@ -15542,6 +15551,14 @@ "node-int64": "^0.4.0" } }, + "bson": { + "version": "4.6.1", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.6.1.tgz", + "integrity": "sha512-I1LQ7Hz5zgwR4QquilLNZwbhPw0Apx7i7X9kGMBTsqPdml/03Q9NBtD9nt/19ahjlphktQImrnderxqpzeVDjw==", + "requires": { + "buffer": "^5.6.0" + } + }, "buffer": { "version": "5.7.1", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", diff --git a/server/package.json b/server/package.json index a861d0c0a1..70caeea1c8 100644 --- a/server/package.json +++ b/server/package.json @@ -21,8 +21,8 @@ "test:e2e": "NODE_ENV=test jest --runInBand --config ./test/jest-e2e.json --detectOpenHandles", "db:create": "ts-node ./scripts/create-database.ts", "db:drop": "ts-node ./scripts/drop-database.ts", - "db:migrate": "npm run typeorm migration:run", - "db:seed": "ts-node -r tsconfig-paths/register ./scripts/seeds.ts", + "db:migrate": "ts-node -r tsconfig-paths/register --transpile-only ./node_modules/typeorm/cli.js migration:run", + "db:seed": "ts-node -r tsconfig-paths/register --transpile-only ./scripts/seeds.ts", "db:reset": "npm run db:drop && npm run db:create && npm run db:migrate", "typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js --config ormconfig.ts" }, diff --git a/server/tsconfig.json b/server/tsconfig.json index 703c62e05f..d7ffc05161 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -18,6 +18,7 @@ "@controllers/*": ["src/controllers/*"], "@repositories/*": ["src/repositories/*"], "@dto/*": ["src/dto/*"] - } - } + }, + }, + "exclude": ["node_modules", "dist"] }