Files
plane/docker-compose-cloud.yml
guru_sainath 1064e4fcdc Feat: Jira, Linear integration and GitHub importer (#1400)
* fix: silo service initial setup

* feat: moved controllers inside runner

* feat: created core definations for apps and context

* feat: added interfaces required for apps

* feat: moved shared definations

* feat: added logger as an injection for apps

* feat: changes to controllers

* feat: added vm based app execution inside manager

* feat: added dummy implementations for core constructs

* feat: moved controllers to routes

* feat: created worker moduler for processing message queue requests

* feat: added task manager defination

* feat: added context and manager separations

* feat: created defaults for the defination implementation

* feat: added async task management inside apps

* feat: removed unnecessary arguments in function calls

* feat: added context builder for app instances

* feat: added runtime type safety with zod

* feat: added packaged definition structure for silo apps

* feat: added package based configuration for engine and definitions

* feat: added custom eslint rule for throws decorator

* feat: added custom eslint rule for try catch

* feat: removed custom eslint config

* feat: added authentication controller inside engine

* feat: added auth in silo engine

* feat: migrated jira ui module to plane

* chore: removed jira app

* chore: moved engine components into src_engine directory

* feat: added base template inside new src directory

* feat: migrated worker interface with mq and redis store inside new src

* feat: created jira migrator and jira importer types

* chore: migrated worker's helper inside the base directory

* feat: added logic for booting up root worker

* feat: added jira and jira auth service inside jira app

* feat: added all transformers inside jira package

* feat: added authorization types for jira

* feat: added jira authentication oauth class

* feat: added jira transformer and pull mechanisms inside jira app

* feat: addded batching logic inside jira migrator

* feat: embedded silo sdk inside jira migrator

* feat: added plane migrator inside engine

* feat: added plane migrator for push inside migration controller

* feat: added controller methods for migration

* feat: added credentials and job routes in controller

* feat: added linear importer

* feat: added linear pull function as importer

* feat: added transformation for linear data

* feat: added pull mechanism for linear

* feat: attached linear data importer with migration controller

* fix: removed hardcoded jira from cycle and module migrator

* feat: fixed build errors

* chore: addeed example env

* feat: added authentication routes for jira

* feat: added linear route controller

* fix: restructuring

* fix: sdk configs setup

* fix: merge conflicts

* fix: sdk setup

* chore: added jira and linear importers and separate packages

* feat: moved transformation parts to linear package

* feat: decoupled jira logical parts with worker

* fix: linear silo app to use linear package

* fix: build errors and dependency resolution with packages in silo

* fix: module build errors in silo

* fix: linear authorization flow

* feat: added logic for segregated workers

* feat: attached task manager with the base starter

* feat: added migrations, query and schema into db directory

* feat: added linear importer and jira importer app structure

* feat: added silo core package

* chore: migrated worker and main engine controller inside apps

* fix: made linear integration working

* silo: added cors

* feat: added base64 state changes with jira

* chore: updated silo env

* fix: jira token cookie

* feat: added issue attachments in linear job

* feat: added credentials controller

* feat: added github package inside silo packages

* feat: added resource fetching in jira api service

* feat: added credentials locking

* feat: created resources endpoint for jira

* feat: added endpoints for getting jira data

* fix: credentials not working

* chore: exported jira types

* chore: added jira states

* fix: jira project pagination issue

* chore: initiated silo folder in web

* feat: added github routes and services in silo app

* fix: build updates

* fix: updated plane sdk and updated jira importer

* chore: updated the importer layout

* chore: removed as any from table component

* chore: integrated importer dashboard

* fix: tsup fixes

* fix: removed unnecessary files

* fix: removing tsup for building silo packages

* fix: build related issues

* fix: build issues

* fix: eslint fixes

* fix: silo build errors

* fix: silo app build errors types

* fix: reverting the cloud branch

* fix: updated package json in silo service

* fix: branch build cloud updated

* fix: build errors in apps while using sdk due to ts-alias paths

* fix: branch build cloud workflow fixes

* fix: docker compose setup updates

* fix: docker compose build fixes

* fix: docker build fixes envs updated in example file

* chore: updates folder structure and handled job services

* chore: resolved build errors in silo chore

* fix: docker compose cloud added

* fix: build process docker compose

* chore: updated jira workflow

* chore: handled the job start and jira dashboard

* chore: updated constants, file naming convention

* chore: resolved merge conflicts

* chore: integrated linear and updated jobs query

* feat: added hostname changes

* conflict: updated jira config and added issue transformation count in dashboard

* conflict: updated job

* chore: updated Jira status

* chore: updated Jira status

* fix: batch processing

* fix: batch key release

* chore: updated workflow for building images

* fix: detached silo build

* chore: updates linear and resolved build errors

* fix: batch key missing

* fix: linear workflow

* chore: updated linear queries

* fix: docker compose fixed for running silo services

---------

Co-authored-by: sriram veeraghanta <veeraghanta.sriram@gmail.com>
Co-authored-by: Henit Chobisa <chobisa.henit@gmail.com>
2024-10-08 23:16:28 +05:30

288 lines
9.2 KiB
YAML

x-silo-env: &silo-env
environment:
- PORT=3000
- BATCH_SIZE=${BATCH_SIZE:-60}
- MQ_PREFETCH_COUNT=${MQ_PREFETCH_COUNT:-5}
- APP_BASE_URL=${APP_BASE_URL:-http://web:3000}
- SILO_API_BASE_URL=${SILO_API_BASE_URL:-http://localhost:5050}
- DB_URL=${DB_URL:-postgresql://plane:plane@plane-db/silo}
- AMQP_URL=${AMQP_URL:-amqp://guest:guest@plane-mq:5672/}
- REDIS_URL=${REDIS_URL:-redis://plane-redis:6379/}
- SENTRY_DSN=${SENTRY_DSN:-" "}
- JIRA_CLIENT_ID=${JIRA_CLIENT_ID:-""}
- JIRA_CLIENT_SECRET=${JIRA_CLIENT_SECRET:-""}
- LINEAR_CLIENT_ID=${LINEAR_CLIENT_ID:-""}
- LINEAR_CLIENT_SECRET=${LINEAR_CLIENT_SECRET:-""}
x-monitor-env: &monitor-env
environment:
- SERVICE_HTTP_WEB=web:3000
- SERVICE_HTTP_API=api:8000
- SERVICE_HTTP_LIVE=live:3000
- SERVICE_HTTP_PROXY=proxy:80
- SERVICE_HTTP_MINIO=plane-minio:9090
- SERVICE_TCP_REDIS=plane-redis:6379
- SERVICE_TCP_POSTGRES=plane-db:5432
- TRUSTED_PROXIES=${TRUSTED_PROXIES:-0.0.0.0/0}
x-proxy-env: &proxy-env
environment:
- SITE_ADDRESS=${SITE_ADDRESS:-localhost:80}
- CERT_EMAIL=${CERT_EMAIL:-admin@example.com}
- CERT_ACME_CA=${CERT_ACME_CA:-}
- CERT_ACME_DNS=${CERT_ACME_DNS:-}
- BUCKET_NAME=${BUCKET_NAME:-uploads}
- FILE_SIZE_LIMIT=${FILE_SIZE_LIMIT:-5242880}
- LISTEN_HTTP_PORT=${LISTEN_HTTP_PORT:-80}
- LISTEN_HTTPS_PORT=${LISTEN_HTTPS_PORT:-443}
x-app-env: &app-env
environment:
- WEB_URL=${WEB_URL:-http://localhost}
- DEBUG=${DEBUG:-0}
- SENTRY_DSN=${SENTRY_DSN:-""}
- SENTRY_ENVIRONMENT=${SENTRY_ENVIRONMENT:-"production"}
- CORS_ALLOWED_ORIGINS=${CORS_ALLOWED_ORIGINS:-}
- API_BASE_URL=${API_BASE_URL:-http://api:8000}
# Gunicorn Workers
- GUNICORN_WORKERS=${GUNICORN_WORKERS:-2}
#DB SETTINGS
- PGHOST=${PGHOST:-plane-db}
- PGDATABASE=${PGDATABASE:-plane}
- POSTGRES_USER=${POSTGRES_USER:-plane}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-plane}
- POSTGRES_DB=${POSTGRES_DB:-plane}
- PGDATA=${PGDATA:-/var/lib/postgresql/data}
- SILO_DB=${SILO_DB:-silo}
- DATABASE_URL=${DATABASE_URL:-postgresql://plane:plane@plane-db/plane}
# REDIS SETTINGS
- REDIS_HOST=${REDIS_HOST:-plane-redis}
- REDIS_PORT=${REDIS_PORT:-6379}
- REDIS_URL=${REDIS_URL:-redis://plane-redis:6379/}
# RabbitMQ Settings
- RABBITMQ_HOST=${RABBITMQ_HOST:-plane-mq}
- RABBITMQ_PORT=${RABBITMQ_PORT:-5672}
- RABBITMQ_DEFAULT_USER=${RABBITMQ_USER:-plane}
- RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD:-plane}
- RABBITMQ_DEFAULT_VHOST=${RABBITMQ_VHOST:-plane}
- RABBITMQ_VHOST=${RABBITMQ_VHOST:-plane}
- AMQP_URL=${AMQP_URL:-amqp://plane:plane@plane-mq:5672/plane}
# Application secret
- SECRET_KEY=${SECRET_KEY:-60gp0byfz2dvffa45cxl20p1scy9xbpf6d8c5y0geejgkyp1b5}
# DATA STORE SETTINGS
- USE_MINIO=${USE_MINIO:-1}
- AWS_REGION=${AWS_REGION:-""}
- AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-"access-key"}
- AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-"secret-key"}
- AWS_S3_ENDPOINT_URL=${AWS_S3_ENDPOINT_URL:-http://plane-minio:9000}
- AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME:-uploads}
- MINIO_ROOT_USER=${MINIO_ROOT_USER:-"access-key"}
- MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-"secret-key"}
- BUCKET_NAME=${BUCKET_NAME:-uploads}
- FILE_SIZE_LIMIT=${FILE_SIZE_LIMIT:-5242880}
- SSL_VERIFY=${SSL_VERIFY:-1}
- FEATURE_FLAG_SERVER_BASE_URL=${FEATURE_FLAG_SERVER_BASE_URL:-http://monitor:8080}
- PAYMENT_SERVER_BASE_URL=${PAYMENT_SERVER_BASE_URL:-http://monitor:8080}
x-live-env: &live-env
environment:
- API_BASE_URL=${API_BASE_URL:-http://api:8000}
- REDIS_HOST=${REDIS_HOST:-plane-redis}
- REDIS_PORT=${REDIS_PORT:-6379}
- REDIS_URL=${REDIS_URL:-redis://plane-redis:6379/}
services:
admin:
<<: *app-env
image: registry.plane.tools/plane/admin-enterprise:${APP_RELEASE_VERSION}
restart: unless-stopped
command: node admin/server.js admin
deploy:
replicas: ${ADMIN_REPLICAS:-1}
depends_on:
- api
- web
silo:
<<: *silo-env
image: registry.plane.tools/plane/silo-enterprise:${APP_RELEASE_VERSION}
restart: always
command: node silo/start.js
depends_on:
plane-mq:
condition: service_healthy
silo_migrator:
condition: service_completed_successfully
silo_migrator:
<<: *silo-env
image: registry.plane.tools/plane/silo-enterprise:${APP_RELEASE_VERSION}
restart: "no"
command: npm --prefix ./silo run db:migrate
depends_on:
plane-db:
condition: service_healthy
plane-mq:
condition: service_healthy
web:
<<: *app-env
image: registry.plane.tools/plane/web-enterprise:${APP_RELEASE_VERSION}
restart: unless-stopped
command: node web/server.js web
deploy:
replicas: ${WEB_REPLICAS:-1}
depends_on:
- api
- worker
space:
<<: *app-env
image: registry.plane.tools/plane/space-enterprise:${APP_RELEASE_VERSION}
restart: unless-stopped
command: node space/server.js space
deploy:
replicas: ${SPACE_REPLICAS:-1}
depends_on:
- api
- web
live:
<<: *live-env
image: registry.plane.tools/plane/live-enterprise:${APP_RELEASE_VERSION}
restart: unless-stopped
command: node live/dist/server.js live
deploy:
replicas: ${LIVE_REPLICAS:-1}
depends_on:
- api
- web
monitor:
<<: *monitor-env
image: registry.plane.tools/plane/monitor-enterprise:${APP_RELEASE_VERSION}
restart: on-failure:5
volumes:
- ${INSTALL_DIR}/data/monitor:/app
api:
<<: *app-env
image: registry.plane.tools/plane/backend-enterprise:${APP_RELEASE_VERSION}
restart: unless-stopped
command: ./bin/docker-entrypoint-api-ee.sh
deploy:
replicas: ${API_REPLICAS:-1}
volumes:
- ${INSTALL_DIR}/logs/api:/code/plane/logs
depends_on:
- plane-db
- plane-redis
worker:
<<: *app-env
image: registry.plane.tools/plane/backend-enterprise:${APP_RELEASE_VERSION}
restart: unless-stopped
command: ./bin/docker-entrypoint-worker.sh
volumes:
- ${INSTALL_DIR}/logs/worker:/code/plane/logs
depends_on:
- api
- plane-db
- plane-redis
beat-worker:
<<: *app-env
image: registry.plane.tools/plane/backend-enterprise:${APP_RELEASE_VERSION}
restart: unless-stopped
command: ./bin/docker-entrypoint-beat.sh
volumes:
- ${INSTALL_DIR}/logs/beat-worker:/code/plane/logs
depends_on:
- api
- plane-db
- plane-redis
migrator:
<<: *app-env
image: registry.plane.tools/plane/backend-enterprise:${APP_RELEASE_VERSION}
restart: "no"
command: ./bin/docker-entrypoint-migrator.sh
volumes:
- ${INSTALL_DIR}/logs/migrator:/code/plane/logs
depends_on:
- plane-db
- plane-redis
plane-db:
<<: *app-env
image: registry.plane.tools/plane/postgres:15.5-alpine
restart: unless-stopped
command: >
bash -c '
docker-entrypoint.sh postgres &
sleep 5 &&
result=$(PGPASSWORD=${POSTGRES_PASSWORD} psql -U $POSTGRES_USER -tAc "SELECT 1 FROM pg_database WHERE datname='${SILO_DB}'") &&
echo "$result" | grep -q 1 && result=0 || result=1 &&
if [ $result -eq 0 ]; then
echo "Database '${SILO_DB}' already exists"
else
PGPASSWORD=${POSTGRES_PASSWORD} psql -U $POSTGRES_USER -c "CREATE DATABASE ${SILO_DB};"
echo "Created database '${SILO_DB}'"
fi &&
wait
'
healthcheck:
test: ["CMD-SHELL", "PGPASSWORD=${POSTGRES_PASSWORD} psql -U $POSTGRES_USER -tAc \"SELECT CASE WHEN EXISTS(SELECT 1 FROM pg_database WHERE datname='${SILO_DB}') THEN 1 ELSE 0 END AS ${SILO_DB}, CASE WHEN EXISTS(SELECT 1 FROM pg_database WHERE datname='${POSTGRES_DB}') THEN 1 ELSE 0 END AS ${POSTGRES_DB}\" | grep -q '1|1' && echo 'Both databases exist' || (echo 'One or both databases are missing' && exit 1)"]
interval: 30s
timeout: 10s
retries: 3
# volumes:
# - ${INSTALL_DIR}/data/db:/var/lib/postgresql/data
plane-redis:
<<: *app-env
image: registry.plane.tools/plane/valkey:7.2.5-alpine
restart: unless-stopped
volumes:
- ${INSTALL_DIR}/data/redis:/data
plane-mq:
<<: *app-env
image: registry.plane.tools/plane/rabbitmq:3.13.6-management-alpine
restart: unless-stopped
healthcheck:
test: rabbitmq-diagnostics -q ping
interval: 10s
timeout: 10s
retries: 5
volumes:
- ${INSTALL_DIR}/data/mq:/var/lib/rabbitmq
plane-minio:
<<: *app-env
image: registry.plane.tools/plane/minio:latest
restart: unless-stopped
command: server /export --console-address ":9090"
volumes:
- ${INSTALL_DIR}/data/minio/uploads:/export
- ${INSTALL_DIR}/data/minio/data:/data
# Comment this if you already have a reverse proxy running
proxy:
<<: *proxy-env
image: registry.plane.tools/plane/caddy:latest
restart: unless-stopped
ports:
- ${LISTEN_HTTP_PORT:-80}:${LISTEN_HTTP_PORT:-80}
- ${LISTEN_HTTPS_PORT:-443}:${LISTEN_HTTPS_PORT:-443}
volumes:
- ${INSTALL_DIR}/Caddyfile:/etc/caddy/Caddyfile
- ${INSTALL_DIR}/caddy/config:/config
- ${INSTALL_DIR}/caddy/data:/data
depends_on:
- web
- api
- space
- admin
- live