From 94327b83112d9e5c22366aa2538839d6ac6e04bd Mon Sep 17 00:00:00 2001 From: Manish Gupta <59428681+mguptahub@users.noreply.github.com> Date: Fri, 8 Mar 2024 15:16:32 +0530 Subject: [PATCH 1/6] chore: feature build process optimization (#3907) * process changed to build tar and use for deployment * fixes --- .github/workflows/feature-deployment.yml | 186 +++++++++++++++++++---- 1 file changed, 155 insertions(+), 31 deletions(-) diff --git a/.github/workflows/feature-deployment.yml b/.github/workflows/feature-deployment.yml index 7b9f5ffcc..12549cff5 100644 --- a/.github/workflows/feature-deployment.yml +++ b/.github/workflows/feature-deployment.yml @@ -4,70 +4,194 @@ on: workflow_dispatch: inputs: web-build: - required: true + required: false + description: 'Build Web' type: boolean default: true space-build: - required: true + required: false + description: 'Build Space' type: boolean default: false +env: + BUILD_WEB: ${{ github.event.inputs.web-build }} + BUILD_SPACE: ${{ github.event.inputs.space-build }} + jobs: + setup-feature-build: + name: Feature Build Setup + runs-on: ubuntu-latest + steps: + - name: Checkout + run: | + echo "BUILD_WEB=$BUILD_WEB" + echo "BUILD_SPACE=$BUILD_SPACE" + outputs: + web-build: ${{ env.BUILD_WEB}} + space-build: ${{env.BUILD_SPACE}} + + feature-build-web: + if: ${{ needs.setup-feature-build.outputs.web-build == 'true' }} + needs: setup-feature-build + name: Feature Build Web + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ vars.FEATURE_PREVIEW_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.FEATURE_PREVIEW_AWS_SECRET_ACCESS_KEY }} + AWS_BUCKET: ${{ vars.FEATURE_PREVIEW_AWS_BUCKET }} + NEXT_PUBLIC_API_URL: ${{ vars.FEATURE_PREVIEW_NEXT_PUBLIC_API_BASE_URL }} + steps: + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + - name: Install AWS cli + run: | + sudo apt-get update + sudo apt-get install -y python3-pip + pip3 install awscli + - name: Checkout + uses: actions/checkout@v4 + with: + path: feature-preview + - name: Install Dependencies + run: | + cd $GITHUB_WORKSPACE/feature-preview + yarn install + - name: Build Web + id: build-web + run: | + cd $GITHUB_WORKSPACE/feature-preview + yarn build --filter=web + cd $GITHUB_WORKSPACE + + TAR_NAME="web.tar.gz" + tar -czf $TAR_NAME ./feature-preview + + FILE_EXPIRY=$(date -u -d "+2 days" +"%Y-%m-%dT%H:%M:%SZ") + aws s3 cp $TAR_NAME s3://${{ env.AWS_BUCKET }}/${{github.sha}}/$TAR_NAME --expires $FILE_EXPIRY + + feature-build-space: + if: ${{ needs.setup-feature-build.outputs.space-build == 'true' }} + needs: setup-feature-build + name: Feature Build Space + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ vars.FEATURE_PREVIEW_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.FEATURE_PREVIEW_AWS_SECRET_ACCESS_KEY }} + AWS_BUCKET: ${{ vars.FEATURE_PREVIEW_AWS_BUCKET }} + NEXT_PUBLIC_DEPLOY_WITH_NGINX: 1 + outputs: + do-build: ${{ needs.setup-feature-build.outputs.space-build }} + s3-url: ${{ steps.build-space.outputs.S3_PRESIGNED_URL }} + steps: + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + - name: Install AWS cli + run: | + sudo apt-get update + sudo apt-get install -y python3-pip + pip3 install awscli + - name: Checkout + uses: actions/checkout@v4 + with: + path: plane + - name: Install Dependencies + run: | + cd $GITHUB_WORKSPACE/plane + yarn install + - name: Build Space + id: build-space + run: | + cd $GITHUB_WORKSPACE/plane + yarn build --filter=space + cd $GITHUB_WORKSPACE + + TAR_NAME="space.tar.gz" + tar -czf $TAR_NAME ./plane + + FILE_EXPIRY=$(date -u -d "+2 days" +"%Y-%m-%dT%H:%M:%SZ") + aws s3 cp $TAR_NAME s3://${{ env.AWS_BUCKET }}/${{github.sha}}/$TAR_NAME --expires $FILE_EXPIRY + feature-deploy: + if: ${{ always() && (needs.setup-feature-build.outputs.web-build == 'true' || needs.setup-feature-build.outputs.space-build == 'true') }} + needs: [feature-build-web, feature-build-space] name: Feature Deploy runs-on: ubuntu-latest env: - KUBE_CONFIG_FILE: ${{ secrets.KUBE_CONFIG }} - BUILD_WEB: ${{ (github.event.inputs.web-build == '' && true) || github.event.inputs.web-build }} - BUILD_SPACE: ${{ (github.event.inputs.space-build == '' && false) || github.event.inputs.space-build }} - + AWS_ACCESS_KEY_ID: ${{ vars.FEATURE_PREVIEW_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.FEATURE_PREVIEW_AWS_SECRET_ACCESS_KEY }} + AWS_BUCKET: ${{ vars.FEATURE_PREVIEW_AWS_BUCKET }} + KUBE_CONFIG_FILE: ${{ secrets.FEATURE_PREVIEW_KUBE_CONFIG }} steps: + - name: Install AWS cli + run: | + sudo apt-get update + sudo apt-get install -y python3-pip + pip3 install awscli - name: Tailscale uses: tailscale/github-action@v2 with: oauth-client-id: ${{ secrets.TAILSCALE_OAUTH_CLIENT_ID }} oauth-secret: ${{ secrets.TAILSCALE_OAUTH_SECRET }} tags: tag:ci - - name: Kubectl Setup run: | - curl -LO "https://dl.k8s.io/release/${{secrets.KUBE_VERSION}}/bin/linux/amd64/kubectl" + curl -LO "https://dl.k8s.io/release/${{ vars.FEATURE_PREVIEW_KUBE_VERSION }}/bin/linux/amd64/kubectl" chmod +x kubectl mkdir -p ~/.kube echo "$KUBE_CONFIG_FILE" > ~/.kube/config chmod 600 ~/.kube/config - - name: HELM Setup run: | curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 chmod 700 get_helm.sh ./get_helm.sh - - name: App Deploy run: | - helm --kube-insecure-skip-tls-verify repo add feature-preview ${{ secrets.FEATURE_PREVIEW_HELM_CHART_URL }} - GIT_BRANCH=${{ github.ref_name }} - APP_NAMESPACE=${{ secrets.FEATURE_PREVIEW_NAMESPACE }} + WEB_S3_URL="" + if [ ${{ env.BUILD_WEB }} == true ]; then + WEB_S3_URL=$(aws s3 presign s3://${{ vars.FEATURE_PREVIEW_AWS_BUCKET }}/${{github.sha}}/web.tar.gz --expires-in 3600) + fi - METADATA=$(helm install feature-preview/${{ secrets.FEATURE_PREVIEW_HELM_CHART_NAME }} \ - --kube-insecure-skip-tls-verify \ - --generate-name \ - --namespace $APP_NAMESPACE \ - --set shared_config.git_repo=${{github.server_url}}/${{ github.repository }}.git \ - --set shared_config.git_branch="$GIT_BRANCH" \ - --set web.enabled=${{ env.BUILD_WEB }} \ - --set space.enabled=${{ env.BUILD_SPACE }} \ - --output json \ - --timeout 1000s) + SPACE_S3_URL="" + if [ ${{ env.BUILD_SPACE }} == true ]; then + SPACE_S3_URL=$(aws s3 presign s3://${{ vars.FEATURE_PREVIEW_AWS_BUCKET }}/${{github.sha}}/space.tar.gz --expires-in 3600) + fi - APP_NAME=$(echo $METADATA | jq -r '.name') + if [ ${{ env.BUILD_WEB }} == true ] || [ ${{ env.BUILD_SPACE }} == true ]; then - INGRESS_HOSTNAME=$(kubectl get ingress -n feature-builds --insecure-skip-tls-verify \ - -o jsonpath='{.items[?(@.metadata.annotations.meta\.helm\.sh\/release-name=="'$APP_NAME'")]}' | \ - jq -r '.spec.rules[0].host') + helm --kube-insecure-skip-tls-verify repo add feature-preview ${{ vars.FEATURE_PREVIEW_HELM_CHART_URL }} - echo "****************************************" - echo "APP NAME ::: $APP_NAME" - echo "INGRESS HOSTNAME ::: $INGRESS_HOSTNAME" - echo "****************************************" + APP_NAMESPACE="${{ vars.FEATURE_PREVIEW_NAMESPACE }}" + DEPLOY_SCRIPT_URL="${{ vars.FEATURE_PREVIEW_DEPLOY_SCRIPT_URL }}" + + METADATA=$(helm --kube-insecure-skip-tls-verify install feature-preview/${{ vars.FEATURE_PREVIEW_HELM_CHART_NAME }} \ + --generate-name \ + --namespace $APP_NAMESPACE \ + --set ingress.primaryDomain=${{vars.FEATURE_PREVIEW_PRIMARY_DOMAIN || 'feature.plane.tools' }} \ + --set web.image=${{vars.FEATURE_PREVIEW_DOCKER_BASE}} \ + --set web.enabled=${{ env.BUILD_WEB || false }} \ + --set web.artifact_url=$WEB_S3_URL \ + --set space.image=${{vars.FEATURE_PREVIEW_DOCKER_BASE}} \ + --set space.enabled=${{ env.BUILD_SPACE || false }} \ + --set space.artifact_url=$SPACE_S3_URL \ + --set shared_config.deploy_script_url=$DEPLOY_SCRIPT_URL \ + --output json \ + --timeout 1000s) + + APP_NAME=$(echo $METADATA | jq -r '.name') + + INGRESS_HOSTNAME=$(kubectl get ingress -n feature-builds --insecure-skip-tls-verify \ + -o jsonpath='{.items[?(@.metadata.annotations.meta\.helm\.sh\/release-name=="'$APP_NAME'")]}' | \ + jq -r '.spec.rules[0].host') + + echo "****************************************" + echo "APP NAME ::: $APP_NAME" + echo "INGRESS HOSTNAME ::: $INGRESS_HOSTNAME" + echo "****************************************" + fi From 8cc372679c526c096b144af1b9a2b990b1e0db2b Mon Sep 17 00:00:00 2001 From: Manish Gupta Date: Fri, 8 Mar 2024 15:56:05 +0530 Subject: [PATCH 2/6] Web Build fixes --- .github/workflows/feature-deployment.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/feature-deployment.yml b/.github/workflows/feature-deployment.yml index 12549cff5..766f30514 100644 --- a/.github/workflows/feature-deployment.yml +++ b/.github/workflows/feature-deployment.yml @@ -40,7 +40,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ vars.FEATURE_PREVIEW_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.FEATURE_PREVIEW_AWS_SECRET_ACCESS_KEY }} AWS_BUCKET: ${{ vars.FEATURE_PREVIEW_AWS_BUCKET }} - NEXT_PUBLIC_API_URL: ${{ vars.FEATURE_PREVIEW_NEXT_PUBLIC_API_BASE_URL }} + NEXT_PUBLIC_API_BASE_URL: ${{ vars.FEATURE_PREVIEW_NEXT_PUBLIC_API_BASE_URL }} steps: - name: Set up Node.js uses: actions/setup-node@v4 @@ -54,21 +54,21 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: - path: feature-preview + path: plane - name: Install Dependencies run: | - cd $GITHUB_WORKSPACE/feature-preview + cd $GITHUB_WORKSPACE/plane yarn install - name: Build Web id: build-web run: | - cd $GITHUB_WORKSPACE/feature-preview + cd $GITHUB_WORKSPACE/plane yarn build --filter=web cd $GITHUB_WORKSPACE TAR_NAME="web.tar.gz" - tar -czf $TAR_NAME ./feature-preview - + tar -czf $TAR_NAME ./plane + FILE_EXPIRY=$(date -u -d "+2 days" +"%Y-%m-%dT%H:%M:%SZ") aws s3 cp $TAR_NAME s3://${{ env.AWS_BUCKET }}/${{github.sha}}/$TAR_NAME --expires $FILE_EXPIRY @@ -82,6 +82,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.FEATURE_PREVIEW_AWS_SECRET_ACCESS_KEY }} AWS_BUCKET: ${{ vars.FEATURE_PREVIEW_AWS_BUCKET }} NEXT_PUBLIC_DEPLOY_WITH_NGINX: 1 + NEXT_PUBLIC_API_BASE_URL: ${{ vars.FEATURE_PREVIEW_NEXT_PUBLIC_API_BASE_URL }} outputs: do-build: ${{ needs.setup-feature-build.outputs.space-build }} s3-url: ${{ steps.build-space.outputs.S3_PRESIGNED_URL }} From 6c6b7156bbb46fa5d2ebbd4e0d851b264d5a2cd3 Mon Sep 17 00:00:00 2001 From: Manish Gupta Date: Fri, 8 Mar 2024 16:00:18 +0530 Subject: [PATCH 3/6] helm variable update --- .github/workflows/feature-deployment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/feature-deployment.yml b/.github/workflows/feature-deployment.yml index 766f30514..c5eec3cd3 100644 --- a/.github/workflows/feature-deployment.yml +++ b/.github/workflows/feature-deployment.yml @@ -182,6 +182,7 @@ jobs: --set space.enabled=${{ env.BUILD_SPACE || false }} \ --set space.artifact_url=$SPACE_S3_URL \ --set shared_config.deploy_script_url=$DEPLOY_SCRIPT_URL \ + --set shared_config.api_base_url=${{vars.FEATURE_PREVIEW_NEXT_PUBLIC_API_BASE_URL}} \ --output json \ --timeout 1000s) From cb78ccad1f9f0d9266dda5c7fc365a59d562a5d1 Mon Sep 17 00:00:00 2001 From: sriram veeraghanta Date: Fri, 8 Mar 2024 16:58:37 +0530 Subject: [PATCH 4/6] fix: 1click deployment fixes --- deploy/1-click/plane-app | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/deploy/1-click/plane-app b/deploy/1-click/plane-app index e6bd24b9e..ace0a0b79 100644 --- a/deploy/1-click/plane-app +++ b/deploy/1-click/plane-app @@ -494,13 +494,6 @@ function install() { update_env "CORS_ALLOWED_ORIGINS" "http://$MY_IP" update_config "INSTALLATION_DATE" "$(date '+%Y-%m-%d')" - - if command -v crontab &> /dev/null; then - sudo touch /etc/cron.daily/makeplane - sudo chmod +x /etc/cron.daily/makeplane - sudo echo "0 2 * * * root /usr/local/bin/plane-app --upgrade" > /etc/cron.daily/makeplane - sudo crontab /etc/cron.daily/makeplane - fi show_message "Plane Installed Successfully ✅" show_message "" @@ -606,11 +599,6 @@ function uninstall() { sudo rm $PLANE_INSTALL_DIR/variables-upgrade.env &> /dev/null sudo rm $PLANE_INSTALL_DIR/config.env &> /dev/null sudo rm $PLANE_INSTALL_DIR/docker-compose.yaml &> /dev/null - - if command -v crontab &> /dev/null; then - sudo crontab -r &> /dev/null - sudo rm /etc/cron.daily/makeplane &> /dev/null - fi # rm -rf $PLANE_INSTALL_DIR &> /dev/null show_message "- Configuration Cleaned ✅" From f493a03f56edc0d445e95c21111ef9fd33de095d Mon Sep 17 00:00:00 2001 From: sriram veeraghanta Date: Tue, 12 Mar 2024 01:01:03 +0530 Subject: [PATCH 5/6] fix: adding github workflow events --- .github/workflows/build-branch.yml | 32 ++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-branch.yml b/.github/workflows/build-branch.yml index 44bae0efa..42cb18035 100644 --- a/.github/workflows/build-branch.yml +++ b/.github/workflows/build-branch.yml @@ -2,6 +2,27 @@ name: Branch Build on: workflow_dispatch: + inputs: + build-web: + required: false + description: "Build Web" + type: boolean + default: false + build-space: + required: false + description: "Build Space" + type: boolean + default: false + build-api: + required: false + description: "Build API" + type: boolean + default: false + build-proxy: + required: false + description: "Build Proxy" + type: boolean + default: false push: branches: - master @@ -18,15 +39,15 @@ jobs: name: Build-Push Web/Space/API/Proxy Docker Image runs-on: ubuntu-latest outputs: - gh_branch_name: ${{ steps.set_env_variables.outputs.TARGET_BRANCH }} + gh_branch_name: ${{ steps.set_env_variables.outputs.TARGET_BRANCH }} gh_buildx_driver: ${{ steps.set_env_variables.outputs.BUILDX_DRIVER }} gh_buildx_version: ${{ steps.set_env_variables.outputs.BUILDX_VERSION }} gh_buildx_platforms: ${{ steps.set_env_variables.outputs.BUILDX_PLATFORMS }} gh_buildx_endpoint: ${{ steps.set_env_variables.outputs.BUILDX_ENDPOINT }} - build_frontend: ${{ steps.changed_files.outputs.frontend_any_changed }} - build_space: ${{ steps.changed_files.outputs.space_any_changed }} - build_backend: ${{ steps.changed_files.outputs.backend_any_changed }} - build_proxy: ${{ steps.changed_files.outputs.proxy_any_changed }} + build_frontend: ${{ steps.changed_files.outputs.frontend_any_changed || github.event.inputs.build-web }} + build_space: ${{ steps.changed_files.outputs.space_any_changed || github.event.inputs.build-space }} + build_backend: ${{ steps.changed_files.outputs.backend_any_changed || github.event.inputs.build-api }} + build_proxy: ${{ steps.changed_files.outputs.proxy_any_changed || github.event.inputs.build-proxy }} steps: - id: set_env_variables @@ -280,4 +301,3 @@ jobs: DOCKER_BUILDKIT: 1 DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} - From 6ec9c64f7ca855df05f83f5329d631006a04b1a6 Mon Sep 17 00:00:00 2001 From: sriram veeraghanta Date: Tue, 12 Mar 2024 01:06:37 +0530 Subject: [PATCH 6/6] fix: build branch optional builds workflow changes --- .github/workflows/build-branch.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-branch.yml b/.github/workflows/build-branch.yml index 42cb18035..0d8d2af09 100644 --- a/.github/workflows/build-branch.yml +++ b/.github/workflows/build-branch.yml @@ -44,10 +44,10 @@ jobs: gh_buildx_version: ${{ steps.set_env_variables.outputs.BUILDX_VERSION }} gh_buildx_platforms: ${{ steps.set_env_variables.outputs.BUILDX_PLATFORMS }} gh_buildx_endpoint: ${{ steps.set_env_variables.outputs.BUILDX_ENDPOINT }} - build_frontend: ${{ steps.changed_files.outputs.frontend_any_changed || github.event.inputs.build-web }} - build_space: ${{ steps.changed_files.outputs.space_any_changed || github.event.inputs.build-space }} - build_backend: ${{ steps.changed_files.outputs.backend_any_changed || github.event.inputs.build-api }} - build_proxy: ${{ steps.changed_files.outputs.proxy_any_changed || github.event.inputs.build-proxy }} + build_frontend: ${{ steps.changed_files.outputs.frontend_any_changed }} + build_space: ${{ steps.changed_files.outputs.space_any_changed }} + build_backend: ${{ steps.changed_files.outputs.backend_any_changed }} + build_proxy: ${{ steps.changed_files.outputs.proxy_any_changed }} steps: - id: set_env_variables @@ -95,7 +95,7 @@ jobs: - nginx/** branch_build_push_frontend: - if: ${{ needs.branch_build_setup.outputs.build_frontend == 'true' || github.event_name == 'release' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }} + if: ${{ needs.branch_build_setup.outputs.build_frontend == 'true' || github.event.inputs.build-web=='true' || github.event_name == 'release' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }} runs-on: ubuntu-20.04 needs: [branch_build_setup] env: @@ -147,7 +147,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} branch_build_push_space: - if: ${{ needs.branch_build_setup.outputs.build_space == 'true' || github.event_name == 'release' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }} + if: ${{ needs.branch_build_setup.outputs.build_space == 'true' || github.event.inputs.build-space=='true' || github.event_name == 'release' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }} runs-on: ubuntu-20.04 needs: [branch_build_setup] env: @@ -199,7 +199,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} branch_build_push_backend: - if: ${{ needs.branch_build_setup.outputs.build_backend == 'true' || github.event_name == 'release' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }} + if: ${{ needs.branch_build_setup.outputs.build_backend == 'true' || github.event.inputs.build-api=='true' || github.event_name == 'release' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }} runs-on: ubuntu-20.04 needs: [branch_build_setup] env: @@ -251,7 +251,7 @@ jobs: DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} branch_build_push_proxy: - if: ${{ needs.branch_build_setup.outputs.build_proxy == 'true' || github.event_name == 'release' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }} + if: ${{ needs.branch_build_setup.outputs.build_proxy == 'true' || github.event.inputs.build-web=='true' || github.event_name == 'release' || needs.branch_build_setup.outputs.gh_branch_name == 'master' }} runs-on: ubuntu-20.04 needs: [branch_build_setup] env: