From 0487579f4ef010f64161e413707f35c8dac68fcb Mon Sep 17 00:00:00 2001 From: pablohashescobar Date: Tue, 19 Mar 2024 11:43:50 +0530 Subject: [PATCH] dev: workspace export and import functionality for instance admins --- apiserver/plane/app/urls/workspace.py | 6 - apiserver/plane/app/views/__init__.py | 1 - apiserver/plane/app/views/workspace/base.py | 30 +--- .../plane/bgtasks/workspace_export_task.py | 14 +- .../plane/bgtasks/workspace_import_task.py | 169 ++++++++++++++++++ apiserver/plane/license/api/views/__init__.py | 2 + apiserver/plane/license/api/views/instance.py | 107 +++++++++-- apiserver/plane/license/urls.py | 16 +- 8 files changed, 291 insertions(+), 54 deletions(-) create mode 100644 apiserver/plane/bgtasks/workspace_import_task.py diff --git a/apiserver/plane/app/urls/workspace.py b/apiserver/plane/app/urls/workspace.py index d84f9a215b..aef34103e2 100644 --- a/apiserver/plane/app/urls/workspace.py +++ b/apiserver/plane/app/urls/workspace.py @@ -1,7 +1,6 @@ from django.urls import path from plane.app.views import ( - ExportWorkspaceEndpoint, ExportWorkspaceUserActivityEndpoint, TeamMemberViewSet, UserLastProjectWithWorkspaceEndpoint, @@ -236,9 +235,4 @@ urlpatterns = [ WorkspaceCyclesEndpoint.as_view(), name="workspace-cycles", ), - path( - "workspaces//export/", - ExportWorkspaceEndpoint.as_view(), - name="workspace-exports", - ), ] diff --git a/apiserver/plane/app/views/__init__.py b/apiserver/plane/app/views/__init__.py index 7801d81c87..b84832db93 100644 --- a/apiserver/plane/app/views/__init__.py +++ b/apiserver/plane/app/views/__init__.py @@ -38,7 +38,6 @@ from .workspace.base import ( UserWorkspaceDashboardEndpoint, WorkspaceThemeViewSet, ExportWorkspaceUserActivityEndpoint, - ExportWorkspaceEndpoint, ) from .workspace.member import ( diff --git a/apiserver/plane/app/views/workspace/base.py b/apiserver/plane/app/views/workspace/base.py index 1b10e9904d..daf95a1876 100644 --- a/apiserver/plane/app/views/workspace/base.py +++ b/apiserver/plane/app/views/workspace/base.py @@ -4,6 +4,8 @@ import io from datetime import date from dateutil.relativedelta import relativedelta + +# Django imports from django.db import IntegrityError from django.db.models import ( Count, @@ -15,8 +17,6 @@ from django.db.models import ( ) from django.db.models.fields import DateField from django.db.models.functions import Cast, ExtractDay, ExtractWeek - -# Django imports from django.http import HttpResponse from django.utils import timezone @@ -36,7 +36,6 @@ from plane.app.serializers import ( WorkspaceThemeSerializer, ) from plane.app.views.base import BaseAPIView, BaseViewSet -from plane.bgtasks.workspace_export_task import workspace_export from plane.db.models import ( Issue, IssueActivity, @@ -416,28 +415,3 @@ class ExportWorkspaceUserActivityEndpoint(BaseAPIView): 'attachment; filename="workspace-user-activity.csv"' ) return response - - -class ExportWorkspaceEndpoint(BaseAPIView): - - permission_classes = [ - WorkSpaceAdminPermission, - ] - - def post(self, request, slug): - current_origin = ( - request.META.get("HTTP_ORIGIN") - or f"{request.scheme}://{request.get_host()}" - ) - - workspace_export.delay( - slug=slug, - origin=current_origin, - email=request.user.email, - ) - return Response( - { - "message": "An email will be sent to download the exports when they are ready" - }, - status=status.HTTP_200_OK, - ) diff --git a/apiserver/plane/bgtasks/workspace_export_task.py b/apiserver/plane/bgtasks/workspace_export_task.py index 6851b11ad1..227f1d5adf 100644 --- a/apiserver/plane/bgtasks/workspace_export_task.py +++ b/apiserver/plane/bgtasks/workspace_export_task.py @@ -152,23 +152,23 @@ def upload_to_s3(zip_file, workspace_id, slug): @shared_task -def workspace_export(slug, origin, email): +def workspace_export(workspace_id, email): # Get the workspace - workspace = Workspace.objects.get(slug=slug) - workspace_id = workspace.id + workspace = Workspace.objects.get(pk=workspace_id) + slug = workspace.slug # Store all files files = [] # Users that need to be exported - emails = WorkspaceMember.objects.filter(workspace__slug=slug).values_list( - "member__email", flat=True - ) + emails = WorkspaceMember.objects.filter( + workspace_id=workspace_id + ).values_list("member__email", flat=True) users = User.objects.filter(email__in=emails).values() users_json = json.dumps(list(users), cls=DjangoJSONEncoder) files.append({"filename": "users.json", "data": users_json}) - workspace = list(Workspace.objects.filter(slug=slug).values()) + workspace = list(Workspace.objects.filter(pk=workspace_id).values()) workspace_json = json.dumps(workspace, cls=DjangoJSONEncoder) files.append({"filename": "workspaces.json", "data": workspace_json}) diff --git a/apiserver/plane/bgtasks/workspace_import_task.py b/apiserver/plane/bgtasks/workspace_import_task.py new file mode 100644 index 0000000000..ba228d2cff --- /dev/null +++ b/apiserver/plane/bgtasks/workspace_import_task.py @@ -0,0 +1,169 @@ +# Python imports +import json + +# Third party imports +from celery import shared_task + +from plane.db.models import ( + APIToken, + CommentReaction, + Cycle, + CycleFavorite, + CycleIssue, + CycleUserProperties, + Estimate, + EstimatePoint, + FileAsset, + Inbox, + InboxIssue, + Issue, + IssueActivity, + IssueAssignee, + IssueAttachment, + IssueComment, + IssueLabel, + IssueLink, + IssueMention, + IssueProperty, + IssueReaction, + IssueRelation, + IssueSequence, + IssueSubscriber, + IssueView, + IssueViewFavorite, + IssueVote, + Label, + Module, + ModuleFavorite, + ModuleIssue, + ModuleLink, + ModuleMember, + ModuleUserProperties, + Notification, + Page, + PageFavorite, + PageLabel, + PageLog, + Project, + ProjectDeployBoard, + ProjectFavorite, + ProjectIdentifier, + ProjectMember, + ProjectMemberInvite, + ProjectPublicMember, + State, + User, + UserNotificationPreference, + Webhook, + Workspace, + WorkspaceMember, + WorkspaceMemberInvite, + WorkspaceTheme, + WorkspaceUserProperties, +) +from plane.utils.exception_logger import log_exception + + +@shared_task +def workspace_import(workspace_data): + try: + # Create Users + users = workspace_data.get("users.json") + User.objects.bulk_create( + [User(**user) for user in json.loads(users)], + batch_size=100, + ignore_conflicts=True, + ) + + # Workspaces + workspaces = workspace_data.get("workspaces.json") + Workspace.objects.bulk_create( + [Workspace(**workspace) for workspace in json.loads(workspaces)], + batch_size=100, + ignore_conflicts=True, + ) + + models = { + # Workspace + WorkspaceMemberInvite: "workspace_member_invites.json", + WorkspaceMember: "workspace_members.json", + WorkspaceTheme: "workspace_themes.json", + WorkspaceUserProperties: "workspace_user_properties.json", + # Projects + Project: "projects.json", + ProjectDeployBoard: "project_deploy_boards.json", + ProjectFavorite: "project_favorites.json", + ProjectIdentifier: "project_identifier.json", + ProjectMember: "project_members.json", + ProjectMemberInvite: "project_member_invites.json", + ProjectPublicMember: "project_public_members.json", + # States + State: "states.json", + # Labels + Label: "labels.json", + # Estimate + Estimate: "estimates.json", + EstimatePoint: "estimate_points.json", + # Issues + Issue: "issues.json", + IssueComment: "issue_comments.json", + IssueAssignee: "issue_assignees.json", + IssueLabel: "issue_labels.json", + IssueLink: "issue_links.json", + IssueMention: "issue_mention.json", + IssueVote: "issue_votes.json", + IssueSubscriber: "issue_subscribers.json", + IssueProperty: "issue_properties.json", + IssueSequence: "issue_sequences.json", + IssueReaction: "issue_reactions.json", + IssueRelation: "issue_relations.json", + IssueAttachment: "issue_attachments.json", + IssueActivity: "issue_activities.json", + # APIToken + APIToken: "api_tokens.json", + # Assets + FileAsset: "file_assets.json", + CommentReaction: "comment_reactions.json", + # Cycles + Cycle: "cycles.json", + CycleIssue: "cycle_issues.json", + CycleFavorite: "cycle_favorites.json", + CycleUserProperties: "cycle_user_properties.json", + # Modules + Module: "modules.json", + ModuleIssue: "module_issues.json", + ModuleFavorite: "module_favorites.json", + ModuleLink: "module_links.json", + ModuleMember: "module_members.json", + ModuleUserProperties: "module_user_properties", + # Page + Page: "pages.json", + PageLog: "page_logs.json", + PageLabel: "page_labels.json", + PageFavorite: "page_favorites.json", + # Webhook + Webhook: "webhooks.json", + # Views + IssueView: "views.json", + IssueViewFavorite: "view_favorites.json", + # Notification + Notification: "notifications.json", + UserNotificationPreference: "user_notification_preferences.json", + # Inbox + Inbox: "inboxes.json", + InboxIssue: "inbox_issues.json", + } + + for model in models: + file_name = models[model] + data = workspace_data.get(file_name) + model.objects.bulk_create( + [model(**model_data) for model_data in json.loads(data)], + batch_size=100, + ignore_conflicts=True, + ) + + return + except Exception as e: + log_exception(e) + return diff --git a/apiserver/plane/license/api/views/__init__.py b/apiserver/plane/license/api/views/__init__.py index 3a66c94c58..3f9c176a49 100644 --- a/apiserver/plane/license/api/views/__init__.py +++ b/apiserver/plane/license/api/views/__init__.py @@ -4,4 +4,6 @@ from .instance import ( InstanceConfigurationEndpoint, InstanceAdminSignInEndpoint, SignUpScreenVisitedEndpoint, + ExportWorkspaceEndpoint, + ImportWorkspaceEndpoint, ) diff --git a/apiserver/plane/license/api/views/instance.py b/apiserver/plane/license/api/views/instance.py index 627904a16d..4e0baef9b0 100644 --- a/apiserver/plane/license/api/views/instance.py +++ b/apiserver/plane/license/api/views/instance.py @@ -1,33 +1,39 @@ # Python imports import uuid +import zipfile # Django imports -from django.utils import timezone from django.contrib.auth.hashers import make_password -from django.core.validators import validate_email from django.core.exceptions import ValidationError +from django.core.validators import validate_email +from django.utils import timezone # Third party imports from rest_framework import status -from rest_framework.response import Response +from rest_framework.parsers import FormParser, JSONParser, MultiPartParser from rest_framework.permissions import AllowAny +from rest_framework.response import Response from rest_framework_simplejwt.tokens import RefreshToken # Module imports +from plane.app.serializers.workspace import WorkspaceLiteSerializer from plane.app.views import BaseAPIView -from plane.license.models import Instance, InstanceAdmin, InstanceConfiguration -from plane.license.api.serializers import ( - InstanceSerializer, - InstanceAdminSerializer, - InstanceConfigurationSerializer, -) +from plane.bgtasks.workspace_export_task import workspace_export +from plane.bgtasks.workspace_import_task import workspace_import +from plane.db.models import User, Workspace from plane.license.api.permissions import ( InstanceAdminPermission, ) -from plane.db.models import User +from plane.license.api.serializers import ( + InstanceAdminSerializer, + InstanceConfigurationSerializer, + InstanceSerializer, +) +from plane.license.models import Instance, InstanceAdmin, InstanceConfiguration from plane.license.utils.encryption import encrypt_data from plane.utils.cache import cache_response, invalidate_cache + class InstanceEndpoint(BaseAPIView): def get_permissions(self): if self.request.method == "PATCH": @@ -272,3 +278,84 @@ class SignUpScreenVisitedEndpoint(BaseAPIView): instance.is_signup_screen_visited = True instance.save() return Response(status=status.HTTP_204_NO_CONTENT) + + +class InstanceWorkspacesEndpoint(BaseAPIView): + + permission_classes = [ + InstanceAdminPermission, + ] + + def get(self, request): + workspaces = Workspace.objects.all() + serializer = WorkspaceLiteSerializer(workspaces, many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + +class ExportWorkspaceEndpoint(BaseAPIView): + + permission_classes = [ + InstanceAdminPermission, + ] + + def post(self, request): + workspace_id = request.data.get("workspace_id", False) + + if not workspace_id: + return Response( + {"error": "Workspace ID is required"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + workspace_export.delay( + workspace_id=workspace_id, + email=request.user.email, + ) + return Response( + { + "message": "An email will be sent to download the exports when they are ready" + }, + status=status.HTTP_200_OK, + ) + + +class ImportWorkspaceEndpoint(BaseAPIView): + + parser_classes = ( + MultiPartParser, + FormParser, + JSONParser, + ) + + permission_classes = [ + InstanceAdminPermission, + ] + + def post(self, request): + file_obj = request.FILES.get("zip_file") + if file_obj is None: + return Response( + "No file uploaded.", status=status.HTTP_400_BAD_REQUEST + ) + + # Ensure the uploaded file is a ZIP file + if not zipfile.is_zipfile(file_obj): + return Response( + "Uploaded file is not a valid zip file.", + status=status.HTTP_400_BAD_REQUEST, + ) + + # Reading contents of the ZIP file + file_contents = {} + with zipfile.ZipFile(file_obj, "r") as zip_ref: + for file_name in zip_ref.namelist(): + with zip_ref.open(file_name) as file: + # Assuming the file content is text. Use file.read() for binary content. + content = file.read().decode("utf-8") + file_contents[file_name] = content + + workspace_import.delay(workspace_data=file_contents) + return Response( + {"message": "Files processed.", "file_count": len(file_contents)}, + status=status.HTTP_200_OK, + ) diff --git a/apiserver/plane/license/urls.py b/apiserver/plane/license/urls.py index e6315e021d..cf303a162b 100644 --- a/apiserver/plane/license/urls.py +++ b/apiserver/plane/license/urls.py @@ -1,10 +1,12 @@ from django.urls import path from plane.license.api.views import ( - InstanceEndpoint, + ExportWorkspaceEndpoint, + ImportWorkspaceEndpoint, InstanceAdminEndpoint, - InstanceConfigurationEndpoint, InstanceAdminSignInEndpoint, + InstanceConfigurationEndpoint, + InstanceEndpoint, SignUpScreenVisitedEndpoint, ) @@ -39,4 +41,14 @@ urlpatterns = [ SignUpScreenVisitedEndpoint.as_view(), name="instance-sign-up", ), + path( + "export-workspace/", + ExportWorkspaceEndpoint.as_view(), + name="workspace-exports", + ), + path( + "import-workspace/", + ImportWorkspaceEndpoint.as_view(), + name="workspace-imports", + ), ]