Merge branch 'pr_files_dsyms' into 'master'

Implements /api/0/{org}/{prog}/files/dsyms POST API

See merge request glitchtip/glitchtip-backend!83

difs/permissions.py

@@ -9,3 +9,8 @@ class DifsAssemblePermission(ReleasePermission):
 class ProjectReprocessingPermission(ReleasePermission):
     def get_user_scopes(self, obj, user):
         return obj.get_user_scopes(user)
+
+
+class DymsPermission(ReleasePermission):
+    def get_user_scopes(self, obj, user):
+        return obj.get_user_scopes(user)

difs/tasks.py

@@ -119,7 +119,8 @@ def difs_extract_metadata_from_file(file):
                     "code_id": obj.code_id,
                     "debug_id": obj.debug_id,
                     "kind": obj.kind,
-                    "features": list(obj.features)
+                    "features": list(obj.features),
+                    "symbol_type": "native"
                 }
                 for obj in archive.iter_objects()
             ]
@@ -141,6 +142,7 @@ def difs_create_difs(project, name, file):
         arch = metadata["arch"]
         kind = metadata["kind"]
         features = metadata["features"]
+        symbol_type = metadata["symbol_type"]
 
         dif = DebugInformationFile(
             project=project,
@@ -151,7 +153,8 @@ def difs_create_difs(project, name, file):
                 "debug_id": debug_id,
                 "code_id": code_id,
                 "kind": kind,
-                "features": features
+                "features": features,
+                "symbol_type": symbol_type
             }
         )
         dif.save()

difs/tests.py

@@ -8,9 +8,12 @@ from difs.tasks import (
     difs_concat_file_blobs_to_disk,
     ChecksumMismatched
 )
+from django.core.files.uploadedfile import SimpleUploadedFile
 from files.models import File
 from model_bakery import baker
 from hashlib import sha1
+import contextlib
+from unittest.mock import patch, MagicMock
 
 
 class DifsAssembleAPITestCase(GlitchTipTestCase):
@@ -103,6 +106,137 @@ class DifsAssembleAPITestCase(GlitchTipTestCase):
         self.assertEqual(response.data, expected_response)
 
 
+class DsymsAPIViewTestCase(GlitchTipTestCase):
+    def setUp(self):
+        self.create_user_and_project()
+        self.url = f"/api/0/projects/{self.organization.slug}/{self.project.slug}/files/dsyms/"  # noqa
+        self.uuid = "afb116cf-efec-49af-a7fe-281ac680d8a0"
+        self.checksum = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
+        settings.GLITCHTIP_ENABLE_DIFS = True
+
+    def tearDown(self):
+        settings.GLITCHTIP_ENABLE_DIFS = False
+
+    @contextlib.contextmanager
+    def patch(self):
+        proguard_file = MagicMock()
+        proguard_file.read.return_value = b""
+
+        uploaded_zip_file = MagicMock()
+        uploaded_zip_file.namelist.return_value = iter(
+            [f"proguard/{self.uuid}.txt"])
+        uploaded_zip_file.open.return_value.__enter__.return_value = proguard_file  # noqa
+
+        with patch('zipfile.is_zipfile', return_value=True), \
+                patch('zipfile.ZipFile') as ZipFile:
+
+            ZipFile.return_value.__enter__.return_value = uploaded_zip_file
+            yield
+
+    def test_post(self):
+        """
+        It should return the expected response
+        """
+        upload_file = SimpleUploadedFile(
+            "example.zip",
+            b"random_content",
+            content_type="multipart/form-data"
+        )
+        data = {
+            "file": upload_file
+        }
+
+        with self.patch():
+            response = self.client.post(self.url, data)
+
+        expected_response = [{
+            "id": response.data[0]["id"],
+            "debugId": self.uuid,
+            "cpuName": "any",
+            'objectName': 'proguard-mapping',
+            'symbolType': 'proguard',
+            'headers': {'Content-Type': 'text/x-proguard+plain'},
+            'size': 0,
+            'sha1': self.checksum,
+            "dateCreated": response.data[0]["dateCreated"],
+            "data": {
+                "features": ["mapping"]
+            }
+        }]
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.data), 1)
+        self.assertEqual(response.data, expected_response)
+
+    def test_post_existing_file(self):
+        """
+        It should success and return the expected response
+        """
+
+        baker.make(
+            "files.FileBlob",
+            checksum=self.checksum
+        )
+
+        fileobj = baker.make(
+            "files.File",
+            checksum=self.checksum
+        )
+
+        dif = baker.make(
+            "difs.DebugInformationFile",
+            file=fileobj,
+            project=self.project
+        )
+
+        upload_file = SimpleUploadedFile(
+            "example.zip",
+            b"random_content",
+            content_type="multipart/form-data"
+        )
+        data = {
+            "file": upload_file
+        }
+
+        with self.patch():
+            response = self.client.post(self.url, data)
+
+        expected_response = [{
+            "id": dif.id,
+            "debugId": self.uuid,
+            "cpuName": "any",
+            'objectName': 'proguard-mapping',
+            'symbolType': 'proguard',
+            'headers': {'Content-Type': 'text/x-proguard+plain'},
+            'size': 0,
+            'sha1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
+            "dateCreated": response.data[0]["dateCreated"],
+            "data": {
+                "features": ["mapping"]
+            }
+        }]
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.data), 1)
+        self.assertEqual(response.data, expected_response)
+
+    def test_post_invalid_zip_file(self):
+        upload_file = SimpleUploadedFile(
+            "example.zip",
+            b"random_content",
+            content_type="multipart/form-data"
+        )
+        data = {
+            "file": upload_file
+        }
+        response = self.client.post(self.url, data)
+
+        expected_response = {"error": "Invalid file type uploaded"}
+
+        self.assertEqual(response.data, expected_response)
+        self.assertEqual(response.status_code, 400)
+
+
 class DifsTasksTestCase(GlitchTipTestCase):
     def setUp(self):
         self.create_user_and_project()

difs/urls.py

@@ -1,16 +1,23 @@
 from django.urls import path
-from .views import DifsAssembleAPIView, ProjectReprocessingAPIView
+from .views import (
+    DifsAssembleAPIView, ProjectReprocessingAPIView, DsymsAPIView
+)
 
 
 urlpatterns = [
     path(
-        "projects/<slug:organization_slug>/<slug:project_slug>/files/difs/assemble/", #noqa
+        "projects/<slug:organization_slug>/<slug:project_slug>/files/difs/assemble/",  # noqa
         DifsAssembleAPIView.as_view(),
         name="difs-assemble",
     ),
     path(
-        "projects/<slug:organization_slug>/<slug:project_slug>/reprocessing/",
+        "projects/<slug:organization_slug>/<slug:project_slug>/reprocessing/",  # noqa
         ProjectReprocessingAPIView.as_view(),
         name="project-reporcessing"
+    ),
+    path(
+        "projects/<slug:organization_slug>/<slug:project_slug>/files/dsyms/",  # noqa
+        DsymsAPIView.as_view(),
+        name="dyms"
     )
 ]

difs/views.py

@@ -1,28 +1,46 @@
 """ Port of sentry.api.endpoints.debug_files.DifAssembleEndpoint """
 from django.shortcuts import get_object_or_404
 from django.conf import settings
+from django.db import transaction
 from rest_framework.response import Response
 from organizations_ext.models import Organization
 from projects.models import Project
-from rest_framework import views, exceptions
+from rest_framework import views, exceptions, status
 from .tasks import (
     difs_assemble, DIF_STATE_CREATED, DIF_STATE_OK,
     DIF_STATE_NOT_FOUND
 )
 from .models import DebugInformationFile
-from files.models import FileBlob
+from files.models import FileBlob, File
 from .permissions import (
     DifsAssemblePermission, ProjectReprocessingPermission,
+    DymsPermission
 )
+from django.core.files import File as DjangoFile
+import zipfile
+import io
+import re
+import tempfile
+from hashlib import sha1
+from symbolic import ProguardMapper
+
+
+MAX_UPLOAD_BLOB_SIZE = 8 * 1024 * 1024  # 8MB
+
+
+def check_difs_enabled(func):
+    def wrapper(*args, **kwargs):
+        if settings.GLITCHTIP_ENABLE_DIFS is not True:
+            raise exceptions.PermissionDenied()
+        return func(*args, **kwargs)
+    return wrapper
 
 
 class DifsAssembleAPIView(views.APIView):
     permission_classes = [DifsAssemblePermission]
 
+    @check_difs_enabled
     def post(self, request, organization_slug, project_slug):
-        if settings.GLITCHTIP_ENABLE_DIFS != True:
-            raise exceptions.PermissionDenied()
-
         organization = get_object_or_404(
             Organization,
             slug=organization_slug.lower(),
@@ -82,12 +100,198 @@ class DifsAssembleAPIView(views.APIView):
 
 class ProjectReprocessingAPIView(views.APIView):
     """
-    Non implemented. It is a dummy API to keep `sentry-cli upload-dif` happy
+    Not implemented. It is a dummy API to keep `sentry-cli upload-dif` happy
     """
 
     permission_classes = [ProjectReprocessingPermission]
 
+    @check_difs_enabled
     def post(self, request, organization_slug, project_slug):
-        if settings.GLITCHTIP_ENABLE_DIFS != True:
-            raise exceptions.PermissionDenied()
         return Response()
+
+
+def extract_proguard_id(name):
+    match = re.search('proguard/([-a-fA-F0-9]+).txt', name)
+    if match is None:
+        return
+    return match.group(1)
+
+
+def extract_proguard_metadata(proguard_file):
+    try:
+        mapper = ProguardMapper.open(proguard_file)
+
+        if (mapper is None):
+            return
+
+        metadata = {
+            "arch": "any",
+            "feature": "mapping"
+        }
+
+        return metadata
+
+    except Exception:
+        pass
+
+
+class DsymsAPIView(views.APIView):
+    """
+    Implementation of /files/dsyms API View
+    """
+    permission_classes = [DymsPermission]
+
+    @check_difs_enabled
+    def post(self, request, organization_slug, project_slug):
+        organization = get_object_or_404(
+            Organization,
+            slug=organization_slug.lower(),
+            users=self.request.user
+        )
+
+        self.check_object_permissions(request, organization)
+
+        project = get_object_or_404(
+            Project, slug=project_slug.lower()
+        )
+
+        if project.organization.id != organization.id:
+            raise exceptions.PermissionDenied(
+                "The project is not under this organization")
+
+        if "file" not in request.data:
+            return Response(
+                {"error": "No file uploaded"},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        try:
+            file = request.data["file"]
+            if file.size > MAX_UPLOAD_BLOB_SIZE:
+                return Response(
+                    {"error": "File size too large"},
+                    status=status.HTTP_400_BAD_REQUEST,
+                )
+
+            content = file.read()
+
+            buffer = io.BytesIO(content)
+
+            if zipfile.is_zipfile(buffer) is False:
+                return Response(
+                    {"error": "Invalid file type uploaded"},
+                    status=status.HTTP_400_BAD_REQUEST,
+                )
+
+            results = []
+
+            with zipfile.ZipFile(buffer) as uploaded_zip_file:
+                for filename in uploaded_zip_file.namelist():
+                    proguard_id = extract_proguard_id(filename)
+                    if proguard_id is None:
+                        return Response(
+                            {"error": "Invalid proguard mapping file uploaded"},  # noqa
+                            status=status.HTTP_400_BAD_REQUEST,
+                        )
+
+                    with uploaded_zip_file.open(filename) as proguard_file:
+                        result = self.create_dif_from_read_only_file(
+                            proguard_file,
+                            project,
+                            proguard_id,
+                            filename)
+                        if result is None:
+                            return Response(
+                                {"error": "Invalid proguard mapping file uploaded"},  # noqa
+                                status=status.HTTP_400_BAD_REQUEST,
+                            )
+                        results.append(result)
+
+            return Response(results)
+
+        except Exception as e:
+            return Response(
+                {"error": str(e)},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+    def create_dif_from_read_only_file(
+        self,
+        proguard_file,
+        project,
+        proguard_id,
+        filename
+    ):
+        with tempfile.NamedTemporaryFile("br+") as tmp:
+            content = proguard_file.read()
+            tmp.write(content)
+            tmp.flush()
+            metadata = extract_proguard_metadata(tmp.name)
+            if metadata is None:
+                return None
+            checksum = sha1(content).hexdigest()
+            with transaction.atomic():
+                size = len(content)
+
+                blob = FileBlob.objects.filter(
+                    checksum=checksum
+                ).first()
+
+                if blob is None:
+                    blob = FileBlob(
+                        checksum=checksum,  # noqa
+                        size=size
+                    )
+                    blob.blob.save(filename, DjangoFile(tmp))
+                    blob.save()
+
+                fileobj = File.objects.filter(
+                    checksum=checksum
+                ).first()
+
+                if fileobj is None:
+                    fileobj = File()
+                    fileobj.name = filename
+                    fileobj.headers = {}
+                    fileobj.checksum = checksum
+                    fileobj.size = size
+                    fileobj.save()
+
+                    fileobj.blobs.set([blob])
+
+                dif = DebugInformationFile.objects.filter(
+                    file__checksum=checksum,
+                    project=project
+                ).first()
+
+                if dif is None:
+                    dif = DebugInformationFile()
+                    dif.name = filename
+                    dif.project = project
+                    dif.file = fileobj
+                    dif.data = {
+                        "arch": metadata["arch"],
+                        "debug_id": proguard_id,
+                        "symbol_type": "proguard",
+                        "features": ["mapping"]
+                    }
+                    dif.save()
+
+                result = {
+                    "id": dif.id,
+                    "debugId": proguard_id,
+                    "cpuName": "any",
+                    "objectName": "proguard-mapping",
+                    "symbolType": "proguard",
+                    "size": size,
+                    "sha1": checksum,
+                    "data": {
+                        "features": ["mapping"]
+                    },
+                    "headers": {
+                        "Content-Type": "text/x-proguard+plain"
+                    },
+                    "dateCreated": fileobj.created,
+                }
+
+                return result