From d5563370bbd7767300118a603f329006d55a220a Mon Sep 17 00:00:00 2001 From: David Burke Date: Fri, 15 Mar 2024 10:41:49 -0400 Subject: [PATCH] Remove more issues/events --- apps/alerts/tests/test_webhooks.py | 2 +- apps/alerts/webhooks.py | 2 - apps/djstripe_ext/tests/test_tasks.py | 4 +- apps/djstripe_ext/tests/tests.py | 12 +- apps/issue_events/api/issues.py | 3 +- .../management/commands/ingest_benchmark.py | 2 - apps/issue_events/tests/test_issues_api.py | 3 +- apps/observability/tests.py | 2 +- .../tests/test_throttling.py | 20 +++- apps/projects/models.py | 25 ++--- apps/projects/tests/test_statistics.py | 2 +- apps/stats/tests/test_api_permissions.py | 2 +- apps/stats/tests/tests.py | 2 +- events/fields.py | 104 ------------------ events/models.py | 21 ---- events/utils.py | 26 ----- issues/models.py | 16 --- issues/utils.py | 43 -------- locustfile.py | 1 - 19 files changed, 43 insertions(+), 249 deletions(-) delete mode 100644 events/fields.py delete mode 100644 events/models.py delete mode 100644 events/utils.py delete mode 100644 issues/models.py delete mode 100644 issues/utils.py diff --git a/apps/alerts/tests/test_webhooks.py b/apps/alerts/tests/test_webhooks.py index 89d2610e..97d69bc6 100644 --- a/apps/alerts/tests/test_webhooks.py +++ b/apps/alerts/tests/test_webhooks.py @@ -3,7 +3,7 @@ from unittest import mock from django.test import TestCase from model_bakery import baker -from events.models import LogLevel +from apps.issue_events.constants import LogLevel from ..constants import RecipientType from ..models import Notification diff --git a/apps/alerts/webhooks.py b/apps/alerts/webhooks.py index a5c371a2..96cbb111 100644 --- a/apps/alerts/webhooks.py +++ b/apps/alerts/webhooks.py @@ -7,8 +7,6 @@ from django.conf import settings from .constants import RecipientType if TYPE_CHECKING: - from issues.models import Issue - from .models import Notification diff --git a/apps/djstripe_ext/tests/test_tasks.py b/apps/djstripe_ext/tests/test_tasks.py index 199c2ada..5a2698ca 100644 --- a/apps/djstripe_ext/tests/test_tasks.py +++ b/apps/djstripe_ext/tests/test_tasks.py @@ -41,7 +41,7 @@ class OrganizationWarnThrottlingTestCase(TestCase): subscription.current_period_start + timedelta(days=30) ) subscription.save() - baker.make("events.Event", issue__project=project, _quantity=9) + baker.make("issue_events.IssueEvent", issue__project=project, _quantity=9) warn_organization_throttle() self.assertEqual(len(mail.outbox), 1) warn_organization_throttle() @@ -58,6 +58,6 @@ class OrganizationWarnThrottlingTestCase(TestCase): warn_organization_throttle() self.assertEqual(len(mail.outbox), 1) - baker.make("events.Event", issue__project=project, _quantity=9) + baker.make("issue_events.IssueEvent", issue__project=project, _quantity=9) warn_organization_throttle() self.assertEqual(len(mail.outbox), 2) diff --git a/apps/djstripe_ext/tests/tests.py b/apps/djstripe_ext/tests/tests.py index 9f1d96e6..6ef69407 100644 --- a/apps/djstripe_ext/tests/tests.py +++ b/apps/djstripe_ext/tests/tests.py @@ -88,10 +88,16 @@ class SubscriptionAPITestCase(APITestCase): + "events_count/" ) with freeze_time(timezone.datetime(2020, 3, 1)): - baker.make("events.Event", issue__project__organization=self.organization) + baker.make( + "issue_events.IssueEvent", + issue__project__organization=self.organization, + ) with freeze_time(timezone.datetime(2020, 1, 5)): - baker.make("events.Event") - baker.make("events.Event", issue__project__organization=self.organization) + baker.make("issue_events.IssueEvent") + baker.make( + "issue_events.IssueEvent", + issue__project__organization=self.organization, + ) baker.make( "performance.TransactionEvent", group__project__organization=self.organization, diff --git a/apps/issue_events/api/issues.py b/apps/issue_events/api/issues.py index badf6595..9f5f0e0d 100644 --- a/apps/issue_events/api/issues.py +++ b/apps/issue_events/api/issues.py @@ -15,13 +15,12 @@ from pydantic.functional_validators import BeforeValidator from typing_extensions import Annotated from apps.organizations_ext.models import Organization -from events.models import LogLevel from glitchtip.api.authentication import AuthHttpRequest from glitchtip.api.pagination import paginate from glitchtip.api.permissions import has_permission from glitchtip.utils import async_call_celery_task -from ..constants import EventStatus +from ..constants import EventStatus, LogLevel from ..models import Issue from ..schema import IssueDetailSchema, IssueSchema, IssueTagSchema from ..tasks import delete_issue_task diff --git a/apps/issue_events/management/commands/ingest_benchmark.py b/apps/issue_events/management/commands/ingest_benchmark.py index caf18a82..61a829cd 100644 --- a/apps/issue_events/management/commands/ingest_benchmark.py +++ b/apps/issue_events/management/commands/ingest_benchmark.py @@ -8,9 +8,7 @@ from django.urls import reverse from apps.organizations_ext.models import Organization from apps.projects.models import Project -from events.models import Event from events.test_data.event_generator import get_seeded_benchmark_events -from events.views import EventStoreAPIView class Command(BaseCommand): diff --git a/apps/issue_events/tests/test_issues_api.py b/apps/issue_events/tests/test_issues_api.py index 010026ab..da5b3b96 100644 --- a/apps/issue_events/tests/test_issues_api.py +++ b/apps/issue_events/tests/test_issues_api.py @@ -11,10 +11,9 @@ from freezegun import freeze_time from model_bakery import baker from apps.event_ingest.model_functions import PipeConcat -from events.models import LogLevel from glitchtip.test_utils.test_case import APIPermissionTestCase, GlitchTipTestCaseMixin -from ..constants import EventStatus +from ..constants import EventStatus, LogLevel from ..models import Issue logger = logging.getLogger(__name__) diff --git a/apps/observability/tests.py b/apps/observability/tests.py index 6e155e45..211d8f1a 100644 --- a/apps/observability/tests.py +++ b/apps/observability/tests.py @@ -104,7 +104,7 @@ class ObservabilityAPITestCase(APITestCase): self.assertEqual(projs_metric, 1) # delete project - proj.delete() + proj.force_delete() # test metrics = self._get_metrics() diff --git a/apps/organizations_ext/tests/test_throttling.py b/apps/organizations_ext/tests/test_throttling.py index 28cc18dc..a26b9340 100644 --- a/apps/organizations_ext/tests/test_throttling.py +++ b/apps/organizations_ext/tests/test_throttling.py @@ -31,14 +31,18 @@ class OrganizationThrottlingTestCase(TestCase): current_period_end=timezone.make_aware(timezone.datetime(2000, 1, 31)), ) baker.make( - "events.Event", issue__project__organization=organization, _quantity=3 + "issue_events.IssueEvent", + issue__project__organization=organization, + _quantity=3, ) set_organization_throttle() organization.refresh_from_db() self.assertTrue(organization.is_accepting_events) baker.make( - "events.Event", issue__project__organization=organization, _quantity=8 + "issue_events.IssueEvent", + issue__project__organization=organization, + _quantity=8, ) set_organization_throttle() organization.refresh_from_db() @@ -60,7 +64,9 @@ class OrganizationThrottlingTestCase(TestCase): # Throttle again baker.make( - "events.Event", issue__project__organization=organization, _quantity=10 + "issue_events.IssueEvent", + issue__project__organization=organization, + _quantity=10, ) baker.make( "performance.TransactionEvent", @@ -90,7 +96,7 @@ class OrganizationThrottlingTestCase(TestCase): status="active", current_period_end=timezone.make_aware(timezone.datetime(2000, 2, 1)), ) - baker.make("events.Event", issue__project=project, _quantity=3) + baker.make("issue_events.IssueEvent", issue__project=project, _quantity=3) baker.make( "performance.TransactionEvent", group__project=project, @@ -117,7 +123,9 @@ class OrganizationThrottlingTestCase(TestCase): status="active", ) baker.make( - "events.Event", issue__project__organization=organization, _quantity=2 + "issue_events.IssueEvent", + issue__project__organization=organization, + _quantity=2, ) with self.assertNumQueries(4): set_organization_throttle() @@ -131,7 +139,7 @@ class OrganizationThrottlingTestCase(TestCase): user = baker.make("users.user") organization.add_user(user) project = baker.make("projects.Project", organization=organization) - baker.make("events.Event", issue__project=project, _quantity=2) + baker.make("issue_events.IssueEvent", issue__project=project, _quantity=2) set_organization_throttle() organization.refresh_from_db() self.assertFalse(organization.is_accepting_events) diff --git a/apps/projects/models.py b/apps/projects/models.py index 35ccca8e..e6ac7e6b 100644 --- a/apps/projects/models.py +++ b/apps/projects/models.py @@ -10,6 +10,7 @@ from django.db.models import Count, Q from django.utils.text import slugify from django_extensions.db.fields import AutoSlugField +from apps.issue_events.models import Issue, IssueEvent from apps.observability.metrics import clear_metrics_cache from glitchtip.base_models import CreatedModel, SoftDeleteModel @@ -64,16 +65,12 @@ class Project(CreatedModel, SoftDeleteModel): def force_delete(self, *args, **kwargs): """Really delete the project and all related data.""" - # avoid circular import - from events.models import Event - from issues.models import Issue - # bulk delete all events - events_qs = Event.objects.filter(issue__project=self) + events_qs = IssueEvent.objects.filter(issue__project=self) events_qs._raw_delete(events_qs.db) # bulk delete all issues in batches of 1k - issues_qs = self.issue_set.order_by("id") + issues_qs = self.issues.order_by("id") while True: try: issue_delimiter = issues_qs.values_list("id", flat=True)[ @@ -285,21 +282,21 @@ class EventProjectHourlyStatistic(ProjectStatisticBase): ): # Redundant filter optimization - otherwise all rows are scanned return project_queryset.filter( - issue__event__created__gte=previous_hour, - issue__event__created__lt=next_hour, + issues__issueevent__received__gte=previous_hour, + issues__issueevent__received__lt=next_hour, ).aggregate( previous_hour_count=Count( - "issue__event", + "issues__issueevent", filter=Q( - issue__event__created__gte=previous_hour, - issue__event__created__lt=current_hour, + issues__issueevent__received__gte=previous_hour, + issues__issueevent__received__lt=current_hour, ), ), current_hour_count=Count( - "issue__event", + "issues__issueevent", filter=Q( - issue__event__created__gte=current_hour, - issue__event__created__lt=next_hour, + issues__issueevent__received__gte=current_hour, + issues__issueevent__received__lt=next_hour, ), ), ) diff --git a/apps/projects/tests/test_statistics.py b/apps/projects/tests/test_statistics.py index bd99d4fd..afa83982 100644 --- a/apps/projects/tests/test_statistics.py +++ b/apps/projects/tests/test_statistics.py @@ -11,7 +11,7 @@ class ProjectStatisticsTestCase(GlitchTipTestCase): self.project = baker.make("projects.Project") def test_event_update_count(self): - baker.make("events.Event", issue__project=self.project) + baker.make("issue_events.IssueEvent", issue__project=self.project) EventProjectHourlyStatistic.update(self.project.pk, timezone.now()) self.assertTrue( EventProjectHourlyStatistic.objects.filter( diff --git a/apps/stats/tests/test_api_permissions.py b/apps/stats/tests/test_api_permissions.py index fdf8e945..4b55434a 100644 --- a/apps/stats/tests/test_api_permissions.py +++ b/apps/stats/tests/test_api_permissions.py @@ -9,7 +9,7 @@ class StatsAPIPermissionTests(APIPermissionTestCase): self.create_user_org() self.set_client_credentials(self.auth_token.token) self.event = baker.make( - "events.Event", issue__project__organization=self.organization + "issue_events.IssueEvent", issue__project__organization=self.organization ) self.url = reverse( "stats-v2", kwargs={"organization_slug": self.organization.slug} diff --git a/apps/stats/tests/tests.py b/apps/stats/tests/tests.py index 0e4ff28f..36a6fd20 100644 --- a/apps/stats/tests/tests.py +++ b/apps/stats/tests/tests.py @@ -13,7 +13,7 @@ class StatsV2APITestCase(GlitchTipTestCase): ) def test_get(self): - baker.make("events.Event", issue__project=self.project) + baker.make("issue_events.IssueEvent", issue__project=self.project) start = timezone.now() - timezone.timedelta(hours=2) end = timezone.now() res = self.client.get( diff --git a/events/fields.py b/events/fields.py deleted file mode 100644 index a23c0693..00000000 --- a/events/fields.py +++ /dev/null @@ -1,104 +0,0 @@ -import re -from typing import List -from urllib.parse import parse_qs - -from rest_framework import serializers -from rest_framework.exceptions import ErrorDetail, ValidationError - - -class ErrorValueDetail(ErrorDetail): - """Extended ErrorDetail with validation value""" - - value = None - - def __new__(cls, string, code=None, value=None): - self = super().__new__(cls, string, code) - self.value = value - return self - - def __repr__(self): - return "ErrorDetail(string=%r, code=%r, value=%r)" % ( - str(self), - self.code, - self.value, - ) - - -class GenericField(serializers.Field): - def to_internal_value(self, data): - return data - - -class ForgivingFieldMixin: - def update_handled_errors_context(self, errors: List[ErrorValueDetail]): - if errors: - handled_errors = self.context.get("handled_errors", {}) - self.context["handled_errors"] = handled_errors | {self.field_name: errors} - - -class ForgivingHStoreField(ForgivingFieldMixin, serializers.HStoreField): - def run_child_validation(self, data): - result = {} - errors: List[ErrorValueDetail] = [] - - for key, value in data.items(): - if value is None: - continue - key = str(key) - - try: - result[key] = self.child.run_validation(value) - except ValidationError as e: - for detail in e.detail: - errors.append(ErrorValueDetail(str(detail), detail.code, value)) - - if errors: - self.update_handled_errors_context(errors) - return result - - -class ForgivingDisallowRegexField(ForgivingFieldMixin, serializers.CharField): - """Disallow bad matches, set disallow_regex kwarg to use""" - - def __init__(self, **kwargs): - self.disallow_regex = kwargs.pop("disallow_regex", None) - super().__init__(**kwargs) - - def to_internal_value(self, data): - data = super().to_internal_value(data) - if self.disallow_regex: - pattern = re.compile(self.disallow_regex) - if pattern.match(data) is None: - error = ErrorValueDetail( - "invalid characters in string", "invalid_data", data - ) - self.update_handled_errors_context([error]) - return None - return data - - -class QueryStringField(serializers.ListField): - """ - Can be given as unparsed string, dictionary, or list of tuples - Should store as List[List[str]] where inner List is always of length 2 - """ - - child = serializers.ListField(child=serializers.CharField()) - - def to_internal_value(self, data): - if isinstance(data, str) and data: - qs = parse_qs(data) - result = [] - for key, values in qs.items(): - for value in values: - result.append([key, value]) - return result - elif isinstance(data, dict): - return [[key, value] for key, value in data.items()] - elif isinstance(data, list): - result = [] - for item in data: - if isinstance(item, list) and len(item) >= 2: - result.append(item[:2]) - return result - return None diff --git a/events/models.py b/events/models.py deleted file mode 100644 index dc50eef9..00000000 --- a/events/models.py +++ /dev/null @@ -1,21 +0,0 @@ -from glitchtip.model_utils import FromStringIntegerChoices - - -class LogLevel(FromStringIntegerChoices): - NOTSET = 0, "sample" - DEBUG = 1, "debug" - INFO = 2, "info" - WARNING = 3, "warning" - ERROR = 4, "error" - FATAL = 5, "fatal" - - @classmethod - def from_string(cls, string: str): - result = super().from_string(string) - if result: - return result - if string == "critical": - return cls.FATAL - if string == "log": - return cls.INFO - return cls.ERROR diff --git a/events/utils.py b/events/utils.py deleted file mode 100644 index 3764974a..00000000 --- a/events/utils.py +++ /dev/null @@ -1,26 +0,0 @@ -import hashlib -from typing import List, Optional - -from issues.models import EventType - - -def default_hash_input(title: str, culprit: str, type: EventType) -> str: - return title + culprit + str(type) - - -def generate_hash( - title: str, culprit: str, type: EventType, extra: Optional[List[str]] = None -) -> str: - """Generate insecure hash used for grouping issues""" - if extra: - hash_input = "".join( - [ - default_hash_input(title, culprit, type) - if part == "{{ default }}" - else part - for part in extra - ] - ) - else: - hash_input = default_hash_input(title, culprit, type) - return hashlib.md5(hash_input.encode()).hexdigest() diff --git a/issues/models.py b/issues/models.py deleted file mode 100644 index 50eda4e1..00000000 --- a/issues/models.py +++ /dev/null @@ -1,16 +0,0 @@ -from django.db import models - -from glitchtip.model_utils import FromStringIntegerChoices - - -class EventType(models.IntegerChoices): - DEFAULT = 0, "default" - ERROR = 1, "error" - CSP = 2, "csp" - TRANSACTION = 3, "transaction" - - -class EventStatus(FromStringIntegerChoices): - UNRESOLVED = 0, "unresolved" - RESOLVED = 1, "resolved" - IGNORED = 2, "ignored" diff --git a/issues/utils.py b/issues/utils.py deleted file mode 100644 index 5e15e92f..00000000 --- a/issues/utils.py +++ /dev/null @@ -1,43 +0,0 @@ -import string - -digs = string.digits + string.ascii_uppercase - - -def base32_decode(base32_value: str) -> int: - """ - Convert base32 string to integer - Example 'A' -> 10 - """ - return int(base32_value, 32) - - -def int2base(x: int, base: int) -> str: - """ - Convert base 10 integer to any base string that can be represented with numbers and - upper case letters - Example int2base(10, 32) -> 'A' - Source: https://stackoverflow.com/a/2267446/443457 - """ - if x < 0: - sign = -1 - elif x == 0: - return digs[0] - else: - sign = 1 - x *= sign - digits = [] - while x: - digits.append(digs[int(x % base)]) - x = int(x / base) - if sign < 0: - digits.append("-") - digits.reverse() - return "".join(digits) - - -def base32_encode(base10_value: int) -> str: - """ - Convert base 10 integer to base32 string - Example 10 -> 'A' - """ - return int2base(base10_value, 32) diff --git a/locustfile.py b/locustfile.py index e0a16b73..4bb53bf0 100644 --- a/locustfile.py +++ b/locustfile.py @@ -1,5 +1,4 @@ import datetime -import os import time from celery.result import AsyncResult