trueog/glitchtip-backend
2
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove more of old issues and events

Browse Source
This commit is contained in:
David Burke committed 2024-03-15 10:18:01 -04:00
1 parent 48095e25b5
commit e915961d5b
9 files changed
+390 -313
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

apps/alerts/migrations/0001_squashed_0007_alter_alertrecipient_id_alter_notification_id_and_more.py
-1
View File
@@ -14,7 +14,6 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
("issues", "0001_squashed_0010_auto_20210117_1543"),
(
"projects",
"0001_squashed_0009_alter_project_id_alter_projectcounter_id_and_more",
apps/alerts/migrations/0001_squashed_0010_alter_notification_issues.py
+125
View File
@@ -0,0 +1,125 @@
# Generated by Django 5.0.3 on 2024-03-15 14:05
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
replaces = [
(
"alerts",
"0001_squashed_0007_alter_alertrecipient_id_alter_notification_id_and_more",
),
("alerts", "0008_alter_alertrecipient_recipient_type"),
("alerts", "0009_alter_alertrecipient_recipient_type"),
("alerts", "0010_alter_notification_issues"),
]
initial = True
dependencies = [
("issue_events", "0001_initial"),
(
"projects",
"0001_squashed_0009_alter_project_id_alter_projectcounter_id_and_more",
),
]
operations = [
migrations.CreateModel(
name="ProjectAlert",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"timespan_minutes",
models.PositiveSmallIntegerField(blank=True, null=True),
),
("quantity", models.PositiveSmallIntegerField(blank=True, null=True)),
("created", models.DateTimeField(auto_now_add=True, db_index=True)),
(
"project",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="projects.project",
),
),
("name", models.CharField(blank=True, max_length=255)),
(
"uptime",
models.BooleanField(
default=False,
help_text="Send alert on any uptime monitor check failure",
),
),
],
),
migrations.CreateModel(
name="AlertRecipient",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"recipient_type",
models.CharField(
choices=[
("email", "Email"),
("webhook", "General Slack-compatible webhook"),
("discord", "Discord"),
("googlechat", "Google Chat webhook"),
],
max_length=16,
),
),
("url", models.URLField(blank=True, max_length=2000)),
(
"alert",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="alerts.projectalert",
),
),
],
options={
"unique_together": {("alert", "recipient_type", "url")},
},
),
migrations.CreateModel(
name="Notification",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("created", models.DateTimeField(auto_now_add=True, db_index=True)),
("is_sent", models.BooleanField(default=False)),
("issues", models.ManyToManyField(to="issue_events.issue")),
(
"project_alert",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="alerts.projectalert",
),
),
],
),
]
apps/difs/tasks.py
-40
View File
@@ -11,7 +11,6 @@ from apps.difs.stacktrace_processor import StacktraceProcessor
from apps.event_ingest.schema import ErrorIssueEventSchema, StackTraceFrame
from apps.files.models import File, FileBlob
from apps.projects.models import Project
from events.models import Event
def getLogger():
@@ -48,45 +47,6 @@ def difs_assemble(project_slug, name, checksum, chunks, debug_id):
getLogger().error("difs_assemble: %s", err)
def difs_run_resolve_stacktrace(event_id):
difs_resolve_stacktrace.delay(event_id)
@shared_task
def difs_resolve_stacktrace(event_id):
event = Event.objects.get(event_id=event_id)
event_json = event.data
exception = event_json.get("exception")
if exception is None:
# It is not a crash report event
return
project_id = event.issue.project_id
difs = DebugInformationFile.objects.filter(project_id=project_id).order_by(
"-created"
)
resolved_stracktrackes = []
for dif in difs:
if StacktraceProcessor.is_supported(event_json, dif) is False:
continue
blobs = [dif.file.blob]
with difs_concat_file_blobs_to_disk(blobs) as symbol_file:
remapped_stacktrace = StacktraceProcessor.resolve_stacktrace(
event_json, symbol_file.name
)
if remapped_stacktrace is not None and remapped_stacktrace.score > 0:
resolved_stracktrackes.append(remapped_stacktrace)
if len(resolved_stracktrackes) > 0:
best_remapped_stacktrace = max(
resolved_stracktrackes, key=lambda item: item.score
)
StacktraceProcessor.update_frames(event, best_remapped_stacktrace.frames)
event.save()
def event_difs_resolve_stacktrace(event: ErrorIssueEventSchema, project_id: int):
difs = DebugInformationFile.objects.filter(project_id=project_id).order_by(
"-created"
apps/performance/migrations/0001_initial.py
-1
View File
@@ -12,7 +12,6 @@ class Migration(migrations.Migration):
"projects",
"0001_squashed_0009_alter_project_id_alter_projectcounter_id_and_more",
),
("events", "0001_squashed_0003_auto_20210116_2110"),
]
operations = [
apps/performance/migrations/0001_squashed_0012_alter_span_op.py
+236
View File
@@ -0,0 +1,236 @@
# Generated by Django 5.0.3 on 2024-03-15 14:08
import datetime
import django.contrib.postgres.fields.hstore
import django.contrib.postgres.search
import django.db.models.deletion
import uuid
from django.db import migrations, models
# Functions from the following migrations need manual copying.
# Move them and any dependencies into this file, then update the
# RunPython operations to refer to the local versions:
# apps.performance.migrations.0004_remove_transactionevent_project_and_more
# apps.performance.migrations.0008_transactionevent_duration
class Migration(migrations.Migration):
replaces = [
("performance", "0001_initial"),
("performance", "0002_auto_20210124_1745"),
("performance", "0003_transactionevent"),
("performance", "0004_remove_transactionevent_project_and_more"),
("performance", "0005_alter_span_data"),
("performance", "0006_rename_title_transactiongroup_transaction_and_more"),
(
"performance",
"0007_transactionevent_tags_transactiongroup_search_vector_and_more",
),
("performance", "0008_transactionevent_duration"),
("performance", "0009_alter_span_description"),
("performance", "0010_alter_span_id_alter_transactiongroup_id"),
("performance", "0011_alter_transactionevent_duration"),
("performance", "0012_alter_span_op"),
]
initial = True
dependencies = [
(
"projects",
"0001_squashed_0009_alter_project_id_alter_projectcounter_id_and_more",
),
]
operations = [
migrations.CreateModel(
name="TransactionEvent",
fields=[
(
"event_id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("created", models.DateTimeField(auto_now_add=True, db_index=True)),
(
"data",
models.JSONField(help_text="General event data that is searchable"),
),
(
"timestamp",
models.DateTimeField(
blank=True,
help_text="Date created as claimed by client it came from",
null=True,
),
),
("transaction", models.CharField(max_length=1024)),
("start_timestamp", models.DateTimeField()),
(
"project",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="projects.project",
),
),
],
options={
"ordering": ["-created"],
},
),
migrations.RemoveField(
model_name="transactionevent",
name="project",
),
migrations.AddField(
model_name="transactionevent",
name="trace_id",
field=models.UUIDField(
db_index=True, default="00000000000000000000000000000000"
),
preserve_default=False,
),
migrations.CreateModel(
name="TransactionGroup",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("created", models.DateTimeField(auto_now_add=True, db_index=True)),
("transaction", models.CharField(max_length=1024)),
("op", models.CharField(max_length=255)),
("method", models.CharField(blank=True, max_length=255, null=True)),
(
"project",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="projects.project",
),
),
(
"search_vector",
django.contrib.postgres.search.SearchVectorField(
editable=False, null=True
),
),
("tags", models.JSONField(default=dict)),
],
options={
"unique_together": {("transaction", "project", "op", "method")},
},
),
migrations.AddField(
model_name="transactionevent",
name="group",
field=models.ForeignKey(
default=1,
on_delete=django.db.models.deletion.CASCADE,
to="performance.transactiongroup",
),
preserve_default=False,
),
migrations.CreateModel(
name="Span",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("created", models.DateTimeField(auto_now_add=True, db_index=True)),
("span_id", models.CharField(max_length=16)),
(
"parent_span_id",
models.CharField(blank=True, max_length=16, null=True),
),
("op", models.CharField(max_length=255)),
(
"description",
models.CharField(blank=True, max_length=1024, null=True),
),
("start_timestamp", models.DateTimeField()),
("timestamp", models.DateTimeField()),
(
"tags",
django.contrib.postgres.fields.hstore.HStoreField(default=dict),
),
("data", models.JSONField(default=dict)),
(
"transaction",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="performance.transactionevent",
),
),
],
options={
"abstract": False,
},
),
migrations.RemoveField(
model_name="transactionevent",
name="transaction",
),
migrations.AddField(
model_name="transactionevent",
name="tags",
field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
),
migrations.AddField(
model_name="transactionevent",
name="duration",
field=models.DurationField(db_index=True, default=datetime.timedelta(0)),
preserve_default=False,
),
migrations.AlterField(
model_name="span",
name="description",
field=models.CharField(blank=True, max_length=2000, null=True),
),
migrations.AlterField(
model_name="span",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.AlterField(
model_name="transactiongroup",
name="id",
field=models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
migrations.RemoveField(
model_name="transactionevent",
name="duration",
),
migrations.AddField(
model_name="transactionevent",
name="duration",
field=models.PositiveIntegerField(
db_index=True, default=0, help_text="Milliseconds"
),
preserve_default=False,
),
migrations.AlterField(
model_name="span",
name="op",
field=models.CharField(blank=True, max_length=255),
),
]
apps/performance/models.py
+26 -1
View File
@@ -1,12 +1,37 @@
import uuid
from django.contrib.postgres.fields import HStoreField
from django.contrib.postgres.search import SearchVectorField
from django.db import models
from apps.projects.tasks import update_transaction_event_project_hourly_statistic
from events.models import AbstractEvent
from glitchtip.base_models import CreatedModel
class AbstractEvent(CreatedModel):
event_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
data = models.JSONField(help_text="General event data that is searchable")
timestamp = models.DateTimeField(
blank=True,
null=True,
help_text="Date created as claimed by client it came from",
)
class Meta:
abstract = True
def __str__(self):
return self.event_id_hex
@property
def event_id_hex(self):
"""The public key without dashes"""
if self.event_id:
if isinstance(self.event_id, str):
return self.event_id
return self.event_id.hex
class TransactionGroup(CreatedModel):
transaction = models.CharField(max_length=1024)
project = models.ForeignKey("projects.Project", on_delete=models.CASCADE)
events/migrations/0004_auto_20240312_1344.py
+2
View File
@@ -30,6 +30,8 @@ def daterange(start_date, end_date):
def migrate_issue_events(apps, schema_editor):
if settings.TESTING:
return
today = now()
days_ago = timedelta(days=settings.GLITCHTIP_MAX_EVENT_LIFE_DAYS + 1)
with connection.cursor() as cursor:
events/models.py
-159
View File
@@ -1,37 +1,6 @@
import uuid
from django.contrib.postgres.fields import HStoreField
from django.db import models
from apps.projects.tasks import update_event_project_hourly_statistic
from glitchtip.base_models import CreatedModel
from glitchtip.model_utils import FromStringIntegerChoices
class AbstractEvent(CreatedModel):
event_id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
data = models.JSONField(help_text="General event data that is searchable")
timestamp = models.DateTimeField(
blank=True,
null=True,
help_text="Date created as claimed by client it came from",
)
class Meta:
abstract = True
def __str__(self):
return self.event_id_hex
@property
def event_id_hex(self):
"""The public key without dashes"""
if self.event_id:
if isinstance(self.event_id, str):
return self.event_id
return self.event_id.hex
class LogLevel(FromStringIntegerChoices):
NOTSET = 0, "sample"
DEBUG = 1, "debug"
@@ -50,131 +19,3 @@ class LogLevel(FromStringIntegerChoices):
if string == "log":
return cls.INFO
return cls.ERROR
class Event(AbstractEvent):
"""
An individual event. An issue is a set of like-events.
Most is stored in "data" but some fields benefit from being real
relational data types such as dates and foreign keys
"""
issue = models.ForeignKey(
"issues.Issue",
on_delete=models.CASCADE,
help_text="Sentry calls this a group",
)
level = models.PositiveSmallIntegerField(
choices=LogLevel.choices, default=LogLevel.ERROR
)
errors = models.JSONField(
null=True,
blank=True,
help_text="Event processing errors from event intake, including validation errors",
)
tags = HStoreField(default=dict)
release = models.ForeignKey(
"releases.Release", blank=True, null=True, on_delete=models.SET_NULL
)
class Meta:
ordering = ["-created"]
def save(self, *args, **kwargs):
is_new = self._state.adding
super().save(*args, **kwargs)
if is_new:
update_event_project_hourly_statistic(
args=[self.issue.project_id, self.created], countdown=60
)
def event_json(self):
"""
OSS Sentry Compatible raw event JSON
Effectively this combines data and relational data
"""
event = self.data
event["event_id"] = self.event_id_hex
event["project"] = self.issue.project_id
event["level"] = self.get_level_display()
event["tags"] = self.tags.items()
if self.errors:
event["errors"] = self.errors
if self.timestamp:
event["datetime"] = self.timestamp.isoformat().replace("+00:00", "Z")
if self.release:
event["release"] = self.release.version
return event
@property
def context(self):
return self.data.get("extra")
@property
def contexts(self):
return self.data.get("contexts")
@property
def culprit(self):
return self.data.get("culprit")
@property
def message(self):
return self.data.get("message")
@property
def user(self):
return self.data.get("user")
def _build_context(self, context: list, base_line_no: int, is_pre: bool):
context_length = len(context)
result = []
for index, pre_context_line in enumerate(context):
if is_pre:
line_no = base_line_no - context_length + index
else:
line_no = base_line_no + 1 + index
result.append(
[
line_no,
pre_context_line,
]
)
return result
@property
def metadata(self):
return self.data.get("metadata")
@property
def packages(self):
return self.data.get("modules")
@property
def platform(self):
return self.data.get("platform")
@property
def sdk(self):
return self.data.get("sdk")
@property
def title(self):
return self.data.get("title")
@property
def type(self):
return self.data.get("type")
def next(self, *args, **kwargs):
try:
return self.get_next_by_created(**kwargs)
except Event.DoesNotExist:
return None
def previous(self, *args, **kwargs):
"""Get previous object by created date, pass filter kwargs"""
try:
return self.get_previous_by_created(**kwargs)
except Event.DoesNotExist:
return None
issues/models.py
+1 -111
View File
@@ -1,14 +1,7 @@
from django.conf import settings
from django.contrib.postgres.indexes import GinIndex
from django.contrib.postgres.search import SearchVectorField
from django.db import connection, models
from django.db import models
from events.models import LogLevel
from glitchtip.base_models import CreatedModel
from glitchtip.model_utils import FromStringIntegerChoices
from .utils import base32_encode
class EventType(models.IntegerChoices):
DEFAULT = 0, "default"
@@ -21,106 +14,3 @@ class EventStatus(FromStringIntegerChoices):
UNRESOLVED = 0, "unresolved"
RESOLVED = 1, "resolved"
IGNORED = 2, "ignored"
class Issue(CreatedModel):
"""
Sentry called this a "group". A issue is a collection of events with meta data
such as resolved status.
"""
# annotations Not implemented
# assigned_to Not implemented
culprit = models.CharField(max_length=1024, blank=True, null=True)
has_seen = models.BooleanField(default=False)
# is_bookmarked Not implement - is per user
is_public = models.BooleanField(default=False)
level = models.PositiveSmallIntegerField(
choices=LogLevel.choices, default=LogLevel.ERROR
)
metadata = models.JSONField()
tags = models.JSONField(default=dict)
project = models.ForeignKey("projects.Project", on_delete=models.CASCADE)
title = models.CharField(max_length=255)
type = models.PositiveSmallIntegerField(
choices=EventType.choices, default=EventType.DEFAULT
)
status = models.PositiveSmallIntegerField(
choices=EventStatus.choices, default=EventStatus.UNRESOLVED
)
# See migration 0004 for trigger that sets search_vector, count, last_seen
short_id = models.PositiveIntegerField(null=True)
search_vector = SearchVectorField(null=True, editable=False)
count = models.PositiveIntegerField(default=1, editable=False)
last_seen = models.DateTimeField(auto_now_add=True, db_index=True)
class Meta:
unique_together = (("project", "short_id"),)
indexes = [GinIndex(fields=["search_vector"], name="search_vector_idx")]
def event(self):
return self.event_set.first()
def __str__(self):
return self.title
def check_for_status_update(self):
"""
Determine if issue should regress back to unresolved
Typically run when processing a new event related to the issue
"""
if self.status == EventStatus.RESOLVED:
self.status = EventStatus.UNRESOLVED
self.save()
# Delete notifications so that new alerts are sent for regressions
self.notification_set.all().delete()
def get_hex_color(self):
if self.level == LogLevel.INFO:
return "#4b60b4"
elif self.level is LogLevel.WARNING:
return "#e9b949"
elif self.level in [LogLevel.ERROR, LogLevel.FATAL]:
return "#e52b50"
@property
def short_id_display(self):
"""
Short IDs are per project issue counters. They show as PROJECT_SLUG-ID_BASE32
The intention is to be human readable identifiers that can reference an issue.
"""
if self.short_id is not None:
return f"{self.project.slug.upper()}-{base32_encode(self.short_id)}"
return ""
def get_detail_url(self):
return f"{settings.GLITCHTIP_URL.geturl()}/{self.project.organization.slug}/issues/{self.pk}"
@classmethod
def update_index(cls, issue_id: int):
"""
Update search index/tag aggregations
"""
with connection.cursor() as cursor:
cursor.execute("CALL update_issue_index(%s)", [issue_id])
class IssueHash(models.Model):
issue = models.ForeignKey("issues.Issue", on_delete=models.CASCADE)
# Redundant project allows for unique constraint
project = models.ForeignKey("projects.Project", on_delete=models.CASCADE)
value = models.UUIDField(db_index=True)
class Meta:
constraints = [
models.UniqueConstraint(fields=["project", "value"], name="project hash")
]
class Comment(CreatedModel):
issue = models.ForeignKey("issues.Issue", on_delete=models.CASCADE)
user = models.ForeignKey("users.User", null=True, on_delete=models.SET_NULL)
text = models.TextField(blank=True, null=True)
class Meta:
ordering = ("-created",)