From 945f82d55650480dca81b113c938bc4255623e3d Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Fri, 14 Jun 2024 15:07:24 +0200 Subject: [PATCH 1/2] django: assign composite data to events All data from a composite request get associated with a new event via eventResources. Signed-off-by: Jonas Remmert --- server/django/sensordata/admin.py | 19 +++++++--- ...time_event_time_remove_event_start_time.py | 21 ++++++++++ server/django/sensordata/models.py | 5 +-- server/django/sensordata/serializers/base.py | 38 +++++++++++++++---- .../composite_resource_serializer.py | 1 + 5 files changed, 68 insertions(+), 16 deletions(-) create mode 100644 server/django/sensordata/migrations/0007_rename_end_time_event_time_remove_event_start_time.py diff --git a/server/django/sensordata/admin.py b/server/django/sensordata/admin.py index 84fd9dd7..70d3a1bb 100644 --- a/server/django/sensordata/admin.py +++ b/server/django/sensordata/admin.py @@ -44,25 +44,34 @@ def get_model_perms(self, request): } +class EventResourceInline(admin.TabularInline): + model = EventResource + extra = 0 + can_delete = False + readonly_fields = ('resource',) + @admin.register(Resource) class ResourceAdmin(admin.ModelAdmin): list_display = ('endpoint', 'resource_type', 'timestamp') search_fields = ('endpoint__endpoint', 'resource_type__name') - list_filter = ('endpoint', 'resource_type', 'timestamp') - + list_filter = ('endpoint__endpoint', 'resource_type', 'timestamp') + readonly_fields = ('endpoint', 'resource_type', 'timestamp', 'int_value', + 'float_value', 'str_value') @admin.register(Event) class EventAdmin(admin.ModelAdmin): - list_display = ('endpoint', 'event_type', 'start_time', 'end_time') - search_fields = ('endpoint__endpoint', 'event_type') + list_display = ('endpoint', 'event_type', 'time') + search_fields = ('endpoint', 'event_type') list_filter = ('endpoint', 'event_type') - + readonly_fields = ('endpoint', 'event_type', 'time') + inlines = [EventResourceInline] @admin.register(EventResource) class EventResourceAdmin(admin.ModelAdmin): list_display = ('event', 'resource') search_fields = ('event__event_type', 'resource__resource_type__name') list_filter = ('event', 'resource__resource_type') + readonly_fields = ('event', 'resource') @admin.register(EndpointOperation) diff --git a/server/django/sensordata/migrations/0007_rename_end_time_event_time_remove_event_start_time.py b/server/django/sensordata/migrations/0007_rename_end_time_event_time_remove_event_start_time.py new file mode 100644 index 00000000..5f8ecafb --- /dev/null +++ b/server/django/sensordata/migrations/0007_rename_end_time_event_time_remove_event_start_time.py @@ -0,0 +1,21 @@ +# Generated by Django 5.0.6 on 2024-06-14 12:36 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("sensordata", "0006_alter_endpointoperation_last_attempt_and_more"), + ] + + operations = [ + migrations.RenameField( + model_name="event", + old_name="end_time", + new_name="time", + ), + migrations.RemoveField( + model_name="event", + name="start_time", + ), + ] diff --git a/server/django/sensordata/models.py b/server/django/sensordata/models.py index 986a1be9..660dfdee 100644 --- a/server/django/sensordata/models.py +++ b/server/django/sensordata/models.py @@ -67,11 +67,10 @@ class Event(models.Model): """ endpoint = models.ForeignKey(Endpoint, on_delete=models.PROTECT) event_type = models.CharField(max_length=100) - start_time = models.DateTimeField() - end_time = models.DateTimeField() + time = models.DateTimeField() def __str__(self): - return f"{self.event_type} from {self.start_time} to {self.end_time} for {self.endpoint}" + return f"{self.endpoint} - {self.event_type} - {self.time}" class EventResource(models.Model): """Acts as a many-to-many bridge table that links resources to their respective events.""" diff --git a/server/django/sensordata/serializers/base.py b/server/django/sensordata/serializers/base.py index 9a33f705..a7e3e015 100644 --- a/server/django/sensordata/serializers/base.py +++ b/server/django/sensordata/serializers/base.py @@ -1,5 +1,5 @@ from rest_framework import serializers -from ..models import ResourceType, Resource +from ..models import ResourceType, Resource, Event, EventResource from django.utils import timezone from ..tasks import process_pending_operations import logging @@ -31,14 +31,31 @@ class ResourceDataSerializer(serializers.Serializer): class HandleResourceMixin: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.event = None - def handle_resource(self, endpoint, obj_id, resource): + + def create_event(self, endpoint, event_type): + """ + Create an Event instance for the given endpoint and event type. If no event + is created, the resource data won't be associated with any event. + """ + event_data = { + 'endpoint': endpoint, + 'event_type': event_type, + 'time': timezone.now() + } + self.event = Event.objects.create(**event_data) + + + def handle_resource(self, endpoint, obj_id, res): # Some LwM2M Resources are currently unsupported, we can skip them for now. - if resource['kind'] == 'multiResource': + if res['kind'] == 'multiResource': logging.error(f"multiResource currently not supported, skipping...") return - res_id = resource['id'] + res_id = res['id'] # Fetch resource information from Database resource_type = ResourceType.objects.get(object_id=obj_id, resource_id=res_id) if not resource_type: @@ -47,10 +64,10 @@ def handle_resource(self, endpoint, obj_id, resource): raise serializers.ValidationError(err) # Validate that datatype is matching the resource type - data_type = dict(ResourceType.TYPE_CHOICES).get(resource['type']) + data_type = dict(ResourceType.TYPE_CHOICES).get(res['type']) res_data_type = dict(ResourceType.TYPE_CHOICES).get(resource_type.data_type) if not data_type: - err = f"Unsupported data type '{resource['type']}', skipping..." + err = f"Unsupported data type '{res['type']}', skipping..." logger.error(err) raise serializers.ValidationError(err) if data_type != res_data_type: @@ -65,10 +82,15 @@ def handle_resource(self, endpoint, obj_id, resource): 'endpoint': endpoint, 'resource_type': resource_type, 'timestamp': timezone.now(), - data_type: resource['value'] + data_type: res['value'] } - Resource.objects.create(**resource_data) + created_res = Resource.objects.create(**resource_data) + + # Create EventResource linking the event and the resource + if self.event is not None: + EventResource.objects.create(event=self.event, resource=created_res) + logger.debug(f"Added EventResource: {self.event} - {created_res}") # Update the registration status if the resource is a registration resource if resource_type.name == 'ep_registered': diff --git a/server/django/sensordata/serializers/composite_resource_serializer.py b/server/django/sensordata/serializers/composite_resource_serializer.py index 932d8c64..c824fa58 100644 --- a/server/django/sensordata/serializers/composite_resource_serializer.py +++ b/server/django/sensordata/serializers/composite_resource_serializer.py @@ -59,6 +59,7 @@ def create(self, validated_data): for _, obj in val.items(): obj_id = obj.get('id') + self.create_event(endpoint, obj_id) for instance in obj['instances']: for resource in instance['resources']: try: From ed0e33fb8850394a219cacf80bf044ee860b6dd7 Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Fri, 14 Jun 2024 15:49:49 +0200 Subject: [PATCH 2/2] django: clean up model permissions and timestamps Signed-off-by: Jonas Remmert --- server/django/sensordata/admin.py | 7 ++-- ...alter_resource_unique_together_and_more.py | 34 +++++++++++++++++++ server/django/sensordata/models.py | 11 +++--- server/django/sensordata/serializers/base.py | 3 -- 4 files changed, 41 insertions(+), 14 deletions(-) create mode 100644 server/django/sensordata/migrations/0008_alter_resource_unique_together_and_more.py diff --git a/server/django/sensordata/admin.py b/server/django/sensordata/admin.py index 70d3a1bb..cc8c7c61 100644 --- a/server/django/sensordata/admin.py +++ b/server/django/sensordata/admin.py @@ -52,11 +52,10 @@ class EventResourceInline(admin.TabularInline): @admin.register(Resource) class ResourceAdmin(admin.ModelAdmin): - list_display = ('endpoint', 'resource_type', 'timestamp') + list_display = ('endpoint', 'resource_type', 'timestamp_created') search_fields = ('endpoint__endpoint', 'resource_type__name') - list_filter = ('endpoint__endpoint', 'resource_type', 'timestamp') - readonly_fields = ('endpoint', 'resource_type', 'timestamp', 'int_value', - 'float_value', 'str_value') + list_filter = ('endpoint__endpoint', 'resource_type', 'timestamp_created') + readonly_fields = ('timestamp_created',) @admin.register(Event) class EventAdmin(admin.ModelAdmin): diff --git a/server/django/sensordata/migrations/0008_alter_resource_unique_together_and_more.py b/server/django/sensordata/migrations/0008_alter_resource_unique_together_and_more.py new file mode 100644 index 00000000..ba368e30 --- /dev/null +++ b/server/django/sensordata/migrations/0008_alter_resource_unique_together_and_more.py @@ -0,0 +1,34 @@ +# Generated by Django 5.0.6 on 2024-06-14 13:46 + +import django.utils.timezone +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("sensordata", "0007_rename_end_time_event_time_remove_event_start_time"), + ] + + operations = [ + migrations.AlterUniqueTogether( + name="resource", + unique_together=set(), + ), + migrations.AddField( + model_name="resource", + name="timestamp_created", + field=models.DateTimeField( + auto_now_add=True, default=django.utils.timezone.now + ), + preserve_default=False, + ), + migrations.AlterField( + model_name="event", + name="time", + field=models.DateTimeField(auto_now_add=True), + ), + migrations.RemoveField( + model_name="resource", + name="timestamp", + ), + ] diff --git a/server/django/sensordata/models.py b/server/django/sensordata/models.py index 660dfdee..201c63e4 100644 --- a/server/django/sensordata/models.py +++ b/server/django/sensordata/models.py @@ -45,13 +45,10 @@ class Resource(models.Model): int_value = models.IntegerField(null=True, blank=True) float_value = models.FloatField(null=True, blank=True) str_value = models.CharField(max_length=512, null=True, blank=True) - timestamp = models.DateTimeField() - - class Meta: - unique_together = ('endpoint', 'resource_type', 'timestamp') + timestamp_created = models.DateTimeField(auto_now_add=True, blank=True) def __str__(self): - return f"{self.endpoint} - {self.resource_type} - {self.timestamp}" + return f"{self.endpoint} - {self.resource_type} - {self.timestamp_created}" # Gets the correct value field, based on the linked ResourceType def get_value(self): @@ -67,7 +64,7 @@ class Event(models.Model): """ endpoint = models.ForeignKey(Endpoint, on_delete=models.PROTECT) event_type = models.CharField(max_length=100) - time = models.DateTimeField() + time = models.DateTimeField(auto_now_add=True, blank=True) def __str__(self): return f"{self.endpoint} - {self.event_type} - {self.time}" @@ -97,7 +94,7 @@ class Status(models.TextChoices): default=Status.QUEUED, ) transmit_counter = models.IntegerField(default=0) - timestamp_created = models.DateTimeField(auto_now_add=True) + timestamp_created = models.DateTimeField(auto_now_add=True, blank=True) last_attempt = models.DateTimeField(auto_now_add=False, null=True) class Firmware(models.Model): diff --git a/server/django/sensordata/serializers/base.py b/server/django/sensordata/serializers/base.py index a7e3e015..32118831 100644 --- a/server/django/sensordata/serializers/base.py +++ b/server/django/sensordata/serializers/base.py @@ -1,6 +1,5 @@ from rest_framework import serializers from ..models import ResourceType, Resource, Event, EventResource -from django.utils import timezone from ..tasks import process_pending_operations import logging @@ -44,7 +43,6 @@ def create_event(self, endpoint, event_type): event_data = { 'endpoint': endpoint, 'event_type': event_type, - 'time': timezone.now() } self.event = Event.objects.create(**event_data) @@ -81,7 +79,6 @@ def handle_resource(self, endpoint, obj_id, res): resource_data = { 'endpoint': endpoint, 'resource_type': resource_type, - 'timestamp': timezone.now(), data_type: res['value'] }