From 6a5cec8cabdab755e612294a47227025ef17949f Mon Sep 17 00:00:00 2001 From: Jonas Remmert Date: Sat, 11 May 2024 00:28:17 +0200 Subject: [PATCH] django: Adapt to the new database model - Rewrite the serializer for single and multi resource - Adapt admin dashboard - Remove listView for now, admin dashboard is sufficient for testing currently Signed-off-by: Jonas Remmert --- server/django/sensordata/admin.py | 50 +++- server/django/sensordata/lwm2m_mappings.py | 23 ++ server/django/sensordata/models.py | 42 +-- server/django/sensordata/serializers.py | 254 +++++++++--------- .../templates/sensordata/resource_list.html | 30 +++ .../sensordata/timetemperature_list.html | 20 -- .../sensordata/tests/test_restapi_multi.py | 25 +- .../sensordata/tests/test_restapi_single.py | 59 ++-- server/django/sensordata/views.py | 16 +- server/django/server/settings.py | 2 +- server/django/server/urls.py | 2 - 11 files changed, 285 insertions(+), 238 deletions(-) create mode 100644 server/django/sensordata/lwm2m_mappings.py create mode 100644 server/django/sensordata/templates/sensordata/resource_list.html delete mode 100644 server/django/sensordata/templates/sensordata/timetemperature_list.html diff --git a/server/django/sensordata/admin.py b/server/django/sensordata/admin.py index 8c38f3f3..03e80cf1 100644 --- a/server/django/sensordata/admin.py +++ b/server/django/sensordata/admin.py @@ -1,3 +1,51 @@ from django.contrib import admin +from .models import ( + Device, + ResourceType, + Resource, + Event, + EventResource, + DeviceOperation, + Firmware +) -# Register your models here. +@admin.register(Device) +class DeviceAdmin(admin.ModelAdmin): + list_display = ('device_id', 'name') + search_fields = ('device_id', 'name') + +@admin.register(ResourceType) +class ResourceTypeAdmin(admin.ModelAdmin): + list_display = ('object_id', 'resource_id', 'name', 'data_type') + search_fields = ('object_id', 'resource_id', 'name') + +@admin.register(Resource) +class ResourceAdmin(admin.ModelAdmin): + list_display = ('device', 'resource_type', 'timestamp') + search_fields = ('device__device_id', 'resource_type__name') + list_filter = ('device', 'resource_type', 'timestamp') + +@admin.register(Event) +class EventAdmin(admin.ModelAdmin): + list_display = ('device', 'event_type', 'start_time', 'end_time') + search_fields = ('device__device_id', 'event_type') + list_filter = ('device', 'event_type') + +@admin.register(EventResource) +class EventResourceAdmin(admin.ModelAdmin): + list_display = ('event', 'resource') + search_fields = ('event__event_type', 'resource__resource_type__name') + list_filter = ('event', 'resource') + +@admin.register(DeviceOperation) +class DeviceOperationAdmin(admin.ModelAdmin): + list_display = ('resource', 'operation_type', 'status', 'timestamp_sent', + 'retransmit_counter', 'last_attempt') + search_fields = ('resource__device__device_id', 'operation_type', 'status') + list_filter = ('resource', 'operation_type', 'status', 'timestamp_sent') + +@admin.register(Firmware) +class FirmwareAdmin(admin.ModelAdmin): + list_display = ('version', 'file_name', 'download_url', 'created_at') + search_fields = ('version', 'file_name') + list_filter = ('created_at',) diff --git a/server/django/sensordata/lwm2m_mappings.py b/server/django/sensordata/lwm2m_mappings.py new file mode 100644 index 00000000..af8304f4 --- /dev/null +++ b/server/django/sensordata/lwm2m_mappings.py @@ -0,0 +1,23 @@ +LWM2M_RESOURCE_MAP = { + (3303, 5700): {"name": "temperature", "data_type": "float"}, + (3303, 5701): {"name": "humidity", "data_type": "float"}, + (3, 0): {"name": "manufacturer", "data_type": "string"}, + (3, 1): {"name": "model_number", "data_type": "string"}, + (3, 2): {"name": "serial_number", "data_type": "string"}, + (3, 3): {"name": "firmware_version", "data_type": "string"}, + (3, 6): {"name": "power_source", "data_type": "int"}, + (3, 7): {"name": "power_source_v", "data_type": "int"}, + (3, 8): {"name": "power_source_i", "data_type": "int"}, + (3, 9): {"name": "battery_level", "data_type": "int"}, + (3, 10): {"name": "memory_free", "data_type": "int"}, + (3, 11): {"name": "error_code", "data_type": "int"}, + (3, 13): {"name": "current_time", "data_type": "time"}, + (3, 14): {"name": "utc_offset", "data_type": "string"}, + (3, 15): {"name": "timezone", "data_type": "string"}, + (3, 16): {"name": "binding_mode", "data_type": "string"}, + (3, 17): {"name": "device_type", "data_type": "string"}, + (3, 18): {"name": "hardware_version", "data_type": "string"}, + (3, 19): {"name": "software_version", "data_type": "string"}, + (3, 20): {"name": "battery_status", "data_type": "int"}, + (3, 21): {"name": "memory_total", "data_type": "int"}, +} diff --git a/server/django/sensordata/models.py b/server/django/sensordata/models.py index 02068e0c..70b166f8 100644 --- a/server/django/sensordata/models.py +++ b/server/django/sensordata/models.py @@ -1,4 +1,3 @@ -from django.contrib import admin from django.db import models class Device(models.Model): @@ -16,6 +15,9 @@ class ResourceType(models.Model): class Meta: unique_together = ('object_id', 'resource_id') + def __str__(self): + return f"{self.object_id}/{self.resource_id} - {self.name}" + class Resource(models.Model): """Stores individual resource data, such as sensor readings, from a device.""" device = models.ForeignKey(Device, on_delete=models.PROTECT) @@ -29,6 +31,9 @@ class Resource(models.Model): class Meta: unique_together = ('device', 'resource_type', 'timestamp') + def __str__(self): + return f"{self.device} - {self.resource_type} - {self.timestamp}" + class Event(models.Model): """ Represents a significant event in the system that is associated with a @@ -65,38 +70,3 @@ class Firmware(models.Model): file_name = models.CharField(max_length=255) download_url = models.URLField() created_at = models.DateTimeField(auto_now_add=True) - -#class Endpoint(models.Model): -# endpoint = models.CharField(max_length=100, default='', unique=True) -# manufacturer = models.CharField(max_length=100, default='') -# model_number = models.CharField(max_length=100, default='') -# serial_number = models.CharField(max_length=100, default='') -# firmware_version = models.CharField(max_length=100, default='') -# reboot = models.IntegerField(default=0) -# factory_reset = models.IntegerField(default=0) -# battery_level = models.IntegerField(default=0) -# last_updated = models.DateTimeField(auto_now=True) -# -#class SensorData(models.Model): -# endpoint = models.CharField(max_length=100, default='') -# time = models.DateTimeField(auto_now_add=True) -# temperature = models.FloatField() -# -#@admin.register(SensorData) -#class SensorDataAdmin(admin.ModelAdmin): -# list_display = [ -# 'time', -# 'endpoint', -# 'temperature', -# ] -# -#@admin.register(Endpoint) -#class EndpointAdmin(admin.ModelAdmin): -# list_display = [ -# 'endpoint', -# 'serial_number', -# 'model_number', -# 'firmware_version', -# 'reboot', -# 'last_updated', -# ] diff --git a/server/django/sensordata/serializers.py b/server/django/sensordata/serializers.py index 68dac342..5d7acc41 100644 --- a/server/django/sensordata/serializers.py +++ b/server/django/sensordata/serializers.py @@ -1,139 +1,137 @@ from rest_framework import serializers -from .models import SensorData, Endpoint +from .models import Device, ResourceType, Resource +from django.utils import timezone +from .lwm2m_mappings import LWM2M_RESOURCE_MAP import binascii import struct import logging -import json logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -class GenericLWM2MSerializer(serializers.Serializer): - """ - A serializer for generic handling and persistence of sensor data received from an - LwM2M server. Primarily designed to iterate over the varying data representation - sent by different types of sensors and store them in the SensorData model. - - Check test_restapi.py for example payloads. - """ - - ep = serializers.CharField() - res = serializers.CharField() - val = serializers.JSONField() - - # Define the path to property and type mapping. Others will be ignored. - path_mapping = { - '3': { # Device Object ID - '0': {'field': 'manufacturer', 'type': 'string'}, - '1': {'field': 'model_number', 'type': 'string'}, - '2': {'field': 'serial_number', 'type': 'string'}, - '3': {'field': 'firmware_version', 'type': 'string'}, - '4': {'field': 'reboot', 'type': 'int'}, - '5': {'field': 'factory_reset', 'type': 'int'}, - '9': {'field': 'battery_level', 'type': 'int'}, - '10': {'field': 'memory_free', 'type': 'int'}, - }, - '3303': { # Temperature Object ID - '5700': {'field': 'temperature', 'type': 'float'}, - }, - } - - def deserialize_sensor_data(self, json_string): - logger.debug("deserializer: deserialize_sensor_data\n" + json_string) - - data = json.loads(json_string) - sensor_data = {} - - sensor_data['endpoint'] = data['ep'] - - resource_path = data['res'] - - if '/' in resource_path: - paths = resource_path.strip('/').split('/') - object_id = paths[0] - # Handle possible index, e.g., /3303/0/5700 -> Object ID: 3303, Index: 0, Resource ID: 5700 - resource_id = paths[-1] + +def decode_opaque_data(hex_value, data_type): + if hex_value == '': + return None + if data_type == 'float': + decoded = binascii.unhexlify(hex_value) + return struct.unpack('>d', decoded)[0] + elif data_type == 'long': + decoded = binascii.unhexlify(hex_value) + return struct.unpack('>l', decoded)[0] + elif data_type == 'int': + decoded = binascii.unhexlify(hex_value) + return struct.unpack('>h', decoded)[0] + elif data_type == 'string': + return hex_value + else: + logger.error(f"Unsupported data type: {data_type}, data: {hex_value}") + raise ValueError(f"Unsupported data type: {data_type}") + + +class ResourceDataSerializer(serializers.Serializer): + kind = serializers.CharField(max_length=50) + id = serializers.IntegerField() + type = serializers.CharField(max_length=50) + value = serializers.CharField(max_length=255, required=False, allow_blank=True) + values = serializers.DictField(child=serializers.CharField(), required=False, allow_null=True) + + +class InstanceSerializer(serializers.Serializer): + kind = serializers.CharField(max_length=50) + id = serializers.IntegerField() + resources = ResourceDataSerializer(many=True) + + +class ValueSerializer(serializers.Serializer): + instances = InstanceSerializer(many=True, required=False) + kind = serializers.CharField(max_length=50) + id = serializers.IntegerField() + type = serializers.CharField(max_length=50, required=False) + value = serializers.CharField(max_length=255, required=False) + + +class LwM2MSerializer(serializers.Serializer): + ep = serializers.CharField(max_length=255) + res = serializers.CharField(max_length=255) + val = ValueSerializer() + + def create(self, validated_data): + ep = validated_data['ep'] + res = validated_data['res'] + val = validated_data['val'] + + # ep maps to Device.device_id + device, _ = Device.objects.get_or_create(device_id=ep, defaults={'name': ep}) + + # Check if value is an object with instances + if val['kind'] == 'obj' and 'instances' in val: + for instance in val['instances']: + for resource in instance['resources']: + self.handle_resource(device, res, resource) else: - object_id = resource_path - - object_path_mapping = self.path_mapping.get(object_id, {}) - - if 'val' in data and isinstance(data['val'], dict): - data_resources = data['val'].get('instances', [data['val']]) - #logger.debug(f"deserializer: Data resources: {data_resources}") - - for instance in data_resources: - resources = instance.get('resources', []) if instance.get('resources') else [instance] - for resource in resources: - resource_id = str(resource['id']) - if resource_id in object_path_mapping: - field_info = object_path_mapping[resource_id] - field_name = field_info['field'] - # If the resource kind is 'singleResource' or the value is directly available - if resource.get('kind') == 'singleResource' or 'value' in resource: - if field_info['type'] == 'float': - sensor_data[field_name] = self.decode_value(resource['value'], - field_info['type']) - elif field_info['type'] == 'int': - sensor_data[field_name] = (int)(resource['value']) - else: - sensor_data[field_name] = resource['value'] - - return sensor_data - - - def decode_value(self, hex_value, data_type): - if data_type == 'float': - # Assuming 8 bytes for double precision, big-endian byte order - decoded = binascii.unhexlify(hex_value) - return struct.unpack('>d', decoded)[0] - elif data_type == 'long': - # Assuming 4 bytes for long int, big-endian byte order - decoded = binascii.unhexlify(hex_value) - return struct.unpack('>l', decoded)[0] - elif data_type == 'string': - # Assuming hex-encoded ASCII string - return binascii.unhexlify(hex_value).decode('ascii') + # Single resource handling + self.handle_resource(device, res, val) + + return device + + def handle_resource(self, device, res, resource): + # Parse resource path to get object_id and resource_id + resource_path_parts = res.strip('/').split('/') + if len(resource_path_parts) == 3: + object_id = int(resource_path_parts[0]) + resource_id = int(resource_path_parts[2]) + elif resource_path_parts[0].isdigit(): + object_id = int(resource_path_parts[0]) + resource_id = resource['id'] else: - raise ValueError(f"Unsupported data type: {data_type}") - - - def create(self, data): - data_deser = self.deserialize_sensor_data(json.dumps(data, indent=4)) - - sensor_data = {} - endpoint_data = {} - - # Decide based on the individual field name whether it is a SensorData or - # Endpoint object and create it. Generic method to handle both types of objects. - sensor_data_field_names = [field.name for field in SensorData._meta.get_fields()] - endpoint_data_field_names = [field.name for field in Endpoint._meta.get_fields()] - - for field_name, field_value in data_deser.items(): - if field_name in sensor_data_field_names: - sensor_data[field_name] = field_value - if field_name in endpoint_data_field_names: - endpoint_data[field_name] = field_value - - # Only add if more than the endpoint could be mapped (actual sensor data) - if len(endpoint_data) > 1: - logger.debug(f"deserializer: Endpoint data: {json.dumps(endpoint_data, indent=4)}") - - unique_field = 'endpoint' - unique_field_value = endpoint_data.pop(unique_field) - - ep_ret = Endpoint.objects.update_or_create( - **{unique_field: unique_field_value}, - defaults=endpoint_data - ) + logger.error(f"Invalid resource path: {res}") + raise serializers.ValidationError("Invalid resource path") + + + # Fetch resource information from static mapping + resource_info = LWM2M_RESOURCE_MAP.get((object_id, resource_id)) + if not resource_info: + raise serializers.ValidationError(f"Resource type {object_id}/{resource_id} not found") + + resource_type, _ = ResourceType.objects.get_or_create( + object_id=object_id, + resource_id=resource_id, + defaults={'name': resource_info['name'], 'data_type': resource_info['data_type']} + ) + + data_type = resource_info['data_type'] + + # Some LwM2M Resources have a OPAQUE type, which needs decoding + if resource['kind'] == 'singleResource': + if resource['type'] == 'OPAQUE': + decoded_value = decode_opaque_data(resource['value'], data_type) + else: + decoded_value = resource['value'] + elif resource['kind'] == 'multiResource': + logging.error(f"multiResource currently not supported, skipping...") + decoded_value = None else: - ep_ret = None - - # Only add if more than the endpoint could be mapped (actual sensor data) - if len(sensor_data) > 1: - sd_ret = SensorData.objects.create(**sensor_data) - logger.debug(f"deserializer: Sensor data: {json.dumps(sensor_data, indent=4)}") + #TODO: Handle multiResource (Maybe json field in DB) + logger.error(f"Unsupported resource kind: {resource['kind']}") + raise serializers.ValidationError(f"Unsupported resource kind: {resource['kind']}") + + # Create the Resource instance based on value type + resource_data = { + 'device': device, + 'resource_type': resource_type, + 'timestamp': timezone.now() + } + + # Assign the decoded value to the appropriate field + if data_type == 'float': + resource_data['float_value'] = decoded_value + elif data_type == 'int': + resource_data['int_value'] = decoded_value + elif data_type == 'string': + resource_data['str_value'] = decoded_value + elif data_type == 'time': + resource_data['str_value'] = decoded_value else: - sd_ret = None + logger.error(f"Unsupported data type: {data_type}") + raise serializers.ValidationError(f"Unsupported data type") - return (sd_ret, ep_ret) + Resource.objects.create(**resource_data) diff --git a/server/django/sensordata/templates/sensordata/resource_list.html b/server/django/sensordata/templates/sensordata/resource_list.html new file mode 100644 index 00000000..49ff1d14 --- /dev/null +++ b/server/django/sensordata/templates/sensordata/resource_list.html @@ -0,0 +1,30 @@ +{% block content %} + + + + + + + + + + {% for tt in resource_list %} + + + + + + {% endfor %} + +
TimeEndpointValue
{{ tt.timestamp }}{{ tt.device.device_id }} + {% if tt.int_value is not None %} + {{ tt.int_value }} + {% elif tt.float_value is not None %} + {{ tt.float_value }} + {% elif tt.str_value is not None %} + {{ tt.str_value }} + {% elif tt.bool_value is not None %} + {{ tt.bool_value }} + {% endif %} +
+{% endblock %} diff --git a/server/django/sensordata/templates/sensordata/timetemperature_list.html b/server/django/sensordata/templates/sensordata/timetemperature_list.html deleted file mode 100644 index 2ecc0c9c..00000000 --- a/server/django/sensordata/templates/sensordata/timetemperature_list.html +++ /dev/null @@ -1,20 +0,0 @@ -{% block content %} - - - - - - - - - - {% for tt in timetemperature_list %} - - - - - - {% endfor %} - -
TimeEndpointTemperature
{{ tt.time }}{{ tt.ep}}{{ tt.temperature }}
-{% endblock %} diff --git a/server/django/sensordata/tests/test_restapi_multi.py b/server/django/sensordata/tests/test_restapi_multi.py index c2fae646..fc4fd748 100644 --- a/server/django/sensordata/tests/test_restapi_multi.py +++ b/server/django/sensordata/tests/test_restapi_multi.py @@ -1,10 +1,13 @@ from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase -from sensordata.models import SensorData -from sensordata.models import Endpoint import binascii import struct +from sensordata.models import ( + Device, + ResourceType, + Resource, +) TEST_PAYLOAD = [ { @@ -164,19 +167,7 @@ def test_create_sensor_data_from_json_payloads(self): """ Ensure we can create new sensor data objects using given JSON payloads. """ - url = reverse('add_sensor_data') + self.url = reverse('add_sensor_data') - for pl in TEST_PAYLOAD: - response = self.client.post(url, pl, format='json') - - self.assertEqual(response.status_code, status.HTTP_201_CREATED) - #self.assertEqual(Endpoint.objects.count(), 1) - - self.assertEqual(Endpoint.objects.get().endpoint, pl['ep']) - self.assertEqual(Endpoint.objects.get().manufacturer, pl['val']['instances'][0]['resources'][0]['value']) - self.assertEqual(Endpoint.objects.get().model_number, pl['val']['instances'][0]['resources'][1]['value']) - self.assertEqual(Endpoint.objects.get().serial_number, pl['val']['instances'][0]['resources'][2]['value']) - self.assertEqual(Endpoint.objects.get().firmware_version, pl['val']['instances'][0]['resources'][3]['value']) - self.assertEqual(Endpoint.objects.get().battery_level, (int)(pl['val']['instances'][0]['resources'][7]['value'])) - - Endpoint.objects.all().delete() + response = self.client.post(self.url, TEST_PAYLOAD, format='json') + self.assertEqual(response.status_code, status.HTTP_201_CREATED) diff --git a/server/django/sensordata/tests/test_restapi_single.py b/server/django/sensordata/tests/test_restapi_single.py index e942d363..bd94a258 100644 --- a/server/django/sensordata/tests/test_restapi_single.py +++ b/server/django/sensordata/tests/test_restapi_single.py @@ -1,9 +1,13 @@ +import binascii +import struct from django.urls import reverse from rest_framework import status from rest_framework.test import APITestCase -from sensordata.models import SensorData -import binascii -import struct +from sensordata.models import ( + Device, + ResourceType, + Resource, +) TEST_PAYLOAD = [ @@ -21,28 +25,39 @@ class SensorDataTests(APITestCase): - def convert_hex_to_double(self, hex_value): - try: - return struct.unpack('>d', binascii.unhexlify(hex_value))[0] - except binascii.Error as e: - raise ValueError(f"Hexadecimal to binary conversion failed: {e}") - except struct.error as e: - raise ValueError(f"Binary to double unpacking failed: {e}") - def test_create_sensor_data_from_json_payloads(self): """ Ensure we can create new sensor data objects using given JSON payloads. """ - url = reverse('add_sensor_data') + self.url = reverse('add_sensor_data') - for pl in TEST_PAYLOAD: - response = self.client.post(url, pl, format='json') - - self.assertEqual(response.status_code, status.HTTP_201_CREATED) + response = self.client.post(self.url, TEST_PAYLOAD, format='json') + self.assertEqual(response.status_code, status.HTTP_201_CREATED) - self.assertEqual(SensorData.objects.count(), 1) - self.assertEqual(SensorData.objects.get().endpoint, pl['ep']) - expected_temperature = self.convert_hex_to_double(pl['val']['value']) - self.assertEqual(SensorData.objects.get().temperature, expected_temperature) - - SensorData.objects.all().delete() + for pl in TEST_PAYLOAD: + ep = pl['ep'] + res = pl['res'] + val = pl['val'] + + # Verify the Device was created + device = Device.objects.get(device_id=ep) + self.assertIsNotNone(device) + self.assertEqual(device.name, ep) + + # Parse resource path to get object_id and resource_id + resource_path_parts = res.strip('/').split('/') + object_id = int(resource_path_parts[0]) + resource_id = int(resource_path_parts[2]) + + # Verify the ResourceType was created + resource_type = ResourceType.objects.get(object_id=object_id, resource_id=resource_id) + self.assertIsNotNone(resource_type) + + # Verify the Resource was created + resource = Resource.objects.get(device=device, resource_type=resource_type) + self.assertIsNotNone(resource) + + # Check values based on the type + self.assertEqual(resource.resource_type.data_type, 'float') + decoded_value = struct.unpack('>d', binascii.unhexlify(val['value']))[0] + self.assertEqual(resource.float_value, decoded_value) diff --git a/server/django/sensordata/views.py b/server/django/sensordata/views.py index 38700140..e5d7af3b 100644 --- a/server/django/sensordata/views.py +++ b/server/django/sensordata/views.py @@ -1,26 +1,20 @@ from rest_framework.views import APIView from rest_framework.response import Response from rest_framework import status -from .serializers import GenericLWM2MSerializer -from .models import SensorData -from django.views.generic import ListView +from .serializers import LwM2MSerializer import logging logger = logging.getLogger(__name__) class CreateSensorDataView(APIView): - def post(self, request, format=None): - serializer = GenericLWM2MSerializer(data=request.data) + def post(self, request): + + serializer = LwM2MSerializer(data=request.data, many=False) if serializer.is_valid(): serializer.save() return Response(serializer.validated_data, status=status.HTTP_201_CREATED) else: + print(serializer.errors) logger.error(f"Errors: {serializer.errors}") return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - -class SensorDataListView(ListView): - model = SensorData - template_name = 'timetemperature_list.html' - context_object_name = 'timetemperature_list' - queryset = SensorData.objects.all() diff --git a/server/django/server/settings.py b/server/django/server/settings.py index 697d30a9..98addcd0 100644 --- a/server/django/server/settings.py +++ b/server/django/server/settings.py @@ -24,7 +24,7 @@ SECRET_KEY = "django-insecure-ttm_sr56l7mv#4smgm*+tffm*$q%!qqp@#q7*_*y38^^#9%@7*" # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = True CSRF_TRUSTED_ORIGINS = ['https://controlscope.de'] diff --git a/server/django/server/urls.py b/server/django/server/urls.py index c1da8f14..eda1fa41 100644 --- a/server/django/server/urls.py +++ b/server/django/server/urls.py @@ -1,9 +1,7 @@ from django.contrib import admin from django.urls import include, path -from sensordata import views as sensordata_views urlpatterns = [ path('admin/', admin.site.urls), path('api/', include('sensordata.urls')), - path('sensordata/list/', sensordata_views.SensorDataListView.as_view(), name='sensordata_list'), ]