Merge pull request #45 from Informatievlaanderen/feat/crowdscan

Feat/crowdscan

backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/config/EventStreamConfig.java

@@ -13,6 +13,7 @@ public class EventStreamConfig {
     private String versionOfPath;
     private String timestampPath;
     private String timezoneId;
+    private String geoLocationPath;
     private Map<String, String> propertyPredicates;
 
     public String getMemberType() {
@@ -60,4 +61,12 @@ public String getVersionOfPath() {
     public void setVersionOfPath(String versionOfPath) {
         this.versionOfPath = versionOfPath;
     }
+
+    public String getGeoLocationPath() {
+        return geoLocationPath;
+    }
+
+    public void setGeoLocationPath(String geoLocationPath) {
+        this.geoLocationPath = geoLocationPath;
+    }
 }

backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyExtractor.java

@@ -0,0 +1,12 @@
+package be.informatievlaanderen.vsds.demonstrator.member.application.services;
+
+import org.apache.jena.rdf.model.Model;
+import org.apache.jena.rdf.model.RDFNode;
+
+import java.util.List;
+
+public interface PropertyExtractor {
+
+	List<RDFNode> getProperties(Model model);
+
+}

backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyPathExtractor.java

@@ -0,0 +1,48 @@
+package be.informatievlaanderen.vsds.demonstrator.member.application.services;
+
+import org.apache.jena.query.*;
+import org.apache.jena.rdf.model.Model;
+import org.apache.jena.rdf.model.RDFNode;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class PropertyPathExtractor implements PropertyExtractor {
+
+	private static final String OBJECT_VAR_NAME = "object";
+	private static final String IRI_OPENING_SYMBOL = "<";
+	private final String queryString;
+
+	private PropertyPathExtractor(String propertyPath) {
+		queryString = "SELECT * where { ?subject %s ?object }".formatted(propertyPath);
+	}
+
+	/**
+	 * This factory method was provided for backwards compatibility.
+	 * In the past we supported properties to be provided as strings in a non IRI
+	 * format.
+	 * When a property is provided as a plain string, we wrap it to an IRI.
+	 * NOTE: Does not work with property paths -> ex:foo/ex:bar will not get auto
+	 * wrapping.
+	 */
+	public static PropertyPathExtractor from(String propertyPath) {
+		return propertyPath.startsWith(IRI_OPENING_SYMBOL)
+				? new PropertyPathExtractor(propertyPath)
+				: new PropertyPathExtractor("<%s>".formatted(propertyPath));
+	}
+
+	@Override
+	public List<RDFNode> getProperties(Model model) {
+		final Query query = QueryFactory.create(queryString);
+		try (QueryExecution queryExecution = QueryExecutionFactory.create(query, model)) {
+			ResultSet resultSet = queryExecution.execSelect();
+
+			List<RDFNode> results = new ArrayList<>();
+			while (resultSet.hasNext()) {
+				results.add(resultSet.next().get(OBJECT_VAR_NAME));
+			}
+			return results;
+		}
+	}
+
+}

backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/valueobjects/IngestedMemberDto.java

@@ -2,6 +2,7 @@
 
 import be.informatievlaanderen.vsds.demonstrator.member.application.config.EventStreamConfig;
 import be.informatievlaanderen.vsds.demonstrator.member.application.exceptions.NoGeometryProvidedException;
+import be.informatievlaanderen.vsds.demonstrator.member.application.services.PropertyPathExtractor;
 import be.informatievlaanderen.vsds.demonstrator.member.domain.member.entities.Member;
 import org.apache.jena.geosparql.implementation.GeometryWrapper;
 import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI;
@@ -15,6 +16,7 @@
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -42,7 +44,7 @@ public Model getModel() {
     }
 
     public Member getMember(EventStreamConfig eventStreamConfig) throws FactoryException, TransformException {
-        Geometry geometry = getGeometry();
+        Geometry geometry = getGeometry(eventStreamConfig);
         String memberId = getMemberId(eventStreamConfig);
         String isVersionOf = getIsVersionOf(eventStreamConfig);
         String timestampString = getTimestamp(eventStreamConfig);
@@ -64,8 +66,14 @@ private LocalDateTime getTimestamp(ZoneId timezoneId, String timestampString) {
         return timestamp;
     }
 
-    private Geometry getGeometry() throws FactoryException, TransformException {
-        List<RDFNode> wktNodes = model.listObjectsOfProperty(model.createProperty("http://www.opengis.net/ont/geosparql#asWKT")).toList();
+    private Geometry getGeometry(EventStreamConfig config) throws FactoryException, TransformException {
+        List<RDFNode> wktNodes = new ArrayList<>();
+        if(config.getGeoLocationPath() != null) {
+            wktNodes.addAll(PropertyPathExtractor.from(config.getGeoLocationPath()).getProperties(model));
+        }
+        else {
+            wktNodes.addAll(model.listObjectsOfProperty(model.createProperty("http://www.opengis.net/ont/geosparql#asWKT")).toList());
+        }
         if (wktNodes.isEmpty()) {
             throw new NoGeometryProvidedException();
         } else {

backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/triple/infra/TripleRepositoryRDF4JImpl.java

@@ -50,6 +50,20 @@ protected void initRepo(RepositoryManager repositoryManager) {
         } catch (Exception e) {
             log.error("Could not create repository. Reason: {}", e.getMessage());
         }
+        // possibly temp solution to separate locations from other members
+        try {
+            String locationsRepoId = "locations";
+            if(!repositoryManager.hasRepositoryConfig(locationsRepoId)) {
+                String indeces = "spoc,cspo";
+                NativeStoreConfig storeConfig = new NativeStoreConfig(indeces);
+                RepositoryImplConfig repositoryImplConfig = new SailRepositoryConfig(storeConfig);
+                RepositoryConfig config = new RepositoryConfig(locationsRepoId, repositoryImplConfig);
+                repositoryManager.addRepositoryConfig(config);
+                log.info("Created repository with id: {}", locationsRepoId);
+            }
+        } catch (Exception e) {
+            log.error("Could not create repository. Reason: {}", e.getMessage());
+        }
     }
 
     @Override

backend/src/main/resources/crowdscan/crowdscan-locaties.nq

@@ -0,0 +1,2 @@
+<https://crowdscan.be/id/zone/${environment}/${zone}> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://crowdscan.be/ns/Zone> .
+<https://crowdscan.be/id/zone/${environment}/${zone}> <http://www.opengis.net/ont/geosparql#asWKT> "${geometry}"^^<http://www.opengis.net/ont/geosparql#wktLiteral> .

demonstrator.env

@@ -22,6 +22,11 @@ LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_FULLNAME=http://schema.org/name
 #LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_CAPACITY=http://schema.mobivoc.org/#totalCapacity
 LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_AVAILABLE=http://schema.mobivoc.org/#currentValue
 #LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_USED=https://w3id.org/gbfs#bikes_in_use
+LDES_STREAMS_CROWDSCAN_MEMBERTYPE=http://def.isotc211.org/iso19156/2011/Observation#OM_Observation
+LDES_STREAMS_CROWDSCAN_TIMESTAMPPATH=http://www.w3.org/ns/prov#generatedAtTime
+LDES_STREAMS_CROWDSCAN_VERSIONOFPATH=http://purl.org/dc/terms/isVersionOf
+LDES_STREAMS_CROWDSCAN_GEOLOCATIONPATH=<http://def.isotc211.org/iso19156/2011/SamplingSurface#SF_SamplingSurface.shape>/<http://www.opengis.net/ont/geosparql#asWKT>
+LDES_STREAMS_CROWDSCAN_PROPERTYPREDICATES_DENSITY=http://schema.org/value
 GRAPHDB_URL=http://rdf4j-server:8080/rdf4j-server/repositories/
 GRAPHDB_REPOSITORYID=test
 SERVER_PORT=8080

docker-compose.yml

@@ -1,14 +1,15 @@
 services:
 
   data-provider:
-    image: ghcr.io/informatievlaanderen/ldi-orchestrator:20230927142432
+    image: ghcr.io/informatievlaanderen/ldi-orchestrator:latest
     container_name: demonstrator-data-provider
     ports:
       - 8082:8080
     volumes:
       - ./gipod.config.yml:/ldio/application.yml:ro
       - ./rml:/ldio/rml:ro
       - ./sparql:/ldio/sparql:ro
+      - ./jsonld:/ldio/jsonld:ro
     depends_on:
       rdf4j-server:
         condition: service_healthy
@@ -64,6 +65,7 @@ services:
     networks:
       - demonstrator
 
+
 networks:
   demonstrator:
     driver: bridge

frontend/src/components/map/composables/usePopup.js

@@ -29,6 +29,10 @@ export function usePopup(collection, properties) {
 <img class="popup-grid-icon popup-bluebike-icon" src="${bikeIcon}">
 <span class="popup-title body-small-regular">${properties.fullname}</span>
 <span>${properties.available}</span><b>beschikbare ${getBikeString(properties.available)}</b>
+</div>`
+        case "crowdscan":
+            return `<div class="popup-grid body body-xxsmall-regular">
+<span>personen dichtheid: ${properties.density}</span>
 </div>`
 
     }

frontend/streams.json

@@ -14,6 +14,11 @@
       "id": "bluebikes",
       "fullName": "Blue Bikes",
       "color": "#05c"
+    },
+    {
+      "id": "crowdscan",
+      "fullName": "CrowdScan",
+      "color": "#A4E2EB"
     }
   ]
 }

gipod.config.yml

@@ -70,4 +70,47 @@ orchestrator:
         - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut
           config:
             endpoint: http://host.docker.internal:8084/api/bluebikes/members
-            content-type: application/n-quads
+            content-type: application/n-quads
+    - name: crowdscan-observations-pipeline
+      input:
+        name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient
+        config:
+          url: https://azure.crowdscan.be/ldes-scewc/observations
+      transformers:
+        - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer
+          config:
+            query: ./sparql/crowdscan.add-query.rq
+            infer: true
+        - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpEnricher
+          config:
+            url-property-path: https://crowdscan.be/ns/HttpRequest.url
+            body-property-path: https://crowdscan.be/ns/HttpRequest.body
+            http-method-property-path: https://crowdscan.be/ns/HttpRequest.method
+            header-property-path: https://crowdscan.be/ns/HttpRequest.header
+            adapter:
+              name: be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter
+        - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer
+          config:
+            query: ./sparql/crowdscan.remove-query.rq
+            infer: false
+      outputs:
+        - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser
+          config:
+            sparql-host: http://rdf4j-server:8080/rdf4j-server
+            repository-id: test
+            named-graph: http://crowdscan
+        - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut
+          config:
+            endpoint: http://host.docker.internal:8084/api/crowdscan/members
+            content-type: application/n-quads
+    - name: crowdscan-locations-pipeline
+      input:
+        name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient
+        config:
+          url: https://azure.crowdscan.be/ldes-scewc/zones
+      outputs:
+        - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser
+          config:
+            sparql-host: http://rdf4j-server:8080/rdf4j-server
+            repository-id: locations
+            named-graph: http://crowdscan-locations

jsonld/count.jsonld

@@ -0,0 +1,14 @@
+{
+    "@context": {
+        "@vocab": "https://www.crowdscan.be/ns/count#",
+        "time": {
+          "@type": "http://www.w3.org/2001/XMLSchema#DateTime"
+        },
+        "timedelta": {
+          "@type": "http://www.w3.org/2001/XMLSchema#int"
+        },
+        "regions": {
+            "@container": "@list"
+        }
+      }
+}

jsonld/map.jsonld

@@ -0,0 +1,11 @@
+{
+    "@context": [
+      {
+        "@vocab": "https://www.crowdscan.be/ns/map#",
+        "last_updated": {
+          "@type": "http://www.w3.org/2001/XMLSchema#DateTime"
+        }
+      },
+      "http://geojson.org/geojson-ld/geojson-context.jsonld"
+    ]
+}

sparql/count.to-observation.rq

@@ -0,0 +1,50 @@
+PREFIX rdf:          <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX xsd:          <http://www.w3.org/2001/XMLSchema#>
+PREFIX schema:       <http://schema.org/>
+PREFIX time:         <http://www.w3.org/2006/time#>
+PREFIX :             <https://www.crowdscan.be/ns/count#>
+
+CONSTRUCT {
+
+  GRAPH ?observation { 
+    ?observation a :Observation ;
+      :time ?time ;
+      :value ?value ;
+      :environment ?environment ;
+      :timedelta ?delta ;
+      :region ?zone .
+  }
+
+} WHERE {
+
+  ?header :environment ?environment .
+  ?header :time ?time .
+  ?header :timedelta ?timedelta .
+  bind(strdt(concat("-PT", str(?timedelta), "M"), xsd:duration) as ?delta) .
+
+  ?payload :payload ?regions .
+  {
+    SELECT (count(?first) as ?zoneCount) WHERE { ?list rdf:first ?first . }
+  }
+
+  OPTIONAL { filter(?zoneCount = 1)
+    ?list rdf:first ?value .
+    bind(0 as ?zone) .
+  }
+
+  OPTIONAL { filter(?zoneCount > 1) 
+    {
+      SELECT ?value (count(?mid) as ?zone) 
+      WHERE 
+      {
+        ?regions rdf:rest* ?mid . 
+        ?mid rdf:rest ?node .
+        ?node rdf:first ?value .
+      }
+      GROUP BY ?node ?value
+    }
+  }
+
+  bind(uri(concat("https://crowdscan.be/id/observation/", ?environment, "/", str(?zone))) as ?observation) .
+
+}

sparql/crowdscan.add-query.rq

@@ -0,0 +1,30 @@
+PREFIX rdf:          <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX xsd:          <http://www.w3.org/2001/XMLSchema#>
+
+PREFIX :             <https://crowdscan.be/ns/>
+PREFIX http:         <https://crowdscan.be/ns/HttpRequest.>
+PREFIX obs:          <http://def.isotc211.org/iso19156/2011/Observation#>
+PREFIX prov:         <http://www.w3.org/ns/prov#>
+
+CONSTRUCT {
+  ?request a :HttpRequest .
+  ?request http:url ?url .
+  ?request http:body ?body .
+  ?request http:method "POST" .
+  ?request http:header ?type .
+  ?request http:header ?accept .
+} WHERE {
+  ?obs rdf:type obs:OM_Observation .
+  ?obs obs:OM_Observation.featureOfInterest ?loc .
+  ?obs prov:generatedAtTime ?obstime .
+
+
+  bind(concat(replace(replace(str(?loc), "www.crowdscan.be", "crowdscan.be"),"box4","beacon"), "/") as ?locpart) .
+
+  # set HTTP request parameters
+  bind(bnode() as ?request) .
+  bind("http://rdf4j-server:8080/rdf4j-server/repositories/locations" as ?url) .
+  bind(concat("describe ?zone From <http://crowdscan-locations> Where { { select (max(?t) as ?max) Where { ?z <http://www.w3.org/ns/prov#generatedAtTime> ?time . Filter(?t < \"", str(?obstime), "\") . bind(str(?time) as ?t) . } } bind(IRI(concat(\"", ?locpart, "\", ?max)) as ?zone) . }") as ?body) .
+  bind("Content-Type: application/sparql-query" as ?type) .
+  bind("Accept: application/n-quads" as ?accept) .
+}

sparql/crowdscan.remove-query.rq

@@ -0,0 +1,15 @@
+PREFIX :             <https://crowdscan.be/ns/>
+
+
+CONSTRUCT {
+    ?s ?p ?o .
+}
+WHERE {
+    ?s ?p ?o .
+
+    {SELECT ?request
+    WHERE {
+        ?request a :HttpRequest .
+    }}
+    FILTER(?s != ?request)
+}