diff --git a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/config/EventStreamConfig.java b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/config/EventStreamConfig.java index 06590a1..523c234 100644 --- a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/config/EventStreamConfig.java +++ b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/config/EventStreamConfig.java @@ -13,6 +13,7 @@ public class EventStreamConfig { private String versionOfPath; private String timestampPath; private String timezoneId; + private String geoLocationPath; private Map propertyPredicates; public String getMemberType() { @@ -60,4 +61,12 @@ public String getVersionOfPath() { public void setVersionOfPath(String versionOfPath) { this.versionOfPath = versionOfPath; } + + public String getGeoLocationPath() { + return geoLocationPath; + } + + public void setGeoLocationPath(String geoLocationPath) { + this.geoLocationPath = geoLocationPath; + } } \ No newline at end of file diff --git a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyExtractor.java b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyExtractor.java new file mode 100644 index 0000000..4ca583b --- /dev/null +++ b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyExtractor.java @@ -0,0 +1,12 @@ +package be.informatievlaanderen.vsds.demonstrator.member.application.services; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.RDFNode; + +import java.util.List; + +public interface PropertyExtractor { + + List getProperties(Model model); + +} diff --git a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyPathExtractor.java b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyPathExtractor.java new file mode 100644 index 0000000..e112947 --- /dev/null +++ b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyPathExtractor.java @@ -0,0 +1,48 @@ +package be.informatievlaanderen.vsds.demonstrator.member.application.services; + +import org.apache.jena.query.*; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.RDFNode; + +import java.util.ArrayList; +import java.util.List; + +public class PropertyPathExtractor implements PropertyExtractor { + + private static final String OBJECT_VAR_NAME = "object"; + private static final String IRI_OPENING_SYMBOL = "<"; + private final String queryString; + + private PropertyPathExtractor(String propertyPath) { + queryString = "SELECT * where { ?subject %s ?object }".formatted(propertyPath); + } + + /** + * This factory method was provided for backwards compatibility. + * In the past we supported properties to be provided as strings in a non IRI + * format. + * When a property is provided as a plain string, we wrap it to an IRI. + * NOTE: Does not work with property paths -> ex:foo/ex:bar will not get auto + * wrapping. + */ + public static PropertyPathExtractor from(String propertyPath) { + return propertyPath.startsWith(IRI_OPENING_SYMBOL) + ? new PropertyPathExtractor(propertyPath) + : new PropertyPathExtractor("<%s>".formatted(propertyPath)); + } + + @Override + public List getProperties(Model model) { + final Query query = QueryFactory.create(queryString); + try (QueryExecution queryExecution = QueryExecutionFactory.create(query, model)) { + ResultSet resultSet = queryExecution.execSelect(); + + List results = new ArrayList<>(); + while (resultSet.hasNext()) { + results.add(resultSet.next().get(OBJECT_VAR_NAME)); + } + return results; + } + } + +} diff --git a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/valueobjects/IngestedMemberDto.java b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/valueobjects/IngestedMemberDto.java index 9412687..b5a2e5a 100644 --- a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/valueobjects/IngestedMemberDto.java +++ b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/valueobjects/IngestedMemberDto.java @@ -2,6 +2,7 @@ import be.informatievlaanderen.vsds.demonstrator.member.application.config.EventStreamConfig; import be.informatievlaanderen.vsds.demonstrator.member.application.exceptions.NoGeometryProvidedException; +import be.informatievlaanderen.vsds.demonstrator.member.application.services.PropertyPathExtractor; import be.informatievlaanderen.vsds.demonstrator.member.domain.member.entities.Member; import org.apache.jena.geosparql.implementation.GeometryWrapper; import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; @@ -15,6 +16,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -42,7 +44,7 @@ public Model getModel() { } public Member getMember(EventStreamConfig eventStreamConfig) throws FactoryException, TransformException { - Geometry geometry = getGeometry(); + Geometry geometry = getGeometry(eventStreamConfig); String memberId = getMemberId(eventStreamConfig); String isVersionOf = getIsVersionOf(eventStreamConfig); String timestampString = getTimestamp(eventStreamConfig); @@ -64,8 +66,14 @@ private LocalDateTime getTimestamp(ZoneId timezoneId, String timestampString) { return timestamp; } - private Geometry getGeometry() throws FactoryException, TransformException { - List wktNodes = model.listObjectsOfProperty(model.createProperty("http://www.opengis.net/ont/geosparql#asWKT")).toList(); + private Geometry getGeometry(EventStreamConfig config) throws FactoryException, TransformException { + List wktNodes = new ArrayList<>(); + if(config.getGeoLocationPath() != null) { + wktNodes.addAll(PropertyPathExtractor.from(config.getGeoLocationPath()).getProperties(model)); + } + else { + wktNodes.addAll(model.listObjectsOfProperty(model.createProperty("http://www.opengis.net/ont/geosparql#asWKT")).toList()); + } if (wktNodes.isEmpty()) { throw new NoGeometryProvidedException(); } else { diff --git a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/triple/infra/TripleRepositoryRDF4JImpl.java b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/triple/infra/TripleRepositoryRDF4JImpl.java index 492ca78..0039d9a 100644 --- a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/triple/infra/TripleRepositoryRDF4JImpl.java +++ b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/triple/infra/TripleRepositoryRDF4JImpl.java @@ -50,6 +50,20 @@ protected void initRepo(RepositoryManager repositoryManager) { } catch (Exception e) { log.error("Could not create repository. Reason: {}", e.getMessage()); } + // possibly temp solution to separate locations from other members + try { + String locationsRepoId = "locations"; + if(!repositoryManager.hasRepositoryConfig(locationsRepoId)) { + String indeces = "spoc,cspo"; + NativeStoreConfig storeConfig = new NativeStoreConfig(indeces); + RepositoryImplConfig repositoryImplConfig = new SailRepositoryConfig(storeConfig); + RepositoryConfig config = new RepositoryConfig(locationsRepoId, repositoryImplConfig); + repositoryManager.addRepositoryConfig(config); + log.info("Created repository with id: {}", locationsRepoId); + } + } catch (Exception e) { + log.error("Could not create repository. Reason: {}", e.getMessage()); + } } @Override diff --git a/backend/src/main/resources/crowdscan/crowdscan-locaties.nq b/backend/src/main/resources/crowdscan/crowdscan-locaties.nq new file mode 100644 index 0000000..4851cc4 --- /dev/null +++ b/backend/src/main/resources/crowdscan/crowdscan-locaties.nq @@ -0,0 +1,2 @@ + . + "${geometry}"^^ . diff --git a/demonstrator.env b/demonstrator.env index 0fcc799..41ce699 100644 --- a/demonstrator.env +++ b/demonstrator.env @@ -22,6 +22,11 @@ LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_FULLNAME=http://schema.org/name #LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_CAPACITY=http://schema.mobivoc.org/#totalCapacity LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_AVAILABLE=http://schema.mobivoc.org/#currentValue #LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_USED=https://w3id.org/gbfs#bikes_in_use +LDES_STREAMS_CROWDSCAN_MEMBERTYPE=http://def.isotc211.org/iso19156/2011/Observation#OM_Observation +LDES_STREAMS_CROWDSCAN_TIMESTAMPPATH=http://www.w3.org/ns/prov#generatedAtTime +LDES_STREAMS_CROWDSCAN_VERSIONOFPATH=http://purl.org/dc/terms/isVersionOf +LDES_STREAMS_CROWDSCAN_GEOLOCATIONPATH=/ +LDES_STREAMS_CROWDSCAN_PROPERTYPREDICATES_DENSITY=http://schema.org/value GRAPHDB_URL=http://rdf4j-server:8080/rdf4j-server/repositories/ GRAPHDB_REPOSITORYID=test SERVER_PORT=8080 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 32c2b12..068a710 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: data-provider: - image: ghcr.io/informatievlaanderen/ldi-orchestrator:20230927142432 + image: ghcr.io/informatievlaanderen/ldi-orchestrator:latest container_name: demonstrator-data-provider ports: - 8082:8080 @@ -9,6 +9,7 @@ services: - ./gipod.config.yml:/ldio/application.yml:ro - ./rml:/ldio/rml:ro - ./sparql:/ldio/sparql:ro + - ./jsonld:/ldio/jsonld:ro depends_on: rdf4j-server: condition: service_healthy @@ -64,6 +65,7 @@ services: networks: - demonstrator + networks: demonstrator: driver: bridge diff --git a/frontend/src/components/map/composables/usePopup.js b/frontend/src/components/map/composables/usePopup.js index 1451816..d04628b 100644 --- a/frontend/src/components/map/composables/usePopup.js +++ b/frontend/src/components/map/composables/usePopup.js @@ -29,6 +29,10 @@ export function usePopup(collection, properties) { ${properties.fullname} ${properties.available}beschikbare ${getBikeString(properties.available)} +` + case "crowdscan": + return `
+personen dichtheid: ${properties.density}
` } diff --git a/frontend/streams.json b/frontend/streams.json index 3b8c65e..0d326b0 100644 --- a/frontend/streams.json +++ b/frontend/streams.json @@ -14,6 +14,11 @@ "id": "bluebikes", "fullName": "Blue Bikes", "color": "#05c" + }, + { + "id": "crowdscan", + "fullName": "CrowdScan", + "color": "#A4E2EB" } ] } \ No newline at end of file diff --git a/gipod.config.yml b/gipod.config.yml index d3a6b10..f4cafef 100644 --- a/gipod.config.yml +++ b/gipod.config.yml @@ -70,4 +70,47 @@ orchestrator: - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut config: endpoint: http://host.docker.internal:8084/api/bluebikes/members - content-type: application/n-quads \ No newline at end of file + content-type: application/n-quads + - name: crowdscan-observations-pipeline + input: + name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient + config: + url: https://azure.crowdscan.be/ldes-scewc/observations + transformers: + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + config: + query: ./sparql/crowdscan.add-query.rq + infer: true + - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpEnricher + config: + url-property-path: https://crowdscan.be/ns/HttpRequest.url + body-property-path: https://crowdscan.be/ns/HttpRequest.body + http-method-property-path: https://crowdscan.be/ns/HttpRequest.method + header-property-path: https://crowdscan.be/ns/HttpRequest.header + adapter: + name: be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + config: + query: ./sparql/crowdscan.remove-query.rq + infer: false + outputs: + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + config: + sparql-host: http://rdf4j-server:8080/rdf4j-server + repository-id: test + named-graph: http://crowdscan + - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut + config: + endpoint: http://host.docker.internal:8084/api/crowdscan/members + content-type: application/n-quads + - name: crowdscan-locations-pipeline + input: + name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient + config: + url: https://azure.crowdscan.be/ldes-scewc/zones + outputs: + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + config: + sparql-host: http://rdf4j-server:8080/rdf4j-server + repository-id: locations + named-graph: http://crowdscan-locations \ No newline at end of file diff --git a/jsonld/count.jsonld b/jsonld/count.jsonld new file mode 100644 index 0000000..02c9b56 --- /dev/null +++ b/jsonld/count.jsonld @@ -0,0 +1,14 @@ +{ + "@context": { + "@vocab": "https://www.crowdscan.be/ns/count#", + "time": { + "@type": "http://www.w3.org/2001/XMLSchema#DateTime" + }, + "timedelta": { + "@type": "http://www.w3.org/2001/XMLSchema#int" + }, + "regions": { + "@container": "@list" + } + } +} \ No newline at end of file diff --git a/jsonld/map.jsonld b/jsonld/map.jsonld new file mode 100644 index 0000000..c59f805 --- /dev/null +++ b/jsonld/map.jsonld @@ -0,0 +1,11 @@ +{ + "@context": [ + { + "@vocab": "https://www.crowdscan.be/ns/map#", + "last_updated": { + "@type": "http://www.w3.org/2001/XMLSchema#DateTime" + } + }, + "http://geojson.org/geojson-ld/geojson-context.jsonld" + ] +} \ No newline at end of file diff --git a/sparql/count.to-observation.rq b/sparql/count.to-observation.rq new file mode 100644 index 0000000..f2823f1 --- /dev/null +++ b/sparql/count.to-observation.rq @@ -0,0 +1,50 @@ +PREFIX rdf: +PREFIX xsd: +PREFIX schema: +PREFIX time: +PREFIX : + +CONSTRUCT { + + GRAPH ?observation { + ?observation a :Observation ; + :time ?time ; + :value ?value ; + :environment ?environment ; + :timedelta ?delta ; + :region ?zone . + } + +} WHERE { + + ?header :environment ?environment . + ?header :time ?time . + ?header :timedelta ?timedelta . + bind(strdt(concat("-PT", str(?timedelta), "M"), xsd:duration) as ?delta) . + + ?payload :payload ?regions . + { + SELECT (count(?first) as ?zoneCount) WHERE { ?list rdf:first ?first . } + } + + OPTIONAL { filter(?zoneCount = 1) + ?list rdf:first ?value . + bind(0 as ?zone) . + } + + OPTIONAL { filter(?zoneCount > 1) + { + SELECT ?value (count(?mid) as ?zone) + WHERE + { + ?regions rdf:rest* ?mid . + ?mid rdf:rest ?node . + ?node rdf:first ?value . + } + GROUP BY ?node ?value + } + } + + bind(uri(concat("https://crowdscan.be/id/observation/", ?environment, "/", str(?zone))) as ?observation) . + +} diff --git a/sparql/crowdscan.add-query.rq b/sparql/crowdscan.add-query.rq new file mode 100644 index 0000000..8847ca4 --- /dev/null +++ b/sparql/crowdscan.add-query.rq @@ -0,0 +1,30 @@ +PREFIX rdf: +PREFIX xsd: + +PREFIX : +PREFIX http: +PREFIX obs: +PREFIX prov: + +CONSTRUCT { + ?request a :HttpRequest . + ?request http:url ?url . + ?request http:body ?body . + ?request http:method "POST" . + ?request http:header ?type . + ?request http:header ?accept . +} WHERE { + ?obs rdf:type obs:OM_Observation . + ?obs obs:OM_Observation.featureOfInterest ?loc . + ?obs prov:generatedAtTime ?obstime . + + + bind(concat(replace(replace(str(?loc), "www.crowdscan.be", "crowdscan.be"),"box4","beacon"), "/") as ?locpart) . + + # set HTTP request parameters + bind(bnode() as ?request) . + bind("http://rdf4j-server:8080/rdf4j-server/repositories/locations" as ?url) . + bind(concat("describe ?zone From Where { { select (max(?t) as ?max) Where { ?z ?time . Filter(?t < \"", str(?obstime), "\") . bind(str(?time) as ?t) . } } bind(IRI(concat(\"", ?locpart, "\", ?max)) as ?zone) . }") as ?body) . + bind("Content-Type: application/sparql-query" as ?type) . + bind("Accept: application/n-quads" as ?accept) . +} diff --git a/sparql/crowdscan.remove-query.rq b/sparql/crowdscan.remove-query.rq new file mode 100644 index 0000000..a574af3 --- /dev/null +++ b/sparql/crowdscan.remove-query.rq @@ -0,0 +1,15 @@ +PREFIX : + + +CONSTRUCT { + ?s ?p ?o . +} +WHERE { + ?s ?p ?o . + + {SELECT ?request + WHERE { + ?request a :HttpRequest . + }} + FILTER(?s != ?request) +} \ No newline at end of file diff --git a/sparql/map.to-zone.rq b/sparql/map.to-zone.rq new file mode 100644 index 0000000..3ab834a --- /dev/null +++ b/sparql/map.to-zone.rq @@ -0,0 +1,53 @@ +PREFIX rdf: +PREFIX xsd: +PREFIX schema: +PREFIX time: +PREFIX geojson: +PREFIX locn: +PREFIX gsp: +PREFIX sf: +PREFIX dcterms: + +PREFIX : + +CONSTRUCT { + + GRAPH ?graph { + ?entity a ?type ; + dcterms:modified ?last_updated ; + :region ?region ; + :sn ?sn ; + :installationTime ?installation_time ; + :environment ?env ; + locn:geometry [a ?geometryType ; gsp:asWKT ?geometry ] . + } + +} WHERE { + + ?feature geojson:properties ?properties . + ?feature locn:geometry ?geometry . + + OPTIONAL { + ?properties rdf:type :gateway . + ?properties :last_updated ?last_updated . + ?properties :env ?env . + ?properties :installation_time ?installation_time . + ?properties :sn ?sn . + bind(sf:Polygon as ?geometryType) . + bind( as ?graph) . + bind(:Gateway as ?type) . + bind(bnode() as ?entity) . + } + + OPTIONAL { + ?properties rdf:type :zone . + ?properties :region ?region . + ?properties :last_updated ?last_updated . + bind(sf:Point as ?geometryType) . + bind(:Zone as ?type) . + ?sub :env ?environment . + bind(uri(concat("https://www.crowdscan.be/id/zone/", ?environment, "/", STR(?region))) as ?entity) . + bind(?entity as ?graph) . + } + +} diff --git a/sparql/observation.to-oslo.rq b/sparql/observation.to-oslo.rq new file mode 100644 index 0000000..a1a5476 --- /dev/null +++ b/sparql/observation.to-oslo.rq @@ -0,0 +1,26 @@ +PREFIX rdf: +PREFIX xsd: +PREFIX schema: +PREFIX time: +PREFIX iso19156-om: +PREFIX sosa: +PREFIX dv-seb: +PREFIX : + +CONSTRUCT { + ?observation a iso19156-om:OM_Observation ; + iso19156-om:OM_Observation.resultTime ?time ; + iso19156-om:OM_Observation.phenomenonTime [ a time:Interval; time:hasXSDDuration ?delta; time:hasEnd [a time:Instant; time:inXSDDateTimeStamp ?time ] ]; + iso19156-om:OM_Observation.result [ a schema:QuantitativeValue ; schema:value ?value ] ; + iso19156-om:OM_Observation.observedProperty :PeopleEstimate ; + iso19156-om:OM_Observation.featureOfInterest ?feature ; + iso19156-om:OM_Observation.procedure [ a sosa:Procedure ; dv-seb:Observatieprocedure.specificatie ] . +} WHERE { + ?observation rdf:type :Observation . + ?observation :time ?time . + ?observation :value ?value . + ?observation :timedelta ?delta . + ?observation :environment ?env . + ?observation :region ?zone . + bind(IRI(concat("https://www.crowdscan.be/id/zone/", ?env, "/", STR(?zone))) AS ?feature) . +} \ No newline at end of file diff --git a/sparql/zone.to-oslo.rq b/sparql/zone.to-oslo.rq new file mode 100644 index 0000000..4f2c9a5 --- /dev/null +++ b/sparql/zone.to-oslo.rq @@ -0,0 +1,60 @@ +PREFIX rdfs: +PREFIX schema: +PREFIX time: +PREFIX xsd: +PREFIX dcterms: +PREFIX sosa: +PREFIX ssn: +PREFIX gsp: +PREFIX sf: +PREFIX locn: +PREFIX iso19156-sf: +PREFIX iso19156-ss: +PREFIX iso19156-sp: +PREFIX iso19156-gfi: +PREFIX iso19156-dssf: + +PREFIX : + +CONSTRUCT { + + ?zone + a iso19156-ss:SF_SamplingSurface ; + dcterms:modified ?modified ; + iso19156-sf:SF_SamplingFeature.sampledFeature ?zoneFeature; + iso19156-ss:SF_SamplingSurface.shape [ + a gsp:Surface ; + gsp:asWKT ?zoneGeometry + ] ; + sosa:isResultOf ?zoneSampling . + + ?zoneFeature + a iso19156-gfi:GFI_DomainFeature, dcterms:Location ; + rdfs:label ?placeName . + + ?zoneSampling + a sosa:Sampling ; + sosa:madeBySampler [ + a sosa:Sampler ; + locn:location [ + a locn:Geometry ; + gsp:asWKT ?gatewayGeometry + ] + ] . + +} WHERE { + + ?zone a :Zone . + ?zone dcterms:modified ?modified . + ?zone :region ?region . + ?zone locn:geometry ?zoneShape . + ?zoneShape gsp:asWKT ?zoneGeometry . + bind(BNODE() as ?zoneSampling) . + bind(BNODE() as ?zoneFeature) . + ?gateway a :Gateway . + ?gateway :environment ?environment . + ?gateway locn:geometry ?gatewayLocation . + ?gatewayLocation gsp:asWKT ?gatewayGeometry . + bind(STRLANG(concat(?environment, " zone ", STR(?region)), "nl") AS ?placeName) . + +} \ No newline at end of file