Fix #244 jurisdiction lookup by country now uses geo:country (rather …

…than geoci:country), cleanup cleaned up geo dao & cache conf now uses facade rather than versioned projects commented TODOs for #252 geo cache & dao archi cleanup patched (replication) test conf
ozwillo · Sep 1, 2015 · 28af904 · 28af904
1 parent f95c01d
commit 28af904
Show file tree

Hide file tree

Showing 5 changed files with 128 additions and 78 deletions.
diff --git a/...tal-front/src/main/java/org/oasis_eu/portal/core/mongo/dao/geo/GeographicalAreaCache.java b/...tal-front/src/main/java/org/oasis_eu/portal/core/mongo/dao/geo/GeographicalAreaCache.java
@@ -43,6 +43,12 @@
 import com.mongodb.Mongo;
 
 /**
+ * TODO move generic & org stuff outside, better Service vs DAO archi
+ * 
+ * TODO LATER make generic DatacoreBusinessFulltextCache<BO> (business object)
+ * and concretize GeographicalAreaFulltextCache extends DatacoreBusinessFulltextCache<GeographicalArea>
+ * created in its own @Configuration class see http://stackoverflow.com/questions/11845871/spring-value-annotation-inheritance-and-common-fields-but-different-values
+ *
  * User: schambon
  * Date: 4/16/15
  */
@@ -72,17 +78,18 @@ public class GeographicalAreaCache {
     @Value("${application.geoarea.replication_batch_size:100}")
     private int batchSize = 100;
 
-    @Value("${application.geoarea.project:geo_0}")
+    /** geo_0/1... can also be used to use a version not yet published (i.e. made visible in geo) */
+    @Value("${application.geoarea.project:geo}")
     private String project;
 
     @Value("${application.geoarea.areaModel:geo:Area_0}")
     private String areaModel; //"geo:Area_0"; // "geoci:City_0"
-    /** ex. Rhône-Alpes, France */
-    @Value("${application.geoarea.primaryNameField:odisp:name}")
-    private String displayNameField = "odisp:name"; //geo:displayName
-    /** ex. Rhône-Alpes */
-    @Value("${application.geoarea.secondaryNameField:geo:name}")
-    private String nameField; // "geoci:name";
+    /** used as secondary search field in cache ex. Rhône-Alpes */
+    @Value("${application.geoarea.nameField:geo:name}")
+    private String nameField; // or city specific "geoci:name", country-specific "geoco:name"
+    /** USED AS PRIMARY SEARCH FIELD IN CACHE ex. Rhône-Alpes, France */
+    @Value("${application.geoarea.displayNameField:odisp:name}")
+    private String displayNameField;// or geo specific "geo:displayName"
     /** ex. "http://data.ozwillo.com/dc/type/geocifr:Commune_0/FR/FR-38/Saint-Clair-de-la-Tour" */
     @Value("${application.geoarea.searchCronField:@id}")
     private String searchCronField; // "@id";
@@ -163,19 +170,21 @@ public float score() {
 
     }
 
-    private Stream<GeographicalArea> findOneToken(String country, String modelType, String lang, String name) {
+    /**
+     * 
+     * @param countryUri URI of DC country Resource, therefore already encoded
+     * @param modelType
+     * @param lang
+     * @param name
+     * @return
+     */
+    private Stream<GeographicalArea> findOneToken(String countryUri, String modelType, String lang, String name) {
         // we search irrespective of the replication status, but we deduplicate based on DC Resource URI.
         // sort spec means we want older results first - so that incoming replicates are discarded as long as
         // there is an online entry
-        String encodedCountry = country; //comes already encoded
-        /*try{
-            encodedCountry = UriComponentsBuilder.fromUriString(country).build().encode().toString();
-        }catch(Exception e){
-            logger.debug("The country URI \"{}\" cannot be encoded : {}", country, e.toString());
-        }*/
         Criteria criteria = where("lang").is(lang);
-        if (encodedCountry != null && !encodedCountry.trim().isEmpty()){
-            criteria.and("country").is(encodedCountry); //filter by country
+        if (countryUri != null && !countryUri.trim().isEmpty()){
+            criteria.and("country").is(countryUri); //filter by country
         }
         if (modelType != null && !modelType.trim().isEmpty()){
             criteria.and("modelType").in(modelType);

diff --git a/...s-portal-front/src/main/java/org/oasis_eu/portal/services/dc/geoarea/GeographicalDAO.java b/...s-portal-front/src/main/java/org/oasis_eu/portal/services/dc/geoarea/GeographicalDAO.java
@@ -1,8 +1,8 @@
 package org.oasis_eu.portal.services.dc.geoarea;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.oasis_eu.portal.core.mongo.dao.geo.GeographicalAreaCache;
 import org.oasis_eu.portal.core.mongo.model.geo.GeographicalArea;
@@ -20,6 +20,12 @@
 import org.springframework.stereotype.Component;
 
 /**
+ * TODO #252 move generic & org stuff outside, better Service vs DAO archi
+ * 
+ * TODO LATER #252 make generic DatacoreBusinessFulltextCache/Dao<BO> (business object)
+ * and concretize GeographicalAreaFulltextCache extends DatacoreBusinessFulltextCache<GeographicalArea>
+ * created in its own @Configuration class see http://stackoverflow.com/questions/11845871/spring-value-annotation-inheritance-and-common-fields-but-different-values
+ *
  * User: Ignacio
  * Date: 7/2/15
  */
@@ -31,26 +37,26 @@ public class GeographicalDAO {
     @Autowired
     private DatacoreClient datacore;
 
-    @Value("${application.geoarea.project:geo_0}")
+    /** geo_0/1... can also be used to use a version not yet published (i.e. made visible in geo) */
+    @Value("${application.geoarea.project:geo}")
     private String geoProject;
-    @Value("${application.dcOrg.project: org_0}")
-    private String dcOrgProjectName;// = "org_0";
-
-    @Value("${application.geoarea.geoCityModel: geo:City_0}")
-    private String geoCityModel;// = "geoci:City_0";
-    @Value("${application.geoarea.cityField: geoci_city:name}")
-    private String cityField;// = "geo_city:name"; // "geoci:name";
-    @Value("${application.geoarea.countryCityField: geoci:country}")
-    private String countryCityField;// = "geoco:Country_0"; //"geocofr:Pays_0";
-
-    @Value("${application.geoarea.geoCountriesModel: geoco:Country_0}")
-    private String geoCountriesModel;// = "geoco:Country_0"; //"geocofr:Pays_0";
+    /** org_0/1... can also be used to use a version not yet published (i.e. made visible in org) */
+    @Value("${application.dcOrg.project:org}")
+    private String dcOrgProjectName;
+
     /**ex. Rhône-Alpes */
-    @Value("${application.geoarea.countryField: geoco:name}")
-    private String countryField;// = "geoco:name"; // "geo:name";
+    @Value("${application.geoarea.nameField:geo:name}")
+    private String nameField; // or city specific "geoci:name", country-specific "geoco:name"
     /** ex. Rhône-Alpes, France */
     @Value("${application.geoarea.displayNameField:odisp:name}")
-    private String displayNameField;// = "odisp:name"; //geo:displayName
+    private String displayNameField;// or geo specific "geo:displayName"
+
+    @Value("${application.geoarea.countryModel:geoco:Country_0}")
+    private String countryModel; // or country specific "geocofr:Pays_0"
+    @Value("${application.geoarea.cityModel:geoci:City_0}")
+    private String cityModel;
+    @Value("${application.geoarea.countryField:geo:country}")
+    private String countryField; // or city specific "geoci:country"
 
     @Autowired
     private Tokenizer tokenizer;
@@ -61,40 +67,41 @@ public class GeographicalDAO {
     // Cities
     public List<GeographicalArea> searchCities(String lang, String queryTerm, String countryUri, int start, int limit) {
 
-        List<GeographicalArea> geographicalArea = new ArrayList<GeographicalArea>();
-
-        fetchResourceByCountryAndNameStartingWith(queryTerm, cityField, ".v", countryUri, countryCityField, geoProject, geoCityModel, limit - start)
-               .stream().forEach(resource -> geographicalArea.add(toGeographicalArea(resource,lang, displayNameField.trim(), cityField.trim())));
-
-       return geographicalArea;
+       return fetchResourceByCountryAndNameStartingWith(queryTerm, nameField, ".v", countryUri, countryField, geoProject, cityModel, limit - start)
+               .stream().map(resource -> toGeographicalArea(resource,lang, displayNameField.trim(), nameField.trim()))
+               .collect(Collectors.toList());
     }
 
     // Countries
     public List<GeographicalArea> searchCountries(String lang, String term, int start, int limit) {
-
-        List<GeographicalArea> geographicalArea = new ArrayList<GeographicalArea>();
-
-        fetchResourceByCountryAndNameStartingWith(term, countryField, ".v", null,null, geoProject,geoCountriesModel, limit - start)
-            .stream().forEach(resource -> geographicalArea.add(toGeographicalArea(resource,lang, displayNameField.trim(), countryField.trim())));
-
-        return geographicalArea;
+        return fetchResourceByCountryAndNameStartingWith(term, nameField, ".v", null,null, geoProject, countryModel, limit - start)
+              .stream().map(resource -> toGeographicalArea(resource,lang, displayNameField.trim(), nameField.trim()))
+              .collect(Collectors.toList());
     }
 
     // Tax Reg Activity
     public List<DCRegActivity> searchTaxRegActivity(String countryUri, String queryTerms, int start, int limit) {
-
-        List<DCRegActivity> taxRegActivities = new ArrayList<DCRegActivity>();
-
-        fetchResourceByCountryAndNameStartingWith(queryTerms, "orgact:code", null, countryUri, "orgact:country", dcOrgProjectName, "orgact:Activity_0", limit - start)
-               .stream().forEach(resource -> taxRegActivities.add(toDCRegActivity(resource, displayNameField.trim(), cityField.trim())));
-
-        return taxRegActivities;
+        return fetchResourceByCountryAndNameStartingWith(queryTerms, "orgact:code", null, countryUri, "orgact:country", dcOrgProjectName, "orgact:Activity_0", limit - start)
+               .stream().map(resource -> toDCRegActivity(resource))
+               .collect(Collectors.toList());
     }
 
 
 
     // Helper & Handler methods
 
+    /**
+     * 
+     * @param queryTerm
+     * @param field
+     * @param subField
+     * @param countryUri (already encoded)
+     * @param countryField
+     * @param projectName
+     * @param modelName
+     * @param batchSize
+     * @return
+     */
     private List<DCResource> fetchResourceByCountryAndNameStartingWith(String queryTerm, String field, String subField,
             String countryUri, String countryField, String projectName, String modelName, int batchSize ){
 
@@ -145,7 +152,7 @@ public GeographicalArea toGeographicalArea(DCResource r, String language, String
             //TODO LATER: Create a full body DC interceptor to test request/response to DATACORE (similar to KernelLoggingInterceptor)
         }
 
-        String country = r.getAsString("geo:country");  /* The true value should be the main referenced model id (geo:country), but today
+        String country = r.getAsString(countryField);  /* The true value should be the main referenced model id (geo:country), but today
                                                            * it is not linked in the models. NB today it makes that some of the fields are not
                                                            * stored due to this field (been empty for those cases) */
         List<String> modelType = r.getAsStringList("@type");
@@ -170,7 +177,8 @@ public GeographicalArea toGeographicalArea(DCResource r, String language, String
 
         return area;
     }
-
+
+    /** TODO move to generic (-integration ? DCResource ??) */
     @SuppressWarnings("unchecked")
     private List<Map<String, String>> getBestI18nValue(DCResource resource, String fieldName, String altFieldName){
         List<Map<String, String>> nameMaps = (List<Map<String, String>>) resource.get(fieldName);
@@ -180,8 +188,9 @@ private List<Map<String, String>> getBestI18nValue(DCResource resource, String f
         }
         return nameMaps;
     }
-
-    public DCRegActivity toDCRegActivity(DCResource r, String nameField, String altFieldName) {
+
+    /** TODO move to OrgDAO */
+    public DCRegActivity toDCRegActivity(DCResource r) {
         String code = r.getAsString("orgact:code");
         String country = r.getAsString("orgact:country");
         String label = r.getAsString("orgact:label");

diff --git a/portal-parent/oasis-portal-front/src/main/resources/application.yml b/portal-parent/oasis-portal-front/src/main/resources/application.yml
@@ -23,21 +23,19 @@ application:
     # (and even then it's better supported by UI sidebar's dropdown being inited to current locale)
     addCurrentToSupportedLocalesIfNone: false
   geoarea:
-    project: geo_1
-    areaModel: geo:Area_0
-    primaryNameField: odisp:name
-    secondaryNameField: geo:name
-    geoCityModel: geoci:City_0
-    cityField: geoci:name
-    geoCountryModel: geoco:Country_0
-    countryField: geoco:name
+    project: geo
     displayNameField: odisp:name
+    areaModel: geo:Area_0
+    nameField: geo:name
+    countryModel: geoco:Country_0
+    cityModel: geoci:City_0
+    countryField: geo:country
     fallbackLanguage: en
     #use: Seconds | Min | Hour | Day-of-month | Month | Day-of-week(sunday=0) | Year = http://www.quartz-scheduler.org/documentation/quartz-1.x/tutorials/crontrigger
     replication: 0 0 1 1/5 * ?  #Runs at 1am every 5 days every month, starting on the first day of the month
     #replication: 0 0/1 * * * ?     #Runs every minute
   dcOrg:
-    project: org_1
+    project: org
     orgModel: org:Organization_0
     baseUri: http://data.ozwillo.com/dc/type
   dcOrgSearch:
@@ -96,7 +94,7 @@ kernel:
   event_bus_endpoint: ${kernel.base_uri}/e
   notifications_endpoint: ${kernel.base_uri}/n
   client_id: portal
-  client_secret: 7706ded0-afe4-4826-9df1-ce03782b893d
+  client_secret: changeit
   # NB. once Datacore cache is in place, move datacore scope to its dedicated system user
   scopes_to_require: openid profile email address phone datacore
   maxUserMembershipsPerPage: 20
@@ -107,12 +105,16 @@ kernel:
 
 datacore:
   #url: http://localhost:8080
-  url: https://plnm-dev-dc             # dev, requires installing gen'd cert, and (MUST) set the ip in /etc/hosts
+  url: https://plnm-dev-dc             # dev, requires (MUST) set the IP in /etc/hosts, and installing gen'd cert :
+  # go to https://data.ozwillo-dev.eu, right click on lock, export certificate to plnm-dev-dc.cer,
+  # then import it and check :
+  # sudo keytool -import -alias plnm-dev-dc -keystore /usr/lib/jvm/jdk1.8.xx/jre/lib/security/cacerts -file plnm-dev-dc.cer
+  # keytool -list -v -keystore /usr/lib/jvm/jdk1.8.xx/jre/lib/security/cacerts
   #url: https://data.ozwillo-dev.eu    # dev, doesn't work with current gen'd cert
   # alt_uri ??
   systemAdminUser:  #Get it using Node.js lib / with portal clientId
-    refreshToken: oxNDM3NDY5MjQ4OTUyLCJleHjkyNDg5NTJ9eyJpZCI6Ijk2Y2IxY2IyLTU1YWEtNDYyZC04NGNmLTVlMTAyMjAzNWVmZS8wTHZsNTdXNlliYWtwSWVuZHQ0UVhnIiwiaWF0IjAiOjMwMTQyN
-    nonce: 970774665428326      #Set it when getting the refreshToken
+    refreshToken: changeit
+    nonce: changeit      #Set it when getting the refreshToken
 
 
 persistence:

diff --git a/...oarea/GeographicalAreaRelicationTest.java → ...area/GeographicalAreaReplicationTest.java b/...oarea/GeographicalAreaRelicationTest.java → ...area/GeographicalAreaReplicationTest.java
@@ -16,11 +16,20 @@
 
 import static org.junit.Assert.assertNotEquals;
 
+
+/**
+ * Its output can also be used to fill non-test portal instance cache, this way :
+ * mongodump -d portal_test -c geographical_area -o portal_geoarea1
+ * mongorestore -d portal -c geographical_area portal_geoarea1/portal_test/geographical_area.bson
+ * 
+ * @author schambon
+ *
+ */
 @RunWith(SpringJUnit4ClassRunner.class)
 @WebAppConfiguration
 @SpringApplicationConfiguration(classes = {OasisPortal.class})
 @IntegrationTest
-public class GeographicalAreaRelicationTest {
+public class GeographicalAreaReplicationTest {
 
     @Autowired
     private GeographicalAreaCache cache;

diff --git a/portal-parent/oasis-portal-front/src/test/resources/application.yml b/portal-parent/oasis-portal-front/src/test/resources/application.yml
@@ -10,7 +10,7 @@ ha:
   failover:
 
 application:
-  url: http://localhost
+  url: http://localhost:8080
   security:
     fetchUserInfo: true
     # requires devmode, to ease up development ex. of Catalog only
@@ -23,11 +23,29 @@ application:
     # (and even then it's better supported by UI sidebar's dropdown being inited to current locale)
     addCurrentToSupportedLocalesIfNone: false
   geoarea:
-    storageModel: geo:City_0
-    nameField: geo_city:name
-    fallbackLanguage: en
+    project: geo
+    displayNameField: odisp:name
+    areaModel: geo:Area_0
+    nameField: geo:name
+    countryModel: geoco:Country_0
+    cityModel: geoci:City_0
+    countryField: geo:country
+    #use: Seconds | Min | Hour | Day-of-month | Month | Day-of-week(sunday=0) | Year = http://www.quartz-scheduler.org/documentation/quartz-1.x/tutorials/crontrigger
+    replication: 0 0 1 1/5 * ?  #Runs at 1am every 5 days every month, starting on the first day of the month
+    #replication: 0 0/1 * * * ?     #Runs every minute
     replication: 0 0 4 25 DEC SUN  # no really, don't.
     replication_test: false
+  dcOrg:
+    project: org
+    orgModel: org:Organization_0
+    baseUri: http://data.ozwillo.com/dc/type
+  dcOrgSearch:
+    sector: org:sector
+    legalName: org:legalName.v
+    regNumber: org:regNumber
+    country: adrpost:country
+  organizationDaysTillDeletedFromTrash: 7
+  applicationInstanceDaysTillDeletedFromTrash: 7
 
 web:
   home: http://www.ozwillo-dev.eu
@@ -44,7 +62,7 @@ web:
 kernel:
   token:
     expiry: 300
-  base_uri: https://oasis-demo.atolcd.com
+  base_uri: http://kernel.ozwillo-dev.eu
   auth:
     issuer: ${kernel.base_uri}/
     auth_endpoint: ${kernel.base_uri}/a/auth
@@ -58,8 +76,8 @@ kernel:
   event_bus_endpoint: ${kernel.base_uri}/e
   notifications_endpoint: ${kernel.base_uri}/n
   application_id: 8b0e62e5-e3af-45de-8834-0d7d8a3f6139
-  client_id: 7706ded0-afe4-4826-9df1-ce03782b893d
-  client_secret: 7706ded0-afe4-4826-9df1-ce03782b893d
+  client_id: portal
+  client_secret: changeit
   scopes_to_require: openid profile email address
 
   portal_endpoints:
@@ -74,6 +92,9 @@ datacore:
   url: https://plnm-dev-dc # dev, requires installing gen'd cert
   #url: https://data.ozwillo-dev.eu # dev, doesn't work with current gen'd cert
   # alt_uri ??
+  systemAdminUser:  #Get it using Node.js lib / with portal clientId
+    refreshToken: changeit
+    nonce: changeit      #Set it when getting the refreshToken
 
 
 persistence: