Check for voPerson & migration scriptis

app/ingester/loginsIngester.py

@@ -192,7 +192,7 @@ def ingestLoginDataPerTenenv(cls, tenenv, session):
 
         for login in loginsNotMapped:
 
-            if (login[0]['voPersonId'] in hashed_user_ids):
+            if ('voPersonId' in login[0] and login[0]['voPersonId'] in hashed_user_ids):
                 cls.logger.info("""Ignore this user with
                     hash {0} as he is at the blacklist""". format(login[0]['voPersonId']))
                 continue

metrics-migrate-stats/.gitignore

@@ -0,0 +1,6 @@
+log/*
+__pycache__
+config.py
+.venv/*
+env
+databases/GeoLite2-Country.mmdb

metrics-migrate-stats/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+

metrics-migrate-stats/Controller/countryStatisticsController.py

@@ -0,0 +1,23 @@
+from datetime import date, timedelta
+from Model.ipStatistics import ipStatistics
+from Model.countryStatistics import countryStatistics
+from datetime import datetime, timedelta
+class countryStatisticsController:
+  @classmethod
+  def getDataNotMapped(self):
+    dateFrom = countryStatistics.getLastDate()
+
+    # we dont have any country statistics saved
+    if dateFrom[0][0] == None:
+      result = ipStatistics.getAllIpStatistics()
+    else:
+      dayAfter = dateFrom[0][0] + timedelta(days=1)
+      dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00')
+
+      yesterday = date.today() - timedelta(days=1)
+      dateTo = yesterday.strftime('%Y-%m-%d 23:59:59')
+
+      result = ipStatistics.getIpStatisticsByDate(dayFrom, dateTo)
+    return result
+
+

metrics-migrate-stats/Controller/countryStatisticsHashedUserIdsController.py

@@ -0,0 +1,23 @@
+from datetime import date, timedelta
+from Model.ipStatistics import ipStatistics
+from Model.countryStatisticsHashedUserId import countryStatisticsHashedUserId
+from datetime import datetime, timedelta
+class countryStatisticsHashedUserIdsController:
+  @classmethod
+  def getDataNotMapped(self):
+    dateFrom = countryStatisticsHashedUserId.getLastDate()
+
+    # we dont have any country statistics saved
+    if dateFrom[0][0] == None:
+      result = ipStatistics.getAllIpStatistics()
+    else:
+      dayAfter = dateFrom[0][0] + timedelta(days=1)
+      dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00')
+
+      yesterday = date.today() - timedelta(days=1)
+      dateTo = yesterday.strftime('%Y-%m-%d 23:59:59')
+
+      result = ipStatistics.getIpStatisticsByDate(dayFrom, dateTo)
+    return result
+
+

metrics-migrate-stats/Controller/identityProvidersMapController.py

@@ -0,0 +1,13 @@
+from datetime import date, timedelta
+from Model.identityProvidersMapFromProxy import identityProvidersMapFromProxy
+from Model.identityProvidersMap import identityProvidersMap
+from datetime import datetime, timedelta
+class identityProvidersMapController:
+
+  @classmethod
+  def getAllData(self):
+
+    result = identityProvidersMapFromProxy.getAllIdps()
+    return result
+
+

metrics-migrate-stats/Controller/serviceProvidersMapController.py

@@ -0,0 +1,13 @@
+from datetime import date, timedelta
+from Model.serviceProvidersMapFromProxy import serviceProvidersMapFromProxy
+from Model.serviceProvidersMap import serviceProvidersMap
+from datetime import datetime, timedelta
+class serviceProvidersMapController:
+
+  @classmethod
+  def getAllData(self):
+
+    result = serviceProvidersMapFromProxy.getAllSps()
+    return result
+
+

metrics-migrate-stats/Controller/statisticsCountryHashedController.py

@@ -0,0 +1,111 @@
+from datetime import date, timedelta
+from multiprocessing.spawn import prepare
+from Model.countries import countries
+from Model.identityProvidersMap import identityProvidersMap
+from Model.ipStatistics import ipStatistics
+from Model.statisticsDetail import statisticsDetail
+from Model.statistics import statistics
+from Model.serviceProvidersMap import serviceProvidersMap
+from Model.statisticsCountryHashed import statisticsCountryHashed
+from Model.statisticsCountryHashedFromComanage import statisticsCountryHashedFromComanage
+from datetime import datetime, timedelta
+from Service.ipDatabase import geoip2Database
+from Logger import log
+import ipaddress
+
+class statisticsCountryHashedController:
+  logger = log.get_logger("statisticsCountryHashedController")
+
+  @classmethod
+  def saveAllData(self, tenenv_id):
+
+    dateFrom = statisticsCountryHashed.getLastDate(tenenv_id)
+    today = date.today()
+    dateTo = today.strftime('%Y-%m-%d 23:59:59')
+     # we dont have any country statistics saved
+    if dateFrom[0][0] == None:
+      result = ipStatistics.getStatsByDate(None, dateTo)
+    else:
+      dayAfter = dateFrom[0][0] + timedelta(days=1)
+      dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00')
+
+      yesterday = date.today() - timedelta(days=1)
+      dateTo = yesterday.strftime('%Y-%m-%d 23:59:59')
+
+      result = ipStatistics.getStatsByDate(dayFrom, dateTo)
+    statisticsCountryHashedController.prepareData(result, tenenv_id)
+
+    # now we would like to save those data we dont have ip information
+    dateTo = statisticsCountryHashed.getFirstDate(tenenv_id)
+    result = statisticsDetail.getStatsByDate(dateTo[0][0].strftime('%Y-%m-%d'))
+    statisticsCountryHashedController.prepareData(result, tenenv_id)
+    dateTo = statisticsCountryHashed.getFirstDate(tenenv_id)
+    result = statistics.getStatsByDate(dateTo[0][0].strftime('%Y-%m-%d'))
+    statisticsCountryHashedController.prepareData(result, tenenv_id)
+    return result
+
+  @classmethod
+  def prepareData(self, data, tenenv_id):
+    ipDatabaseHandler = geoip2Database()
+    mappedItems = 0
+    for item in data:
+      ## find identityprovider id
+      result = identityProvidersMap.getIdpIdByIdentifier(item.sourceidp, tenenv_id)
+      if len(result) > 0:
+        #self.logger.info(result[0][0])
+        idpId = result[0][0]
+      else:
+        self.logger.error("Identity provider identifier not found {0}".format(item.sourceidp))
+        continue
+
+      ## find serviceprovider id
+      result = serviceProvidersMap.getSpIdByIdentifier(item.service, tenenv_id)
+      if len(result) > 0:
+        #self.logger.info(result[0][0])
+        spId = result[0][0]
+      else:
+        self.logger.warning("Service entityid {0} not found".format(item.service))
+        spItem = serviceProvidersMap(item.service, item.service.replace("'", "''"), tenenv_id)
+        serviceProvidersMap.save(spItem)
+        self.logger.info("Service entityid {0} created".format(item.service))
+      try:
+          if hasattr(item, "ip"):
+              ipaddr = ipaddress.ip_network(item.ip).network_address
+          else:
+              item.countrycode = countryData[0]
+              item.country = countryData[1] 
+          # get country code/ name
+          countryData = ipDatabaseHandler.getCountryFromIp(str(ipaddr), item.ipVersion)
+          if (countryData[0] == None):
+              item.countrycode = "UN"
+              item.country = "Unknown"
+          else:
+              item.countrycode = countryData[0]
+              item.country = countryData[1]       
+      except:
+          item.countrycode = "UN"
+          item.country = "Unknown"
+       ## save country if not exists and get id
+      country = countries(item.countrycode, item.country)
+      countries.save(country)
+      result = countries.getIdByCountryCode(item.countrycode)   
+      if len(result) > 0:
+        #self.logger.info(result[0][0])
+        countryId = result[0][0]
+      else:
+        self.logger.error("Country not found")
+        continue
+      self.logger.info("{0} ".format(item.date))
+
+      if hasattr(item, "count"):
+        count = item.count
+      else:
+        count = 1
+      statsCountry = statisticsCountryHashed(item.date, item.hasheduserid, idpId, spId, countryId, count, tenenv_id)
+      statisticsCountryHashed.save(statsCountry)
+      mappedItems += 1
+    self.logger.info("{0} Country Stats created".format(mappedItems))
+
+
+
+

metrics-migrate-stats/Controller/usersController.py

@@ -0,0 +1,27 @@
+from ast import Or
+from datetime import date, timedelta
+from Model.users import users
+from Model.usersFromComanage import usersFromComanage
+from Model.vos import vos
+from datetime import datetime, timedelta
+from Logger import log
+
+class usersController:
+  logger = log.get_logger("usersController")
+
+  @classmethod
+  def saveUsers(self, tenenv_id):
+
+    usersList = []
+    # get memberships with active status
+    usersComanage = usersFromComanage.getAllUsers()
+    # save memberships with active status
+    for item in usersComanage:
+        usersItem = users(item.hasheduserid, item.created, item.status, tenenv_id)
+        usersList.append(usersItem)
+    # save data to tables if any
+    if usersList:
+      users.saveAll(usersList)
+
+
+

metrics-migrate-stats/Controller/voMembershipsController.py

@@ -0,0 +1,81 @@
+from ast import Or
+from datetime import date, timedelta
+from Model.voMemberships import voMemberships
+from Model.voMembershipsFromComanage import voMembershipsFromComanage
+from Model.vos import vos
+from datetime import datetime, timedelta
+from Logger import log
+
+class voMembershipsController:
+  logger = log.get_logger("voMembershipsController")
+
+  @classmethod
+  def getVoId(self, voName, source, tenenv_id):
+    result = vos.getVoIdFromVoName(voName, source, tenenv_id)
+    return result
+
+  @classmethod
+  def getDataNotMapped(self, tenenv_id):
+    voMemberships.truncate(tenenv_id)
+
+    voMembershipsList = []
+    # dateFrom = vos.getLastDate()
+
+    # we dont have any vos saved
+    # if dateFrom[0][0] == None:
+
+    # get memberships with active status
+    activeMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('A')
+    # save memberships with active status
+    for item in activeMemberships:
+      result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id)
+
+      if len(result) > 0:
+        voId = result[0][0]
+        self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0]))
+
+        voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id)
+        voMembershipsList.append(voMembershipsItem)
+    # save data to tables if any
+    if voMembershipsList:
+
+      voMemberships.saveAll(voMembershipsList)
+
+    # get memberships with grace period
+    # save only if there isnt any existing membership or status membership is not Active/ Grace Period
+    gracePeriodVoMembershipsList = []
+    graceperiodMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('GP')
+    for item in graceperiodMemberships:
+      result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id)
+      if len(result) > 0:
+        voId = result[0][0]
+        self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0]))
+        #check if already there is a membership with active status saved
+        statusMembership = voMemberships.getMembershipStatus(voId, item.hasheduserid, tenenv_id)
+        if len(statusMembership) > 0  and (statusMembership[0][0]=='A' or statusMembership[0][0]=='GP'):
+            continue
+        voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id)
+        gracePeriodVoMembershipsList.append(voMembershipsItem)
+    if len(gracePeriodVoMembershipsList)>0:
+        voMemberships.saveAll(gracePeriodVoMembershipsList)
+
+    # get memberships with other statuses
+    # save only if there isnt any existing membership
+    otherStatusVoMembershipsList = []
+    otherStatusMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('Other')
+    for item in otherStatusMemberships:
+      result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id)
+      if len(result) > 0:
+        voId = result[0][0]
+        self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0]))
+        #check if already there is a membership with active status saved
+        statusMembership = voMemberships.getMembershipStatus(voId, item.hasheduserid, tenenv_id)
+        if len(statusMembership) > 0:
+          continue
+        voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id)
+        otherStatusVoMembershipsList.append(voMembershipsItem)
+    if len(otherStatusVoMembershipsList)>0:
+        voMemberships.saveAll(otherStatusVoMembershipsList)
+    return result
+
+

metrics-migrate-stats/Controller/vosController.py

@@ -0,0 +1,24 @@
+from datetime import date, timedelta
+from Model.vosFromComanage import vosFromComanage
+from Model.vos import vos
+from datetime import datetime, timedelta
+
+class vosController:
+  @classmethod
+  def getDataNotMapped(self, tenenv_id):
+    dateFrom = vos.getLastDate(tenenv_id)
+
+    # we dont have any vos saved
+    if dateFrom[0][0] == None:
+      result = vosFromComanage.getAllVos()
+    else:
+      dayAfter = dateFrom[0][0] + timedelta(days=1)
+      dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00')
+
+      yesterday = date.today() - timedelta(days=1)
+      dateTo = yesterday.strftime('%Y-%m-%d 23:59:59')
+
+      result = vosFromComanage.getVosByDate(dayFrom, dateTo)
+    return result
+
+

metrics-migrate-stats/Logger/log.py

@@ -0,0 +1,29 @@
+import logging
+import sys
+from Utils import configParser
+from logging.handlers import TimedRotatingFileHandler
+
+FORMATTER = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+LOG_FILE = "{0}/{1}".format(configParser.getConfig('logging')['folder'] ,configParser.getConfig('logging')['file'])
+LEVEL = configParser.getConfig('logging')['level']
+
+def get_console_handler():
+	console_handler = logging.StreamHandler(sys.stdout)
+	console_handler.setFormatter(FORMATTER)
+	return console_handler
+
+def get_file_handler():
+	file_handler = TimedRotatingFileHandler(LOG_FILE, when='midnight')
+	file_handler.setFormatter(FORMATTER)
+	return file_handler
+
+def get_logger(logger_name):
+
+  logger = logging.getLogger(logger_name)	
+  logger.setLevel(LEVEL)	
+  logger.addHandler(get_console_handler())	
+  logger.addHandler(get_file_handler())
+	# with this pattern, it's rarely necessary to propagate the error up to parent	
+  logger.propagate = False
+
+  return logger