From 499ea41a88ac3f60ecacb2d407d4324957e96851 Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Wed, 15 Nov 2023 09:51:32 +0200 Subject: [PATCH 1/5] check if voPerson exists --- app/ingester/loginsIngester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ingester/loginsIngester.py b/app/ingester/loginsIngester.py index 0a92079..559aa7b 100644 --- a/app/ingester/loginsIngester.py +++ b/app/ingester/loginsIngester.py @@ -192,7 +192,7 @@ def ingestLoginDataPerTenenv(cls, tenenv, session): for login in loginsNotMapped: - if (login[0]['voPersonId'] in hashed_user_ids): + if ('voPersonId' in login[0] and login[0]['voPersonId'] in hashed_user_ids): cls.logger.info("""Ignore this user with hash {0} as he is at the blacklist""". format(login[0]['voPersonId'])) continue From 8f4ce03f87b03536efa7861db580673f70a8a810 Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Fri, 17 Nov 2023 09:51:28 +0200 Subject: [PATCH 2/5] add migration scripts --- metrics-migrate-stats/.gitignore | 6 + metrics-migrate-stats/CHANGELOG.md | 7 + .../Controller/countryStatisticsController.py | 23 +++ ...ountryStatisticsHashedUserIdsController.py | 23 +++ .../identityProvidersMapController.py | 13 ++ .../serviceProvidersMapController.py | 13 ++ .../statisticsCountryHashedController.py | 111 +++++++++++ .../Controller/usersController.py | 27 +++ .../Controller/voMembershipsController.py | 81 ++++++++ .../Controller/vosController.py | 24 +++ metrics-migrate-stats/Logger/log.py | 29 +++ metrics-migrate-stats/Model/countries.py | 27 +++ .../Model/countryStatistics.py | 34 ++++ .../Model/countryStatisticsHashedUserId.py | 37 ++++ metrics-migrate-stats/Model/environments.py | 30 +++ .../Model/identityProvidersMap.py | 35 ++++ .../Model/identityProvidersMapFromProxy.py | 20 ++ metrics-migrate-stats/Model/ipStatistics.py | 58 ++++++ metrics-migrate-stats/Model/pgConnector.py | 133 +++++++++++++ .../Model/serviceProvidersMap.py | 32 ++++ .../Model/serviceProvidersMapFromProxy.py | 20 ++ metrics-migrate-stats/Model/statistics.py | 25 +++ .../Model/statisticsCountryHashed.py | 66 +++++++ .../statisticsCountryHashedFromComanage.py | 30 +++ .../Model/statisticsDetail.py | 24 +++ metrics-migrate-stats/Model/tenants.py | 30 +++ metrics-migrate-stats/Model/tenenvs.py | 32 ++++ .../Model/unknownCountriesLogins.py | 29 +++ .../Model/userCountryStatistics.py | 33 ++++ metrics-migrate-stats/Model/users.py | 45 +++++ .../Model/usersFromComanage.py | 35 ++++ metrics-migrate-stats/Model/voMemberships.py | 59 ++++++ .../Model/voMembershipsFromComanage.py | 58 ++++++ metrics-migrate-stats/Model/vos.py | 46 +++++ .../Model/vosFromComanage.py | 40 ++++ metrics-migrate-stats/Model/vosInfo.py | 36 ++++ metrics-migrate-stats/README.md | 29 +++ metrics-migrate-stats/Service/ipDatabase.py | 18 ++ metrics-migrate-stats/Utils/configParser.py | 32 ++++ metrics-migrate-stats/Utils/install.py | 40 ++++ .../Utils/migrateKeycloakUsers.py | 46 +++++ .../config-templates/pgsql_tables.sql | 177 ++++++++++++++++++ .../config-templates/pgsql_tables_old.sql | 142 ++++++++++++++ metrics-migrate-stats/databases/empty | 0 metrics-migrate-stats/migrateData.py | 104 ++++++++++ .../migrateData_noCOmanage.py | 104 ++++++++++ metrics-migrate-stats/requirements.txt | 2 + metrics-migrate-stats/test.py | 34 ++++ 48 files changed, 2099 insertions(+) create mode 100644 metrics-migrate-stats/.gitignore create mode 100644 metrics-migrate-stats/CHANGELOG.md create mode 100644 metrics-migrate-stats/Controller/countryStatisticsController.py create mode 100644 metrics-migrate-stats/Controller/countryStatisticsHashedUserIdsController.py create mode 100644 metrics-migrate-stats/Controller/identityProvidersMapController.py create mode 100644 metrics-migrate-stats/Controller/serviceProvidersMapController.py create mode 100644 metrics-migrate-stats/Controller/statisticsCountryHashedController.py create mode 100644 metrics-migrate-stats/Controller/usersController.py create mode 100644 metrics-migrate-stats/Controller/voMembershipsController.py create mode 100644 metrics-migrate-stats/Controller/vosController.py create mode 100644 metrics-migrate-stats/Logger/log.py create mode 100644 metrics-migrate-stats/Model/countries.py create mode 100644 metrics-migrate-stats/Model/countryStatistics.py create mode 100644 metrics-migrate-stats/Model/countryStatisticsHashedUserId.py create mode 100644 metrics-migrate-stats/Model/environments.py create mode 100644 metrics-migrate-stats/Model/identityProvidersMap.py create mode 100644 metrics-migrate-stats/Model/identityProvidersMapFromProxy.py create mode 100644 metrics-migrate-stats/Model/ipStatistics.py create mode 100644 metrics-migrate-stats/Model/pgConnector.py create mode 100644 metrics-migrate-stats/Model/serviceProvidersMap.py create mode 100644 metrics-migrate-stats/Model/serviceProvidersMapFromProxy.py create mode 100644 metrics-migrate-stats/Model/statistics.py create mode 100644 metrics-migrate-stats/Model/statisticsCountryHashed.py create mode 100644 metrics-migrate-stats/Model/statisticsCountryHashedFromComanage.py create mode 100644 metrics-migrate-stats/Model/statisticsDetail.py create mode 100644 metrics-migrate-stats/Model/tenants.py create mode 100644 metrics-migrate-stats/Model/tenenvs.py create mode 100644 metrics-migrate-stats/Model/unknownCountriesLogins.py create mode 100644 metrics-migrate-stats/Model/userCountryStatistics.py create mode 100644 metrics-migrate-stats/Model/users.py create mode 100644 metrics-migrate-stats/Model/usersFromComanage.py create mode 100644 metrics-migrate-stats/Model/voMemberships.py create mode 100644 metrics-migrate-stats/Model/voMembershipsFromComanage.py create mode 100644 metrics-migrate-stats/Model/vos.py create mode 100644 metrics-migrate-stats/Model/vosFromComanage.py create mode 100644 metrics-migrate-stats/Model/vosInfo.py create mode 100644 metrics-migrate-stats/README.md create mode 100644 metrics-migrate-stats/Service/ipDatabase.py create mode 100644 metrics-migrate-stats/Utils/configParser.py create mode 100644 metrics-migrate-stats/Utils/install.py create mode 100644 metrics-migrate-stats/Utils/migrateKeycloakUsers.py create mode 100644 metrics-migrate-stats/config-templates/pgsql_tables.sql create mode 100644 metrics-migrate-stats/config-templates/pgsql_tables_old.sql create mode 100644 metrics-migrate-stats/databases/empty create mode 100644 metrics-migrate-stats/migrateData.py create mode 100644 metrics-migrate-stats/migrateData_noCOmanage.py create mode 100644 metrics-migrate-stats/requirements.txt create mode 100644 metrics-migrate-stats/test.py diff --git a/metrics-migrate-stats/.gitignore b/metrics-migrate-stats/.gitignore new file mode 100644 index 0000000..869bcd0 --- /dev/null +++ b/metrics-migrate-stats/.gitignore @@ -0,0 +1,6 @@ +log/* +__pycache__ +config.py +.venv/* +env +databases/GeoLite2-Country.mmdb diff --git a/metrics-migrate-stats/CHANGELOG.md b/metrics-migrate-stats/CHANGELOG.md new file mode 100644 index 0000000..721294a --- /dev/null +++ b/metrics-migrate-stats/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + diff --git a/metrics-migrate-stats/Controller/countryStatisticsController.py b/metrics-migrate-stats/Controller/countryStatisticsController.py new file mode 100644 index 0000000..2de895c --- /dev/null +++ b/metrics-migrate-stats/Controller/countryStatisticsController.py @@ -0,0 +1,23 @@ +from datetime import date, timedelta +from Model.ipStatistics import ipStatistics +from Model.countryStatistics import countryStatistics +from datetime import datetime, timedelta +class countryStatisticsController: + @classmethod + def getDataNotMapped(self): + dateFrom = countryStatistics.getLastDate() + + # we dont have any country statistics saved + if dateFrom[0][0] == None: + result = ipStatistics.getAllIpStatistics() + else: + dayAfter = dateFrom[0][0] + timedelta(days=1) + dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00') + + yesterday = date.today() - timedelta(days=1) + dateTo = yesterday.strftime('%Y-%m-%d 23:59:59') + + result = ipStatistics.getIpStatisticsByDate(dayFrom, dateTo) + return result + + diff --git a/metrics-migrate-stats/Controller/countryStatisticsHashedUserIdsController.py b/metrics-migrate-stats/Controller/countryStatisticsHashedUserIdsController.py new file mode 100644 index 0000000..1c46258 --- /dev/null +++ b/metrics-migrate-stats/Controller/countryStatisticsHashedUserIdsController.py @@ -0,0 +1,23 @@ +from datetime import date, timedelta +from Model.ipStatistics import ipStatistics +from Model.countryStatisticsHashedUserId import countryStatisticsHashedUserId +from datetime import datetime, timedelta +class countryStatisticsHashedUserIdsController: + @classmethod + def getDataNotMapped(self): + dateFrom = countryStatisticsHashedUserId.getLastDate() + + # we dont have any country statistics saved + if dateFrom[0][0] == None: + result = ipStatistics.getAllIpStatistics() + else: + dayAfter = dateFrom[0][0] + timedelta(days=1) + dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00') + + yesterday = date.today() - timedelta(days=1) + dateTo = yesterday.strftime('%Y-%m-%d 23:59:59') + + result = ipStatistics.getIpStatisticsByDate(dayFrom, dateTo) + return result + + diff --git a/metrics-migrate-stats/Controller/identityProvidersMapController.py b/metrics-migrate-stats/Controller/identityProvidersMapController.py new file mode 100644 index 0000000..3e5fc09 --- /dev/null +++ b/metrics-migrate-stats/Controller/identityProvidersMapController.py @@ -0,0 +1,13 @@ +from datetime import date, timedelta +from Model.identityProvidersMapFromProxy import identityProvidersMapFromProxy +from Model.identityProvidersMap import identityProvidersMap +from datetime import datetime, timedelta +class identityProvidersMapController: + + @classmethod + def getAllData(self): + + result = identityProvidersMapFromProxy.getAllIdps() + return result + + diff --git a/metrics-migrate-stats/Controller/serviceProvidersMapController.py b/metrics-migrate-stats/Controller/serviceProvidersMapController.py new file mode 100644 index 0000000..7bc72c4 --- /dev/null +++ b/metrics-migrate-stats/Controller/serviceProvidersMapController.py @@ -0,0 +1,13 @@ +from datetime import date, timedelta +from Model.serviceProvidersMapFromProxy import serviceProvidersMapFromProxy +from Model.serviceProvidersMap import serviceProvidersMap +from datetime import datetime, timedelta +class serviceProvidersMapController: + + @classmethod + def getAllData(self): + + result = serviceProvidersMapFromProxy.getAllSps() + return result + + diff --git a/metrics-migrate-stats/Controller/statisticsCountryHashedController.py b/metrics-migrate-stats/Controller/statisticsCountryHashedController.py new file mode 100644 index 0000000..1f33751 --- /dev/null +++ b/metrics-migrate-stats/Controller/statisticsCountryHashedController.py @@ -0,0 +1,111 @@ +from datetime import date, timedelta +from multiprocessing.spawn import prepare +from Model.countries import countries +from Model.identityProvidersMap import identityProvidersMap +from Model.ipStatistics import ipStatistics +from Model.statisticsDetail import statisticsDetail +from Model.statistics import statistics +from Model.serviceProvidersMap import serviceProvidersMap +from Model.statisticsCountryHashed import statisticsCountryHashed +from Model.statisticsCountryHashedFromComanage import statisticsCountryHashedFromComanage +from datetime import datetime, timedelta +from Service.ipDatabase import geoip2Database +from Logger import log +import ipaddress + +class statisticsCountryHashedController: + logger = log.get_logger("statisticsCountryHashedController") + + @classmethod + def saveAllData(self, tenenv_id): + + dateFrom = statisticsCountryHashed.getLastDate(tenenv_id) + today = date.today() + dateTo = today.strftime('%Y-%m-%d 23:59:59') + # we dont have any country statistics saved + if dateFrom[0][0] == None: + result = ipStatistics.getStatsByDate(None, dateTo) + else: + dayAfter = dateFrom[0][0] + timedelta(days=1) + dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00') + + yesterday = date.today() - timedelta(days=1) + dateTo = yesterday.strftime('%Y-%m-%d 23:59:59') + + result = ipStatistics.getStatsByDate(dayFrom, dateTo) + statisticsCountryHashedController.prepareData(result, tenenv_id) + + # now we would like to save those data we dont have ip information + dateTo = statisticsCountryHashed.getFirstDate(tenenv_id) + result = statisticsDetail.getStatsByDate(dateTo[0][0].strftime('%Y-%m-%d')) + statisticsCountryHashedController.prepareData(result, tenenv_id) + dateTo = statisticsCountryHashed.getFirstDate(tenenv_id) + result = statistics.getStatsByDate(dateTo[0][0].strftime('%Y-%m-%d')) + statisticsCountryHashedController.prepareData(result, tenenv_id) + return result + + @classmethod + def prepareData(self, data, tenenv_id): + ipDatabaseHandler = geoip2Database() + mappedItems = 0 + for item in data: + ## find identityprovider id + result = identityProvidersMap.getIdpIdByIdentifier(item.sourceidp, tenenv_id) + if len(result) > 0: + #self.logger.info(result[0][0]) + idpId = result[0][0] + else: + self.logger.error("Identity provider identifier not found {0}".format(item.sourceidp)) + continue + + ## find serviceprovider id + result = serviceProvidersMap.getSpIdByIdentifier(item.service, tenenv_id) + if len(result) > 0: + #self.logger.info(result[0][0]) + spId = result[0][0] + else: + self.logger.warning("Service entityid {0} not found".format(item.service)) + spItem = serviceProvidersMap(item.service, item.service.replace("'", "''"), tenenv_id) + serviceProvidersMap.save(spItem) + self.logger.info("Service entityid {0} created".format(item.service)) + try: + if hasattr(item, "ip"): + ipaddr = ipaddress.ip_network(item.ip).network_address + else: + item.countrycode = countryData[0] + item.country = countryData[1] + # get country code/ name + countryData = ipDatabaseHandler.getCountryFromIp(str(ipaddr), item.ipVersion) + if (countryData[0] == None): + item.countrycode = "UN" + item.country = "Unknown" + else: + item.countrycode = countryData[0] + item.country = countryData[1] + except: + item.countrycode = "UN" + item.country = "Unknown" + ## save country if not exists and get id + country = countries(item.countrycode, item.country) + countries.save(country) + result = countries.getIdByCountryCode(item.countrycode) + if len(result) > 0: + #self.logger.info(result[0][0]) + countryId = result[0][0] + else: + self.logger.error("Country not found") + continue + self.logger.info("{0} ".format(item.date)) + + if hasattr(item, "count"): + count = item.count + else: + count = 1 + statsCountry = statisticsCountryHashed(item.date, item.hasheduserid, idpId, spId, countryId, count, tenenv_id) + statisticsCountryHashed.save(statsCountry) + mappedItems += 1 + self.logger.info("{0} Country Stats created".format(mappedItems)) + + + + diff --git a/metrics-migrate-stats/Controller/usersController.py b/metrics-migrate-stats/Controller/usersController.py new file mode 100644 index 0000000..ec096b8 --- /dev/null +++ b/metrics-migrate-stats/Controller/usersController.py @@ -0,0 +1,27 @@ +from ast import Or +from datetime import date, timedelta +from Model.users import users +from Model.usersFromComanage import usersFromComanage +from Model.vos import vos +from datetime import datetime, timedelta +from Logger import log + +class usersController: + logger = log.get_logger("usersController") + + @classmethod + def saveUsers(self, tenenv_id): + + usersList = [] + # get memberships with active status + usersComanage = usersFromComanage.getAllUsers() + # save memberships with active status + for item in usersComanage: + usersItem = users(item.hasheduserid, item.created, item.status, tenenv_id) + usersList.append(usersItem) + # save data to tables if any + if usersList: + users.saveAll(usersList) + + + diff --git a/metrics-migrate-stats/Controller/voMembershipsController.py b/metrics-migrate-stats/Controller/voMembershipsController.py new file mode 100644 index 0000000..6a51496 --- /dev/null +++ b/metrics-migrate-stats/Controller/voMembershipsController.py @@ -0,0 +1,81 @@ +from ast import Or +from datetime import date, timedelta +from Model.voMemberships import voMemberships +from Model.voMembershipsFromComanage import voMembershipsFromComanage +from Model.vos import vos +from datetime import datetime, timedelta +from Logger import log + +class voMembershipsController: + logger = log.get_logger("voMembershipsController") + + @classmethod + def getVoId(self, voName, source, tenenv_id): + result = vos.getVoIdFromVoName(voName, source, tenenv_id) + return result + + @classmethod + def getDataNotMapped(self, tenenv_id): + voMemberships.truncate(tenenv_id) + + voMembershipsList = [] + # dateFrom = vos.getLastDate() + + # we dont have any vos saved + # if dateFrom[0][0] == None: + + # get memberships with active status + activeMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('A') + # save memberships with active status + for item in activeMemberships: + result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id) + + if len(result) > 0: + voId = result[0][0] + self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0])) + + voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id) + voMembershipsList.append(voMembershipsItem) + # save data to tables if any + if voMembershipsList: + + voMemberships.saveAll(voMembershipsList) + + # get memberships with grace period + # save only if there isnt any existing membership or status membership is not Active/ Grace Period + gracePeriodVoMembershipsList = [] + graceperiodMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('GP') + for item in graceperiodMemberships: + result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id) + if len(result) > 0: + voId = result[0][0] + self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0])) + #check if already there is a membership with active status saved + statusMembership = voMemberships.getMembershipStatus(voId, item.hasheduserid, tenenv_id) + if len(statusMembership) > 0 and (statusMembership[0][0]=='A' or statusMembership[0][0]=='GP'): + continue + voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id) + gracePeriodVoMembershipsList.append(voMembershipsItem) + if len(gracePeriodVoMembershipsList)>0: + voMemberships.saveAll(gracePeriodVoMembershipsList) + + # get memberships with other statuses + # save only if there isnt any existing membership + otherStatusVoMembershipsList = [] + otherStatusMemberships = voMembershipsFromComanage.getAllVoMembershipsByStatus('Other') + for item in otherStatusMemberships: + result = voMembershipsController.getVoId(item.voName, "registry", tenenv_id) + if len(result) > 0: + voId = result[0][0] + self.logger.info("Vo name {0} with id {1}".format(item.voName, result[0][0])) + #check if already there is a membership with active status saved + statusMembership = voMemberships.getMembershipStatus(voId, item.hasheduserid, tenenv_id) + if len(statusMembership) > 0: + continue + voMembershipsItem = voMemberships(voId, item.hasheduserid, item.status, item.created, tenenv_id) + otherStatusVoMembershipsList.append(voMembershipsItem) + if len(otherStatusVoMembershipsList)>0: + voMemberships.saveAll(otherStatusVoMembershipsList) + return result + + diff --git a/metrics-migrate-stats/Controller/vosController.py b/metrics-migrate-stats/Controller/vosController.py new file mode 100644 index 0000000..aef392e --- /dev/null +++ b/metrics-migrate-stats/Controller/vosController.py @@ -0,0 +1,24 @@ +from datetime import date, timedelta +from Model.vosFromComanage import vosFromComanage +from Model.vos import vos +from datetime import datetime, timedelta + +class vosController: + @classmethod + def getDataNotMapped(self, tenenv_id): + dateFrom = vos.getLastDate(tenenv_id) + + # we dont have any vos saved + if dateFrom[0][0] == None: + result = vosFromComanage.getAllVos() + else: + dayAfter = dateFrom[0][0] + timedelta(days=1) + dayFrom = dayAfter.strftime('%Y-%m-%d 00:00:00') + + yesterday = date.today() - timedelta(days=1) + dateTo = yesterday.strftime('%Y-%m-%d 23:59:59') + + result = vosFromComanage.getVosByDate(dayFrom, dateTo) + return result + + diff --git a/metrics-migrate-stats/Logger/log.py b/metrics-migrate-stats/Logger/log.py new file mode 100644 index 0000000..fdce677 --- /dev/null +++ b/metrics-migrate-stats/Logger/log.py @@ -0,0 +1,29 @@ +import logging +import sys +from Utils import configParser +from logging.handlers import TimedRotatingFileHandler + +FORMATTER = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") +LOG_FILE = "{0}/{1}".format(configParser.getConfig('logging')['folder'] ,configParser.getConfig('logging')['file']) +LEVEL = configParser.getConfig('logging')['level'] + +def get_console_handler(): + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(FORMATTER) + return console_handler + +def get_file_handler(): + file_handler = TimedRotatingFileHandler(LOG_FILE, when='midnight') + file_handler.setFormatter(FORMATTER) + return file_handler + +def get_logger(logger_name): + + logger = logging.getLogger(logger_name) + logger.setLevel(LEVEL) + logger.addHandler(get_console_handler()) + logger.addHandler(get_file_handler()) + # with this pattern, it's rarely necessary to propagate the error up to parent + logger.propagate = False + + return logger \ No newline at end of file diff --git a/metrics-migrate-stats/Model/countries.py b/metrics-migrate-stats/Model/countries.py new file mode 100644 index 0000000..ca9330e --- /dev/null +++ b/metrics-migrate-stats/Model/countries.py @@ -0,0 +1,27 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +from Logger import log +class countries(object): + logger = log.get_logger("countries") + COUNTRIESTABLE = configParser.getConfig('destination_tables')['countries'] + + def __init__(self, countrycode, country): + self.countrycode = countrycode + self.country = country + + + @classmethod + def save(self, countries): + pgConn = destinationPgConnector() + pgConn.execute_and_commit( + "INSERT INTO {0}(countrycode, country) VALUES ('{1}', '{2}') ON CONFLICT DO NOTHING".format(countries.COUNTRIESTABLE, countries.countrycode, countries.country) + ) + + @classmethod + def getIdByCountryCode(self, countryCode): + pgConn = destinationPgConnector() + result = pgConn.execute_select( + "SELECT id FROM {0} WHERE countrycode='{1}'".format(countries.COUNTRIESTABLE, countryCode) + ) + return result + diff --git a/metrics-migrate-stats/Model/countryStatistics.py b/metrics-migrate-stats/Model/countryStatistics.py new file mode 100644 index 0000000..f66d6fd --- /dev/null +++ b/metrics-migrate-stats/Model/countryStatistics.py @@ -0,0 +1,34 @@ +# from Model.pgConnector import destinationPgConnector +# from Utils import configParser +# class countryStatistics(object): +# STATISTICSTABLE = configParser.getConfig('destination_tables')['countries'] + +# def __init__(self, id, date, sourceIdp, service, countryCode, country, count): +# self.id = id +# self.date = date +# self.service = service +# self.sourceIdp = sourceIdp +# self.countrycode = countryCode +# self.country = country +# self.count = count + +# @classmethod +# def getLastDate(self): +# pgConn = destinationPgConnector() +# result = pgConn.execute_select("SELECT max(date::date) FROM {0}".format(countryStatistics.STATISTICSTABLE)) +# return result + +# @classmethod +# def save(self, countryStatistics): +# pgConn = destinationPgConnector() +# pgConn.execute_and_commit( +# "INSERT INTO {0}(date, sourceidp, service, countrycode, country, count) VALUES ('{1}', '{2}', '{3}', '{4}', '{5}', {6}) ON CONFLICT (date, sourceidp, service, countrycode) DO UPDATE SET count = {0}.count + 1".format(countryStatistics.STATISTICSTABLE, countryStatistics.date, countryStatistics.sourceIdp, countryStatistics.service, countryStatistics.countrycode, countryStatistics.country, 1) +# ) + +# @classmethod +# def saveAll(self, countryStatisticsList): +# pgConn = destinationPgConnector() +# values = '' +# for item in countryStatisticsList: +# values += "INSERT INTO {0}(date, sourceidp, service, countrycode, country, count) VALUES ('{1}', '{2}', '{3}', '{4}', '{5}', {6}) ON CONFLICT (date, sourceidp, service, countrycode) DO UPDATE SET count = {0}.count + 1;".format(countryStatistics.STATISTICSTABLE, item.date, item.sourceIdp, item.service, item.countrycode, item.country, 1) +# pgConn.execute_and_commit(values) diff --git a/metrics-migrate-stats/Model/countryStatisticsHashedUserId.py b/metrics-migrate-stats/Model/countryStatisticsHashedUserId.py new file mode 100644 index 0000000..1d047ab --- /dev/null +++ b/metrics-migrate-stats/Model/countryStatisticsHashedUserId.py @@ -0,0 +1,37 @@ +# from Model.pgConnector import destinationPgConnector +# from Utils import configParser +# import hashlib + +# class countryStatisticsHashedUserId(object): +# STATISTICSHASHEDTABLE = configParser.getConfig('destination_tables')['country_hashed_table'] + +# def __init__(self, id, date, hashedUserId, sourceIdp, service, countryCode, country,count): +# self.id = id +# self.date = date +# self.hashedUserId = hashedUserId +# self.sourceIdp = sourceIdp +# self.service = service +# self.countrycode = countryCode +# self.country = country +# self.count = count + +# @classmethod +# def getLastDate(self): +# pgConn = destinationPgConnector() +# result = pgConn.execute_select("SELECT max(date::date) FROM {0}".format(countryStatisticsHashedUserId.STATISTICSHASHEDTABLE)) +# return result + +# @classmethod +# def save(self, countryStatisticsHashedUserId): +# pgConn = destinationPgConnector() +# pgConn.execute_and_commit( +# "INSERT INTO {0}(date, hasheduserid, sourceidp, service, countrycode, country, count) VALUES ('{1}', '{2}', '{3}', '{4}', '{5}', '{6}', {7}) ON CONFLICT (date, hasheduserid, sourceidp, service, countrycode) DO UPDATE SET count = {0}.count + 1".format(countryStatisticsHashedUserId.STATISTICSHASHEDTABLE, countryStatisticsHashedUserId.date, hashlib.md5(countryStatisticsHashedUserId.hashedUserId.encode()).hexdigest(), countryStatisticsHashedUserId.sourceIdp, countryStatisticsHashedUserId.service, countryStatisticsHashedUserId.countrycode, countryStatisticsHashedUserId.country, 1) +# ) + +# @classmethod +# def saveAll(self, countryStatisticsList): +# pgConn = destinationPgConnector() +# values = '' +# for item in countryStatisticsList: +# values += "INSERT INTO {0}(date, hasheduserid, sourceidp, service, countrycode, country, count) VALUES ('{1}', '{2}', '{3}', '{4}', '{5}', '{6}', {7}) ON CONFLICT (date, hasheduserid, sourceidp, service, countrycode) DO UPDATE SET count = {0}.count + 1;".format(countryStatisticsHashedUserId.STATISTICSHASHEDTABLE, item.date, hashlib.md5(item.hashedUserId.encode()).hexdigest(), item.sourceIdp, item.service, item.countrycode, item.country, 1) +# pgConn.execute_and_commit(values) diff --git a/metrics-migrate-stats/Model/environments.py b/metrics-migrate-stats/Model/environments.py new file mode 100644 index 0000000..4afbb77 --- /dev/null +++ b/metrics-migrate-stats/Model/environments.py @@ -0,0 +1,30 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +from Logger import log +class environments(object): + logger = log.get_logger("environments") + ENVIRONMENTSTABLE = configParser.getConfig('destination_tables')['environments'] + + def __init__(self, name, description): + self.name = name + self.description = description + + @classmethod + def save(self, environment): + pgConn = destinationPgConnector() + pgConn.execute_and_commit( + """ + INSERT INTO {0}(name, description) + VALUES ('{1}', '{2}') + ON CONFLICT DO NOTHING + """.format(environments.ENVIRONMENTSTABLE, + environment.name, environment.description) + ) + + @classmethod + def getEnvironmentByName(self, name): + pgConn = destinationPgConnector() + result = pgConn.execute_select(""" + SELECT id FROM {0} WHERE name='{1}'""".format( + environments.ENVIRONMENTSTABLE, name)) + return result diff --git a/metrics-migrate-stats/Model/identityProvidersMap.py b/metrics-migrate-stats/Model/identityProvidersMap.py new file mode 100644 index 0000000..1ed3e67 --- /dev/null +++ b/metrics-migrate-stats/Model/identityProvidersMap.py @@ -0,0 +1,35 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +from Logger import log +class identityProvidersMap(object): + logger = log.get_logger("identityProvidersMap") + IDENTITYPROVIDERSMAPTABLE = configParser.getConfig('destination_tables')['identity_providers_map'] + + def __init__(self, entityid, name, tenenv_id): + self.entityid = entityid + self.name = name + self.tenenv_id = tenenv_id + + def __str__(self): + return f"IdentityProvidersMap(entityid={self.entityid}, name={self.name}, tenenv_id={self.tenenv_id})" + + def __getitem__(self, key): + return getattr(self, key, None) + + @classmethod + def save(self, identityProviderMap): + pgConn = destinationPgConnector() + pgConn.execute_and_commit( + f""" + INSERT INTO {identityProviderMap.IDENTITYPROVIDERSMAPTABLE} (entityid, name, tenenv_id) + VALUES (%(entityid)s, %(name)s, %(tenenv_id)s) + ON CONFLICT DO NOTHING; + """, identityProviderMap) + + @classmethod + def getIdpIdByIdentifier(self, entityid, tenenv_id): + pgConn = destinationPgConnector() + result = pgConn.execute_select(""" + SELECT id FROM {0} WHERE entityid='{1}' AND tenenv_id={2}""".format( + identityProvidersMap.IDENTITYPROVIDERSMAPTABLE, entityid, tenenv_id)) + return result diff --git a/metrics-migrate-stats/Model/identityProvidersMapFromProxy.py b/metrics-migrate-stats/Model/identityProvidersMapFromProxy.py new file mode 100644 index 0000000..e890b65 --- /dev/null +++ b/metrics-migrate-stats/Model/identityProvidersMapFromProxy.py @@ -0,0 +1,20 @@ +from Model.pgConnector import sourcePgConnectorProxy +from Utils import configParser + +class identityProvidersMapFromProxy(object): + IDPPROXYTABLE = configParser.getConfig('source_tables')['identity_providers_map_proxy'] + def __init__(self, entityid, name): + self.entityid = entityid + self.name = name + + @classmethod + def getAllIdps(self): + pgConn = sourcePgConnectorProxy() + result = list(pgConn.execute_select("SELECT entityid, name FROM {0}".format(identityProvidersMapFromProxy.IDPPROXYTABLE))) + data = [] + for row in result: + idpsData = identityProvidersMapFromProxy(row[0], row[1]) + data.append(idpsData) + return data + + diff --git a/metrics-migrate-stats/Model/ipStatistics.py b/metrics-migrate-stats/Model/ipStatistics.py new file mode 100644 index 0000000..7b49b8f --- /dev/null +++ b/metrics-migrate-stats/Model/ipStatistics.py @@ -0,0 +1,58 @@ +from Model.pgConnector import sourcePgConnectorProxy +from Utils import configParser +from datetime import date, timedelta +import hashlib +class ipStatistics(object): + IPSTATISTICSTABLE = configParser.getConfig('source_tables')['ip_table'] + def __init__(self, accessed, sourceIdp, service, userid, ip, ipVersion): + self.date = accessed + self.sourceidp = sourceIdp + self.service = service + self.hasheduserid = userid + self.ip = ip + self.ipVersion = ipVersion + + @classmethod + def getIpStatisticsByDate(self, dateFrom, dateTo): + pgConn = sourcePgConnectorProxy() + if(dateFrom != None): + result = list(pgConn.execute_select("SELECT accessed::date, sourceidp, service, userid, ip, ipversion FROM {0} WHERE accessed::date BETWEEN '{1}' AND '{2}'".format(ipStatistics.IPSTATISTICSTABLE, dateFrom, dateTo))) + else: + result = list(pgConn.execute_select("SELECT accessed::date, sourceidp, service, userid, ip, ipversion FROM {0} WHERE accessed::date <= '{1}'".format(ipStatistics.IPSTATISTICSTABLE, dateTo))) + data = [] + for row in result: + ipData = ipStatistics(row[0], row[1], row[2], row[3], row[4], row[5]) + data.append(ipData) + return data + + @classmethod + def getAllIpStatistics(self): + yesterday = date.today() - timedelta(days=1) + dateTo = yesterday.strftime('%Y-%m-%d 23:59:59') + pgConn = sourcePgConnectorProxy() + result = list(pgConn.execute_select("SELECT accessed::date, sourceidp, service, userid, ip, ipversion FROM {0} WHERE accessed::date <= '{1}'".format(ipStatistics.IPSTATISTICSTABLE, dateTo))) + data = [] + for row in result: + ipData = ipStatistics(row[0], row[1], row[2], row[3], row[4], row[5]) + data.append(ipData) + return data + + @classmethod + def getStatsByDate(self, dateFrom, dateTo ): + pgConn = sourcePgConnectorProxy() + if(dateFrom != None): + result = list(pgConn.execute_select("""SELECT accessed::date, sourceidp, service, userid, ip, ipversion + FROM {0} WHERE accessed::date BETWEEN '{1}' + AND '{2}'""".format(ipStatistics.IPSTATISTICSTABLE, dateFrom, dateTo))) + + else: + result = list(pgConn.execute_select("""SELECT accessed::date, sourceidp, service, userid, ip, ipversion + FROM {0} WHERE accessed::date <= '{1}'""" + .format(ipStatistics.IPSTATISTICSTABLE, dateTo))) + data = [] + for row in result: + statsData = ipStatistics(row[0], row[1], row[2], hashlib.md5(row[3].encode()).hexdigest(), row[4], row[5]) + data.append(statsData) + return data + + diff --git a/metrics-migrate-stats/Model/pgConnector.py b/metrics-migrate-stats/Model/pgConnector.py new file mode 100644 index 0000000..c98c957 --- /dev/null +++ b/metrics-migrate-stats/Model/pgConnector.py @@ -0,0 +1,133 @@ +from configparser import RawConfigParser +import sys +import psycopg2 +# import the error handling libraries for psycopg2 +from psycopg2 import OperationalError, errorcodes, errors +from Logger import log + +def singleton(theClass): + """ decorator for a class to make a singleton out of it """ + classInstances = {} + + def getInstance(*args, **kwargs): + """ creating or just return the one and only class instance. + The singleton depends on the parameters used in __init__ """ + key = (theClass, args, str(kwargs)) + if key not in classInstances: + classInstances[key] = theClass(*args, **kwargs) + return classInstances[key] + + return getInstance + +class pgConnector: + logger = log.get_logger("pgConnector") + conn = None + + def __init__(self, filename = "config.py", section = "source_database"): + + self.filename = filename + self.section = section + self.params = self.config(filename, section) + if self.conn == None: + try: + self.logger.debug('Connecting to the PostgreSQL database...') + self.conn = psycopg2.connect(**self.params) + self.set_utc_timezone() + except psycopg2.OperationalError as err: + self.logger.error(str(err).strip()) + sys.exit(1) + + def config(self, filename='config.py', section='source_database'): + + # create a parser + parser = RawConfigParser() + + # read config file + parser.read(filename) + + # get section, default to source_database + db = {} + + if parser.has_section(section): + params = parser.items(section) + for param in params: + db[param[0]] = param[1] + else: + self.logger.error('Section {0} not found in the {1} file'.format(section, filename)) + raise Exception('Section {0} not found in the {1} file'.format(section, filename)) + + return db + + def execute_select(self, query): + + # create a cursor + cur = self.conn.cursor() + + # execute a statement + cur.execute(query) + + return cur.fetchall() + + def execute_and_commit_with_fetch(self, query): + + try: + cur = self.conn.cursor() + cur.execute(query) + self.conn.commit() + id = cur.fetchone() + if (id is not None): + self.logger.info("{0}".format(id[0])) + return id[0] + else : + return None + except Exception as err: + self.logger.error(str(err).strip()) + sys.exit(1) + + def execute_and_commit(self, query, data=None): + + try: + cur = self.conn.cursor() + if (data is None): + cur.execute(query) + else: + self.logger.info(str(data)) + cur.execute(query, data) + self.conn.commit() + except Exception as err: + self.logger.error(str(err).strip()) + sys.exit(1) + + def close(self): + + self.conn.close() + self.logger.debug('Database connection closed.') + + def set_utc_timezone(self): + try: + self.logger.debug('Setting timezone to UTC...') + with self.conn.cursor() as cur: + cur.execute("SET timezone TO 'UTC'") + self.conn.commit() + self.logger.debug('Timezone set to UTC.') + except Exception as err: + self.logger.error('Error setting timezone to UTC: {}'.format(str(err).strip())) + sys.exit(1) + +# Subclass of pgConnector +@singleton +class sourcePgConnector(pgConnector): + def __init__(self, filename = "config.py", section = "source_database"): + super().__init__(filename, section) + +# Subclass of pgConnector +@singleton +class sourcePgConnectorProxy(pgConnector): + def __init__(self, filename = "config.py", section = "source_database_proxy"): + super().__init__(filename, section) + +# Subclass of pgConnector +@singleton +class destinationPgConnector(pgConnector): + def __init__(self, filename = "config.py", section = "destination_database"): + super().__init__(filename, section) \ No newline at end of file diff --git a/metrics-migrate-stats/Model/serviceProvidersMap.py b/metrics-migrate-stats/Model/serviceProvidersMap.py new file mode 100644 index 0000000..a3b6606 --- /dev/null +++ b/metrics-migrate-stats/Model/serviceProvidersMap.py @@ -0,0 +1,32 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +from Logger import log +class serviceProvidersMap(object): + logger = log.get_logger("serviceProvidersMap") + SERVICEPROVIDERSMAPTABLE = configParser.getConfig('destination_tables')['service_providers_map'] + + def __init__(self, identifier, name, tenenv_id): + self.identifier = identifier + self.name = name + self.tenenv_id = tenenv_id + + + @classmethod + def save(self, serviceProviderMap): + pgConn = destinationPgConnector() + pgConn.execute_and_commit( + """INSERT INTO {0}(identifier, name, tenenv_id) + VALUES ('{1}', '{2}', {3}) ON CONFLICT DO NOTHING + """.format(serviceProviderMap.SERVICEPROVIDERSMAPTABLE, + serviceProviderMap.identifier, serviceProviderMap.name, + serviceProviderMap.tenenv_id) + ) + + @classmethod + def getSpIdByIdentifier(self, identifier, tenenv_id): + pgConn = destinationPgConnector() + result = pgConn.execute_select(""" + SELECT id FROM {0} WHERE identifier='{1}' + AND tenenv_id={2} + """.format(serviceProvidersMap.SERVICEPROVIDERSMAPTABLE, identifier, tenenv_id)) + return result diff --git a/metrics-migrate-stats/Model/serviceProvidersMapFromProxy.py b/metrics-migrate-stats/Model/serviceProvidersMapFromProxy.py new file mode 100644 index 0000000..68450cc --- /dev/null +++ b/metrics-migrate-stats/Model/serviceProvidersMapFromProxy.py @@ -0,0 +1,20 @@ +from Model.pgConnector import sourcePgConnectorProxy +from Utils import configParser + +class serviceProvidersMapFromProxy(object): + SPPROXYTABLE = configParser.getConfig('source_tables')['service_providers_map_proxy'] + def __init__(self, identifier, name): + self.identifier = identifier + self.name = name + + @classmethod + def getAllSps(self): + pgConn = sourcePgConnectorProxy() + result = list(pgConn.execute_select("SELECT identifier, name FROM {0}".format(serviceProvidersMapFromProxy.SPPROXYTABLE))) + data = [] + for row in result: + spsData = serviceProvidersMapFromProxy(row[0], row[1]) + data.append(spsData) + return data + + diff --git a/metrics-migrate-stats/Model/statistics.py b/metrics-migrate-stats/Model/statistics.py new file mode 100644 index 0000000..0781265 --- /dev/null +++ b/metrics-migrate-stats/Model/statistics.py @@ -0,0 +1,25 @@ +from Model.pgConnector import sourcePgConnectorProxy +from Utils import configParser +from datetime import date, timedelta +import hashlib +class statistics(object): + STATISTICSTABLE = configParser.getConfig('source_tables')['statistics'] + def __init__(self, date, sourceIdp, service, count): + self.date = date + self.sourceidp = sourceIdp + self.service = service + self.hasheduserid = "unknown" + self.count = count + @classmethod + def getStatsByDate(self, dateTo): + pgConn = sourcePgConnectorProxy() + print(dateTo) + result = list(pgConn.execute_select("SELECT year,month,day, sourceidp, service, count FROM {0} WHERE CAST(CONCAT(year,'-',LPAD(CAST(month AS varchar),2,'0'),'-',LPAD(CAST(day AS varchar),2,'0')) AS date) < '{1}' ".format(statistics.STATISTICSTABLE, dateTo))) + data = [] + for row in result: + dateCol = [row[0], row[1], row[2]] + ipData = statistics('-'.join(map(str, dateCol)), row[3], row[4], row[5]) + data.append(ipData) + return data + + \ No newline at end of file diff --git a/metrics-migrate-stats/Model/statisticsCountryHashed.py b/metrics-migrate-stats/Model/statisticsCountryHashed.py new file mode 100644 index 0000000..9503c31 --- /dev/null +++ b/metrics-migrate-stats/Model/statisticsCountryHashed.py @@ -0,0 +1,66 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +from Logger import log +class statisticsCountryHashed(object): + logger = log.get_logger("statisticsCountryHashed") + STATISTICSCOUNTRYHASHEDTABLE = configParser.getConfig('destination_tables')['statistics_country_hashed'] + + def __init__(self, date, hasheduserid, sourceidpid, serviceid, countryid, count, tenenv_id): + self.date = date + self.hasheduserid = hasheduserid + self.sourceidpid = sourceidpid + self.serviceid = serviceid + self.countryid = countryid + self.count = count + self.tenenv_id = tenenv_id + + @classmethod + def getLastDate(self, tenenv_id): + pgConn = destinationPgConnector() + result = pgConn.execute_select(""" + SELECT max(date::date) FROM {0} + WHERE tenenv_id={1} + """.format(statisticsCountryHashed.STATISTICSCOUNTRYHASHEDTABLE, tenenv_id)) + return result + + @classmethod + def getFirstDate(self, tenenv_id): + pgConn = destinationPgConnector() + result = pgConn.execute_select(""" + SELECT min(date::date) FROM {0} + WHERE tenenv_id={1} + """.format(statisticsCountryHashed.STATISTICSCOUNTRYHASHEDTABLE, tenenv_id)) + return result + + @classmethod + def save(self, statisticsCountryHashed): + pgConn = destinationPgConnector() + pgConn.execute_and_commit( + """INSERT INTO {0}(date, hasheduserid, sourceidpid, serviceid, countryid, count, tenenv_id) + VALUES ('{1}', '{2}', {3}, {4}, {5}, {6}, {7}) ON CONFLICT (date, hasheduserid, sourceidpid, serviceid, countryid, tenenv_id) + DO UPDATE SET count = {0}.count + 1 + """.format(statisticsCountryHashed.STATISTICSCOUNTRYHASHEDTABLE, + statisticsCountryHashed.date, + statisticsCountryHashed.hasheduserid, + statisticsCountryHashed.sourceidpid, + statisticsCountryHashed.serviceid, + statisticsCountryHashed.countryid, + statisticsCountryHashed.count, + statisticsCountryHashed.tenenv_id + ) + ) + + @classmethod + def getStatsByDate(self, sourceidp, service, dateFrom, dateTo, tenenv_id): + pgConn = destinationPgConnector() + + result = list(pgConn.execute_select("""SELECT date::date, hasheduserid, sourceidp, service, count + FROM {0} WHERE date::date BETWEEN '{1}' AND '{2}' + AND sourceidp='{2}' AND service='{3}' AND tenenv_id={4} + group by date::date, hasheduserid, sourceidp,service,count""" + .format(statisticsCountryHashed.STATISTICSCOUNTRYHASHEDTABLE, dateFrom, dateTo, sourceidp, service, tenenv_id))) + data = [] + for row in result: + statsData = statisticsCountryHashed(row[0], row[1], row[2], row[3], "N", row[4], 0) + data.append(statsData) + return data diff --git a/metrics-migrate-stats/Model/statisticsCountryHashedFromComanage.py b/metrics-migrate-stats/Model/statisticsCountryHashedFromComanage.py new file mode 100644 index 0000000..0a50a9e --- /dev/null +++ b/metrics-migrate-stats/Model/statisticsCountryHashedFromComanage.py @@ -0,0 +1,30 @@ +from Model.pgConnector import sourcePgConnector +from Utils import configParser +from datetime import date, timedelta + +class statisticsCountryHashedFromComanage(object): + STATISTICSCOUNTRYHASHEDCOMANAGETABLE = configParser.getConfig('source_tables')['statistics_country_hashed_comanage'] + def __init__(self, date, hasheduserid, sourceidp, service, countrycode, country, count): + self.hasheduserid = hasheduserid + self.date = date + self.sourceidp = sourceidp + self.service = service + self.countrycode = countrycode + self.country = country + self.count = count + + @classmethod + def getStatsByDate(self, dateFrom, dateTo ): + pgConn = sourcePgConnector() + if(dateFrom != None): + result = list(pgConn.execute_select("SELECT date::date, hasheduserid, sourceidp, service, countrycode, country, count FROM {0} WHERE date::date BETWEEN '{1}' AND '{2}'".format(statisticsCountryHashedFromComanage.STATISTICSCOUNTRYHASHEDCOMANAGETABLE, dateFrom, dateTo))) + + else: + result = list(pgConn.execute_select("SELECT date::date, hasheduserid, sourceidp, service, countrycode, country, count FROM {0} WHERE date::date <= '{1}'".format(statisticsCountryHashedFromComanage.STATISTICSCOUNTRYHASHEDCOMANAGETABLE, dateTo))) + data = [] + for row in result: + statsData = statisticsCountryHashedFromComanage(row[0], row[1], row[2], row[3], row[4], row[5], row[6]) + data.append(statsData) + return data + + diff --git a/metrics-migrate-stats/Model/statisticsDetail.py b/metrics-migrate-stats/Model/statisticsDetail.py new file mode 100644 index 0000000..0363553 --- /dev/null +++ b/metrics-migrate-stats/Model/statisticsDetail.py @@ -0,0 +1,24 @@ +from Model.pgConnector import sourcePgConnectorProxy +from Utils import configParser +from datetime import date, timedelta +import hashlib +class statisticsDetail(object): + STATISTICSDETAILTABLE = configParser.getConfig('source_tables')['statistics_detail'] + def __init__(self, date, sourceIdp, service, userid, count): + self.date = date + self.sourceidp = sourceIdp + self.service = service + self.hasheduserid = userid + self.count = count + @classmethod + def getStatsByDate(self, dateTo): + pgConn = sourcePgConnectorProxy() + print(dateTo) + result = list(pgConn.execute_select("SELECT year,month,day, sourceidp, service, userid FROM {0} WHERE CAST(CONCAT(year,'-',LPAD(CAST(month AS varchar),2,'0'),'-',LPAD(CAST(day AS varchar),2,'0')) AS date) < '{1}' ".format(statisticsDetail.STATISTICSDETAILTABLE, dateTo))) + data = [] + for row in result: + ipData = statisticsDetail(row[0]+'-'+row[1]+'-'+row[2], row[3], row[4], row[5]) + data.append(ipData) + return data + + \ No newline at end of file diff --git a/metrics-migrate-stats/Model/tenants.py b/metrics-migrate-stats/Model/tenants.py new file mode 100644 index 0000000..fd745b8 --- /dev/null +++ b/metrics-migrate-stats/Model/tenants.py @@ -0,0 +1,30 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +from Logger import log +class tenants(object): + logger = log.get_logger("tenants") + TENANTSTABLE = configParser.getConfig('destination_tables')['tenants'] + + def __init__(self, name, description): + self.name = name + self.description = description + + @classmethod + def save(self, tenant): + pgConn = destinationPgConnector() + pgConn.execute_and_commit( + """ + INSERT INTO {0}(name, description) + VALUES ('{1}', '{2}') + ON CONFLICT DO NOTHING + """.format(tenants.TENANTSTABLE, + tenant.name, tenant.description) + ) + + @classmethod + def getTenantByName(self, name): + pgConn = destinationPgConnector() + result = pgConn.execute_select(""" + SELECT id FROM {0} WHERE name='{1}'""".format( + tenants.TENANTSTABLE, name)) + return result diff --git a/metrics-migrate-stats/Model/tenenvs.py b/metrics-migrate-stats/Model/tenenvs.py new file mode 100644 index 0000000..43417a0 --- /dev/null +++ b/metrics-migrate-stats/Model/tenenvs.py @@ -0,0 +1,32 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +from Logger import log +class tenenvs(object): + logger = log.get_logger("tenenvs") + TENANTSTABLE = configParser.getConfig('destination_tables')['tenenvs'] + + def __init__(self, name, description, tenant_id, env_id): + self.name = name + self.description = description + self.tenant_id = tenant_id + self.env_id = env_id + + @classmethod + def save(self, tenenv): + pgConn = destinationPgConnector() + pgConn.execute_and_commit( + """ + INSERT INTO {0}(name, description, tenant_id, env_id) + VALUES ('{1}', '{2}', {3}, {4}) + ON CONFLICT DO NOTHING + """.format(tenenvs.TENANTSTABLE, + tenenv.name, tenenv.description, tenenv.tenant_id, tenenv.env_id) + ) + + @classmethod + def getTenenvByTenantIdAndEnvId(self, tenant_id, env_id): + pgConn = destinationPgConnector() + result = pgConn.execute_select(""" + SELECT id FROM {0} WHERE tenant_id={1} AND env_id={2}""".format( + tenenvs.TENANTSTABLE, tenant_id, env_id)) + return result diff --git a/metrics-migrate-stats/Model/unknownCountriesLogins.py b/metrics-migrate-stats/Model/unknownCountriesLogins.py new file mode 100644 index 0000000..964b5ad --- /dev/null +++ b/metrics-migrate-stats/Model/unknownCountriesLogins.py @@ -0,0 +1,29 @@ +from Model.pgConnector import sourcePgConnectorProxy +from Model.statisticsCountryHashedFromComanage import statisticsCountryHashedFromComanage +from Utils import configParser +from datetime import date, timedelta +import hashlib + +class unknownCoutriesLogins(object): + IPTABLE = configParser.getConfig('source_tables')['ip_table'] + def __init__(self, date, userid, sourceidp, service, count): + self.userid = userid + self.accessed = date + self.sourceidp = sourceidp + self.service = service + self.count = count + + @classmethod + def getStatsByDate(self, sourceidp, service, date): + pgConn = sourcePgConnectorProxy() + + result = list(pgConn.execute_select("SELECT accessed::date, userid, sourceidp, service, count(*) as count FROM {0} WHERE accessed::date='{1}' AND sourceidp='{2}' AND service='{3}' group by accessed::date, userid,sourceidp,service" + .format(unknownCoutriesLogins.IPTABLE, date, sourceidp, service))) + data = [] + for row in result: + # statsData = unknownCoutriesLogins(row[0], hashlib.md5(row[1].encode()).hexdigest(), row[2], row[3], row[4]) + data[hashlib.md5(row[1].encode()).hexdigest()] = row[4] + # data.append(statsData) + return data + + diff --git a/metrics-migrate-stats/Model/userCountryStatistics.py b/metrics-migrate-stats/Model/userCountryStatistics.py new file mode 100644 index 0000000..41bab6f --- /dev/null +++ b/metrics-migrate-stats/Model/userCountryStatistics.py @@ -0,0 +1,33 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +class userCountryStatistics(object): + USERCOUNTRYTABLE = configParser.getConfig('destination_tables')['user_country_table'] + + def __init__(self, id, date, userid, countrycode, country, count): + self.id = id + self.date = date + self.userid = userid + self.countrycode = countrycode + self.country = country + self.count = count + + @classmethod + def getLastDate(self): + pgConn = destinationPgConnector() + result = pgConn.execute_select("SELECT max(date::date) FROM {0}".format(userCountryStatistics.USERCOUNTRYTABLE)) + return result + + @classmethod + def save(self, userCountryStatistics): + pgConn = destinationPgConnector() + pgConn.execute_and_commit( + "INSERT INTO {0}(date, userid, countrycode, country, count) VALUES ('{1}', '{2}', '{3}', '{4}', {5}) ON CONFLICT (date, userid, countrycode) DO UPDATE SET count = {0}.count + 1".format(userCountryStatistics.USERCOUNTRYTABLE, userCountryStatistics.date, userCountryStatistics.userid, userCountryStatistics.countrycode, userCountryStatistics.country, 1) + ) + + @classmethod + def saveAll(self, userCountryStatisticsList): + pgConn = destinationPgConnector() + values = '' + for item in userCountryStatisticsList: + values += "INSERT INTO {0}(date, userid, countrycode, country, count) VALUES ('{1}', '{2}', '{3}', '{4}', {5}) ON CONFLICT (date, userid, countrycode) DO UPDATE SET count = {0}.count + 1;".format(userCountryStatistics.USERCOUNTRYTABLE, item.date, item.userid, item.countrycode, item.country, 1) + pgConn.execute_and_commit(values) \ No newline at end of file diff --git a/metrics-migrate-stats/Model/users.py b/metrics-migrate-stats/Model/users.py new file mode 100644 index 0000000..a4b6603 --- /dev/null +++ b/metrics-migrate-stats/Model/users.py @@ -0,0 +1,45 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +from Logger import log +import hashlib +class users(object): + logger = log.get_logger("users") + USERSTABLE = configParser.getConfig('destination_tables')['users'] + + def __init__(self, userid, created, status, tenenv_id): + + self.hasheduserid = userid + self.created = created + self.status = status + self.tenenv_id = tenenv_id + +# @classmethod +# def getLastDate(self): +# pgConn = destinationPgConnector() +# result = pgConn.execute_select("SELECT max(created::date) FROM {0}".format(voMemberships.VOMEMBERSHIPSTABLE)) +# return result + + + + @classmethod + def saveAll(self, usersList): + pgConn = destinationPgConnector() + values = '' + for item in usersList: + values += """INSERT INTO {0}(hasheduserid, created, status, tenenv_id) + VALUES ('{1}', '{2}','{3}', {4}) + ON CONFLICT(hasheduserid, tenenv_id) DO UPDATE SET status='{3}'; + """.format(users.USERSTABLE, + item.hasheduserid, + item.created, + item.status, + item.tenenv_id) + pgConn.execute_and_commit(values) + +# @classmethod +# def getMembershipStatus(self, voId, hasheduserid): +# pgConn = destinationPgConnector() +# result = pgConn.execute_select(""" +# SELECT status FROM {0} WHERE community_id={1} AND hasheduserid='{2}'""" +# .format(users.USERSTABLE, voId, hasheduserid)) +# return result \ No newline at end of file diff --git a/metrics-migrate-stats/Model/usersFromComanage.py b/metrics-migrate-stats/Model/usersFromComanage.py new file mode 100644 index 0000000..2fb3cc6 --- /dev/null +++ b/metrics-migrate-stats/Model/usersFromComanage.py @@ -0,0 +1,35 @@ +from Model.pgConnector import sourcePgConnector +from Utils import configParser +from datetime import date, timedelta +import hashlib + +class usersFromComanage(object): + USERSCOMANAGETABLE = configParser.getConfig('source_tables')['users_comanage'] + def __init__(self, userid, created, status): + + self.hasheduserid = hashlib.md5(userid.encode()).hexdigest() + self.created = created + self.status = status + + + @classmethod + def getAllUsers(self): + pgConn = sourcePgConnector() + result = (list(pgConn.execute_select(""" + SELECT identifier, cm_co_people.created, cm_co_people.status + FROM cm_co_people + JOIN cm_identifiers + ON cm_co_people.id = cm_identifiers.co_person_id + AND NOT cm_co_people.deleted + AND cm_co_people.co_person_id IS NULL + AND type='epuid' + WHERE NOT cm_identifiers.deleted AND identifier_id IS NULL + AND co_id=2 AND login=true + """))) + data = [] + for row in result: + usersData = usersFromComanage(row[0], row[1], row[2]) + data.append(usersData) + return data + + diff --git a/metrics-migrate-stats/Model/voMemberships.py b/metrics-migrate-stats/Model/voMemberships.py new file mode 100644 index 0000000..702f135 --- /dev/null +++ b/metrics-migrate-stats/Model/voMemberships.py @@ -0,0 +1,59 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +from Logger import log +import hashlib +class voMemberships(object): + logger = log.get_logger("voMemberships") + VOMEMBERSHIPSTABLE = configParser.getConfig('destination_tables')['vo_memberships'] + + def __init__(self, vo_id, userid, status, created, tenenv_id): + self.community_id = vo_id + self.hasheduserid = userid + self.status = status + self.tenenv_id = tenenv_id + self.created = created + +# @classmethod +# def getLastDate(self): +# pgConn = destinationPgConnector() +# result = pgConn.execute_select("SELECT max(created::date) FROM {0}".format(voMemberships.VOMEMBERSHIPSTABLE)) +# return result + + + @classmethod + def truncate(self, tenenv_id): + pgConn = destinationPgConnector() + # remove all data + pgConn.execute_and_commit( + """ + DELETE FROM {0} WHERE tenenv_id = {1} + """.format(voMemberships.VOMEMBERSHIPSTABLE, tenenv_id) + ) + + @classmethod + def saveAll(self, voMembershipsList): + pgConn = destinationPgConnector() + values = '' + for item in voMembershipsList: + values += """INSERT INTO {0}(community_id, hasheduserid, status, created, tenenv_id) + VALUES ('{1}', '{2}','{3}', '{4}', {5}) + ON CONFLICT(community_id, hasheduserid, tenenv_id) DO UPDATE SET status='{3}'; + """.format(voMemberships.VOMEMBERSHIPSTABLE, + item.community_id, + item.hasheduserid, + item.status, + item.created, + item.tenenv_id) + pgConn.execute_and_commit(values) + + @classmethod + def getMembershipStatus(self, voId, hasheduserid, tenenv_id): + pgConn = destinationPgConnector() + result = pgConn.execute_select(""" + SELECT status + FROM {0} + WHERE community_id={1} AND hasheduserid='{2}' + AND tenenv_id={3} + """ + .format(voMemberships.VOMEMBERSHIPSTABLE, voId, hasheduserid, tenenv_id)) + return result \ No newline at end of file diff --git a/metrics-migrate-stats/Model/voMembershipsFromComanage.py b/metrics-migrate-stats/Model/voMembershipsFromComanage.py new file mode 100644 index 0000000..2444215 --- /dev/null +++ b/metrics-migrate-stats/Model/voMembershipsFromComanage.py @@ -0,0 +1,58 @@ +from Model.pgConnector import sourcePgConnector +from Utils import configParser +from datetime import date, timedelta +import hashlib + +class voMembershipsFromComanage(object): + VOMEMBERSHIPSCOMANAGETABLE = configParser.getConfig('source_tables')['vo_memberships_comanage'] + def __init__(self, voName, userid, status, created): + self.status = status + self.hasheduserid = hashlib.md5(userid.encode()).hexdigest() + self.voName = voName + self.source = "registry" + self.created = created + +# @classmethod +# def getVosByDate(self, dateFrom, dateTo): +# pgConn = sourcePgConnector() +# if(dateFrom != None): +# result = list(pgConn.execute_select("SELECT created::date, name, description FROM {0} WHERE NOT deleted AND parent_id IS NULL AND cou_id IS NULL AND created::date BETWEEN '{1}' AND '{2}'".format(vosFromComanage.VOSCOMANAGETABLE, dateFrom, dateTo))) + +# else: +# result = list(pgConn.execute_select("SELECT created::date, name, description FROM {0} WHERE NOT deleted AND parent_id IS NULL AND cou_id IS NULL AND created::date <= '{1}'".format(vosFromComanage.VOSCOMANAGETABLE, dateTo))) +# data = [] +# for row in result: +# vosData = vosFromComanage(row[0], row[1], row[2]) +# data.append(vosData) +# return data + + @classmethod + # getAllVoMemberships + def getAllVoMembershipsByStatus(self, status): + yesterday = date.today() - timedelta(days=1) + dateTo = yesterday.strftime('%Y-%m-%d 23:59:59') + pgConn = sourcePgConnector() + if status == 'Other': + subquery = 'AND cm_co_person_roles.status!=\'A\' AND cm_co_person_roles.status!=\'GP\'' + else: + subquery = 'AND cm_co_person_roles.status=\'{0}\''.format(status) + result = list(pgConn.execute_select(""" + SELECT cm_cous.name, identifier, cm_co_person_roles.status,cm_co_person_roles.created + FROM cm_co_person_roles + JOIN cm_identifiers + ON cm_co_person_roles.co_person_id = cm_identifiers.co_person_id + AND NOT cm_co_person_roles.deleted + AND cm_co_person_roles.co_person_role_id IS NULL + AND type='epuid' + JOIN cm_cous ON cm_cous.id=cm_co_person_roles.cou_id AND cm_cous.cou_id IS NULL + WHERE NOT cm_co_person_roles.deleted + AND NOT cm_identifiers.deleted AND identifier_id IS NULL + {0} + """.format(subquery))) + data = [] + for row in result: + vosData = voMembershipsFromComanage(row[0], row[1], row[2], row[3]) + data.append(vosData) + return data + + diff --git a/metrics-migrate-stats/Model/vos.py b/metrics-migrate-stats/Model/vos.py new file mode 100644 index 0000000..326c1a7 --- /dev/null +++ b/metrics-migrate-stats/Model/vos.py @@ -0,0 +1,46 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +from Logger import log +class vos(object): + logger = log.get_logger("vos") + VOSTABLE = configParser.getConfig('destination_tables')['vos'] + VOSINFORMATIONTABLE = configParser.getConfig('destination_tables')['vos_information'] + + def __init__(self, id, created, tenenv_id): + self.community_id = id #foreign key coming from vosInfo + self.created = created + self.tenenv_id = tenenv_id + + @classmethod + def getVoIdFromVoName(self, voName, source, tenenv_id): + pgConn = destinationPgConnector() + + result = pgConn.execute_select(""" + SELECT id FROM {0} JOIN {1} ON id=community_id + WHERE source='{2}' AND name='{3}' AND {0}.tenenv_id={4} + """.format(vos.VOSTABLE, vos.VOSINFORMATIONTABLE, source, voName, tenenv_id)) + return result + + @classmethod + def getLastDate(self, tenenv_id): + pgConn = destinationPgConnector() + result = pgConn.execute_select(""" + SELECT max(created::date) FROM {0} WHERE tenenv_id={1} + """.format(vos.VOSTABLE, tenenv_id)) + return result + + @classmethod + def save(self, vos): + pgConn = destinationPgConnector() + pgConn.execute_and_commit( + "INSERT INTO {0}(community_id, created, status, tenenv_id) VALUES ('{1}', '{2}', '{3}', {4})".format(vos.VOSTABLE, vos.community_id, vos.created, 'A', vos.tenenv_id) + ) + + +# @classmethod +# def saveAll(self, vosList): +# pgConn = destinationPgConnector() +# values = '' +# for item in vosList: +# values += "INSERT INTO {0}(created, source) VALUES ('{1}', '{2}');".format(vos.VOSTABLE, item.created, item.source) +# pgConn.execute_and_commit(values) diff --git a/metrics-migrate-stats/Model/vosFromComanage.py b/metrics-migrate-stats/Model/vosFromComanage.py new file mode 100644 index 0000000..314971a --- /dev/null +++ b/metrics-migrate-stats/Model/vosFromComanage.py @@ -0,0 +1,40 @@ +from Model.pgConnector import sourcePgConnector +from Utils import configParser +from datetime import date, timedelta + +class vosFromComanage(object): + VOSCOMANAGETABLE = configParser.getConfig('source_tables')['vos_comanage'] + def __init__(self, created, name, description, status): + self.name = name + self.description = description + self.created = created + self.status = status + + @classmethod + def getVosByDate(self, dateFrom, dateTo): + pgConn = sourcePgConnector() + if(dateFrom != None): + result = list(pgConn.execute_select("SELECT created::date, name, description, 'A' FROM {0} WHERE NOT deleted AND parent_id IS NULL AND cou_id IS NULL AND created::date BETWEEN '{1}' AND '{2}'".format(vosFromComanage.VOSCOMANAGETABLE, dateFrom, dateTo))) + + else: + result = list(pgConn.execute_select("SELECT created::date, name, description, 'A' FROM {0} WHERE NOT deleted AND parent_id IS NULL AND cou_id IS NULL AND created::date <= '{1}'".format(vosFromComanage.VOSCOMANAGETABLE, dateTo))) + data = [] + for row in result: + vosData = vosFromComanage(row[0], row[1], row[2], row[3]) + data.append(vosData) + return data + + @classmethod + # getAllVos except those created today + def getAllVos(self): + yesterday = date.today() - timedelta(days=1) + dateTo = yesterday.strftime('%Y-%m-%d 23:59:59') + pgConn = sourcePgConnector() + result = list(pgConn.execute_select("SELECT created::date, name, description, 'A' FROM {0} WHERE NOT deleted AND parent_id IS NULL AND cou_id IS NULL AND created::date <= '{1}'".format(vosFromComanage.VOSCOMANAGETABLE, dateTo))) + data = [] + for row in result: + vosData = vosFromComanage(row[0], row[1], row[2], row[3]) + data.append(vosData) + return data + + diff --git a/metrics-migrate-stats/Model/vosInfo.py b/metrics-migrate-stats/Model/vosInfo.py new file mode 100644 index 0000000..41288a1 --- /dev/null +++ b/metrics-migrate-stats/Model/vosInfo.py @@ -0,0 +1,36 @@ +from Model.pgConnector import destinationPgConnector +from Utils import configParser +from Logger import log +class vosInfo(object): + logger = log.get_logger("vosInfo") + VOSINFOTABLE = configParser.getConfig('destination_tables')['vos_information'] + + def __init__(self, voName, voDescription, status, source, tenenv_id): + + self.voName = voName + self.voDescription = voDescription + self.source = source + self.tenenv_id = tenenv_id + + @classmethod + def save(self, vosInfo): + pgConn = destinationPgConnector() + id = pgConn.execute_and_commit_with_fetch( + """INSERT INTO {0}(name, description, source, tenenv_id) + VALUES ('{1}', '{2}', '{3}', {4}) ON CONFLICT DO NOTHING + RETURNING id""".format( + vosInfo.VOSINFOTABLE, + vosInfo.voName, + vosInfo.voDescription, + vosInfo.source, + vosInfo.tenenv_id) + ) + return id + +# @classmethod +# def saveAll(self, vosList): +# pgConn = destinationPgConnector() +# values = '' +# for item in vosList: +# values += "INSERT INTO {0}(community_id, name, description) VALUES ('{1}', '{2}', '{3}');".format(vosInfo.VOSINFOTABLE, item.id, item.voName, item.voDescription) +# pgConn.execute_and_commit(values) diff --git a/metrics-migrate-stats/README.md b/metrics-migrate-stats/README.md new file mode 100644 index 0000000..5cc6f0b --- /dev/null +++ b/metrics-migrate-stats/README.md @@ -0,0 +1,29 @@ +# migrate-metrics-stats +A Python-based tool for migrating stats from COmanage and SSP to Metrics Database. +## Prerequisites +This script uses MaxMind GeoIP2 Databases (commercial version). +You must put the .mmdb files (`GeoLite2-Country.mmdb`) to the folder `databases`. + +## Installation +``` +git clone https://github.com/rciam/rciam-migrate-metrics-stats.git +cd rciam-migrate-metrics-stats +cp config.py.example config.py +vi config.py +``` + +Create a Python virtualenv, install dependencies, and run the script +``` +virtualenv -p python3 .venv +source .venv/bin/activate +(venv) pip3 install -r requirements.txt +(venv) python3 -m Utils.install +(venv) python3 migrateData.py +🍺 +``` + +## License +Licensed under the Apache 2.0 license, for details see LICENSE. + + + diff --git a/metrics-migrate-stats/Service/ipDatabase.py b/metrics-migrate-stats/Service/ipDatabase.py new file mode 100644 index 0000000..ed09215 --- /dev/null +++ b/metrics-migrate-stats/Service/ipDatabase.py @@ -0,0 +1,18 @@ +from abc import ABC, abstractmethod +from Model.ipStatistics import ipStatistics +from Utils import configParser +import geoip2.database + +class ipDatabase(ABC): + DBFILENAME = configParser.getConfig('ip_database_file')['db_filename'] + @abstractmethod + def getCountryFromIp(self): + pass + +class geoip2Database(ipDatabase): + @classmethod + def getCountryFromIp(self, ip, ipVersion): + gi = geoip2.database.Reader('./databases/{0}'.format(ipDatabase.DBFILENAME)) + return [gi.country(ip).country.iso_code,gi.country(ip).country.name] + + diff --git a/metrics-migrate-stats/Utils/configParser.py b/metrics-migrate-stats/Utils/configParser.py new file mode 100644 index 0000000..aa7db27 --- /dev/null +++ b/metrics-migrate-stats/Utils/configParser.py @@ -0,0 +1,32 @@ +from configparser import RawConfigParser + +CONFIG_FILE = 'config.py' + +def getConfig(section='source_database'): + + # create a parser + + parser = RawConfigParser() + + # read config file + + parser.read(CONFIG_FILE) + + # get section, default to source_database + + config = {} + + if parser.has_section(section): + + params = parser.items(section) + + for param in params: + + config[param[0]] = param[1] + + else: + + raise Exception('Section {0} not found in the {1} file'.format(section, CONFIG_FILE)) + + + return config diff --git a/metrics-migrate-stats/Utils/install.py b/metrics-migrate-stats/Utils/install.py new file mode 100644 index 0000000..3648494 --- /dev/null +++ b/metrics-migrate-stats/Utils/install.py @@ -0,0 +1,40 @@ +from Model.pgConnector import destinationPgConnector + + +pgConn = destinationPgConnector() +# Create tables if not exist +pgConn.execute_and_commit( + open("./config-templates/pgsql_tables.sql", "r").read()) + +# Check if country_statistics has data and country_statistics_hashed doesn't +# dateFrom = countryStatistics.getLastDate() +# dateFromHashed = countryStatisticsHashedUserId.getLastDate() + +# if (dateFrom[0][0] != None and dateFromHashed[0][0] == None): +# logger = log.get_logger("install") +# ipDatabaseHandler = geoip2Database() +# # We must put data at country statistics hashed table +# ipData = ipStatistics.getIpStatisticsByDate(None, dateFrom[0][0]) +# mappedItems = 0 +# countryStatsHashedList = [] + +# for item in ipData: +# # get network address +# ipaddr = ipaddress.ip_network(item.ip).network_address +# # get country code/ name +# countryData = ipDatabaseHandler.getCountryFromIp(str(ipaddr), item.ipVersion) + +# if(countryData[0] != None): +# mappedItems +=1 +# else: +# countryData[0] = 'UN' +# countryData[1] = 'Unknown' +# logger.warning("ip {0} not found at database".format(ipaddr)) +# countryStatisticsHashedItem = countryStatisticsHashedUserId(None, item.accessed, item.userid, item.sourceIdp, item.service, countryData[0], countryData[1], 1) +# countryStatsHashedList.append(countryStatisticsHashedItem) +# if countryStatsHashedList: +# countryStatisticsHashedUserId.saveAll(countryStatsHashedList) +# logger.info("{0} ips mapped to countries".format(mappedItems)) +# else: +# logger.info("No new data found") +pgConn.close() diff --git a/metrics-migrate-stats/Utils/migrateKeycloakUsers.py b/metrics-migrate-stats/Utils/migrateKeycloakUsers.py new file mode 100644 index 0000000..fe03738 --- /dev/null +++ b/metrics-migrate-stats/Utils/migrateKeycloakUsers.py @@ -0,0 +1,46 @@ +import csv +import psycopg2 +import hashlib +from datetime import datetime +from Model.pgConnector import destinationPgConnector + + +pgConn = destinationPgConnector() + +# Define the path to your CSV file +csv_file_path = "einfra_devel_users.csv" + + +def insert_record(hasheduserid, created_date): + sql = "INSERT INTO users (hasheduserid, created, updated, status, tenenv_id) VALUES ('{0}','{1}', NULL, 'A',2)".format( + hasheduserid, created_date + ) + pgConn.execute_and_commit(sql) + + +def convert_to_date(timestamp_str): + """ + Convert a timestamp string to a date string in the format 'YYYY-MM-DD'. + """ + timestamp_ms = int(timestamp_str) + timestamp_sec = timestamp_ms / 1000.0 # Convert milliseconds to seconds + timestamp = datetime.fromtimestamp(timestamp_sec) + return timestamp.strftime('%Y-%m-%d') + +def main(): + # Open and read the CSV file + with open(csv_file_path, "r") as csv_file: + csv_reader = csv.reader(csv_file) + next(csv_reader) # Skip the header row if it exists + for row in csv_reader: + if len(row) != 2: + print(f"Skipping invalid row: {row}") + continue + vo_person_id, created_date = row + insert_record(hashlib.md5(vo_person_id.encode()).hexdigest(), convert_to_date(created_date)) + + print("Data has been inserted into the PostgreSQL database.") + + +if __name__ == "__main__": + main() diff --git a/metrics-migrate-stats/config-templates/pgsql_tables.sql b/metrics-migrate-stats/config-templates/pgsql_tables.sql new file mode 100644 index 0000000..09f8809 --- /dev/null +++ b/metrics-migrate-stats/config-templates/pgsql_tables.sql @@ -0,0 +1,177 @@ +CREATE TABLE IF NOT EXISTS environment_info ( + id SERIAL PRIMARY KEY, + name character varying(255) NOT NULL, + description character varying(255) NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_environment_info ON environment_info(name); + +CREATE TABLE IF NOT EXISTS tenant_info ( + id SERIAL PRIMARY KEY, + name character varying(255) NOT NULL, + description character varying(255) NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_tenant_info ON tenant_info(name); + +CREATE TABLE IF NOT EXISTS tenenv_info ( + id SERIAL PRIMARY KEY, + name character varying(255) NOT NULL, + description character varying(255) NOT NULL, + tenant_id INT, + CONSTRAINT fk_tenant + FOREIGN KEY(tenant_id) + REFERENCES tenant_info(id), + env_id INT, + CONSTRAINT fk_environment + FOREIGN KEY(env_id) + REFERENCES environment_info(id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_tenenv_info ON tenenv_info(tenant_id, env_id); + +CREATE TABLE IF NOT EXISTS community_info ( + id SERIAL PRIMARY KEY, + name character varying(255) NOT NULL, + description character varying(255) NOT NULL, + source character varying(255) NOT NULL, + tenenv_id INT, + CONSTRAINT fk_tenenv + FOREIGN KEY(tenenv_id) + REFERENCES tenenv_info(id) + +); + +-- CREATE INDEX IF NOT EXISTS community_info_i1 ON community_info (name); +CREATE UNIQUE INDEX IF NOT EXISTS idx_community_info ON community_info(name, tenenv_id); + +CREATE TABLE IF NOT EXISTS community ( + community_id INT, + created DATE NOT NULL, + updated DATE, + status character varying(255) NOT NULL, + PRIMARY KEY(community_id), + CONSTRAINT fk_community + FOREIGN KEY(community_id) + REFERENCES community_info(id), + tenenv_id INT NOT NULL, + CONSTRAINT fk_tenenv + FOREIGN KEY(tenenv_id) + REFERENCES tenenv_info(id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_community ON community(community_id, tenenv_id); + +CREATE INDEX IF NOT EXISTS community_i1 ON community (created); +CREATE INDEX IF NOT EXISTS community_i1 ON community (created, tenenv_id); + +CREATE TABLE IF NOT EXISTS members ( + community_id INT, + hasheduserid character varying(1024) NOT NULL, + status character varying(255) NOT NULL, + created DATE NOT NULL, + updated DATE, + CONSTRAINT fk_community + FOREIGN KEY(community_id) + REFERENCES community_info(id), + tenenv_id INT, + CONSTRAINT fk_tenenv + FOREIGN KEY(tenenv_id) + REFERENCES tenenv_info(id) +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_members ON members(community_id, hasheduserid, tenenv_id); + +CREATE TABLE IF NOT EXISTS identityprovidersmap ( + id SERIAL PRIMARY KEY, + entityid character varying(255) NOT NULL, + name character varying(255) NOT NULL, + tenenv_id INT, + CONSTRAINT fk_tenenv + FOREIGN KEY(tenenv_id) + REFERENCES tenenv_info(id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_identityprovidersmap ON identityprovidersmap(entityid, tenenv_id); + +CREATE TABLE IF NOT EXISTS serviceprovidersmap ( + id SERIAL PRIMARY KEY, + identifier character varying(255) NOT NULL, + name character varying(255) NOT NULL, + tenenv_id INT, + CONSTRAINT fk_tenenv + FOREIGN KEY(tenenv_id) + REFERENCES tenenv_info(id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_serviceprovidersmap ON serviceprovidersmap(identifier, tenenv_id); + +CREATE TABLE IF NOT EXISTS country_codes ( + id SERIAL PRIMARY KEY, + countrycode character varying(2) NOT NULL, + country character varying(255) NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_country_codes ON country_codes(countrycode); +CREATE UNIQUE INDEX IF NOT EXISTS idxx_country_codes ON country_codes (country); + +CREATE TABLE IF NOT EXISTS users ( + hasheduserid character varying(1024) NOT NULL, + created DATE NOT NULL, + updated DATE, + status character varying(255) NOT NULL, + tenenv_id INT, + CONSTRAINT fk_tenenv + FOREIGN KEY(tenenv_id) + REFERENCES tenenv_info(id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_users ON users(hasheduserid, tenenv_id); + +CREATE TABLE IF NOT EXISTS statistics_country_hashed ( + id SERIAL PRIMARY KEY, + date DATE NOT NULL, + hasheduserid character varying(1024) NOT NULL, + sourceidpid INT NOT NULL, + serviceid INT NOT NULL, + countryid INT NOT NULL, + count INT NOT NULL, + CONSTRAINT fk_idp + FOREIGN KEY(sourceidpid) + REFERENCES identityprovidersmap(id), + CONSTRAINT fk_service + FOREIGN KEY(serviceid) + REFERENCES serviceprovidersmap(id), + CONSTRAINT fk_country + FOREIGN KEY(countryid) + REFERENCES country_codes(id), + tenenv_id INT, + CONSTRAINT fk_tenenv + FOREIGN KEY(tenenv_id) + REFERENCES tenenv_info(id), + CONSTRAINT unique_row + UNIQUE(date, hasheduserid, sourceidpid, serviceid, countryid, tenenv_id) +); + +CREATE INDEX IF NOT EXISTS statistics_country_hashed_i1 ON statistics_country_hashed (hasheduserid); +CREATE INDEX IF NOT EXISTS statistics_country_hashed_i2 ON statistics_country_hashed (sourceidpid); +CREATE INDEX IF NOT EXISTS statistics_country_hashed_i3 ON statistics_country_hashed (serviceid); +CREATE INDEX IF NOT EXISTS statistics_country_hashed_i4 ON statistics_country_hashed (countryid); + +CREATE TABLE IF NOT EXISTS statistics_raw ( + id SERIAL PRIMARY KEY, + date timestamptz NOT NULL, + type VARCHAR(25), + event_identifier VARCHAR(255), + source VARCHAR(255), + tenenv_id INT NOT NULL, + jsondata JSONB, + CONSTRAINT fk_tenenv + FOREIGN KEY(tenenv_id) + REFERENCES tenenv_info(id), + CONSTRAINT uniqueRaw + UNIQUE(event_identifier, source, tenenv_id) +); + +CREATE INDEX IF NOT EXISTS statistics_raw_i1 ON statistics_raw (date); +CREATE INDEX IF NOT EXISTS statistics_raw_i2 ON statistics_raw (source); +CREATE INDEX IF NOT EXISTS statistics_raw_i3 ON statistics_raw (type); diff --git a/metrics-migrate-stats/config-templates/pgsql_tables_old.sql b/metrics-migrate-stats/config-templates/pgsql_tables_old.sql new file mode 100644 index 0000000..5921705 --- /dev/null +++ b/metrics-migrate-stats/config-templates/pgsql_tables_old.sql @@ -0,0 +1,142 @@ +-- Statistics for country logins including idp and sp +-- CREATE TABLE IF NOT EXISTS statistics_country ( +-- id SERIAL PRIMARY KEY, +-- date DATE NOT NULL, +-- sourceidp character varying(255) NOT NULL, +-- service character varying(255) NOT NULL, +-- countrycode character varying(2) NOT NULL, +-- country character varying(255) NOT NULL, +-- count int NOT NULL +-- ); + +-- CREATE INDEX IF NOT EXISTS statistics_country_i1 ON statistics_country (date); +-- CREATE INDEX IF NOT EXISTS statistics_country_i2 ON statistics_country (sourceidp); +-- CREATE INDEX IF NOT EXISTS statistics_country_i3 ON statistics_country (service); +-- CREATE INDEX IF NOT EXISTS statistics_country_i4 ON statistics_country (countrycode); +-- CREATE INDEX IF NOT EXISTS statistics_country_i5 ON statistics_country (country); +-- CREATE UNIQUE INDEX IF NOT EXISTS idx_statistics_country +-- ON statistics_country(date, sourceidp, service, countrycode); + +-- -- Statistics for country logins including idp,sp and hashed userid +-- CREATE TABLE IF NOT EXISTS statistics_country_hashed ( +-- id SERIAL PRIMARY KEY, +-- date DATE NOT NULL, +-- hasheduserid character varying(255) NOT NULL, +-- sourceidp character varying(255) NOT NULL, +-- service character varying(255) NOT NULL, +-- countrycode character varying(2) NOT NULL, +-- country character varying(255) NOT NULL, +-- count int NOT NULL +-- ); + +-- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i1 ON statistics_country_hashed (date); +-- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i2 ON statistics_country_hashed (sourceidp); +-- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i3 ON statistics_country_hashed (service); +-- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i4 ON statistics_country_hashed (countrycode); +-- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i5 ON statistics_country_hashed (country); +-- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i6 ON statistics_country_hashed (hasheduserid); +-- CREATE UNIQUE INDEX IF NOT EXISTS idx_statistics_country_hashed +-- ON statistics_country_hashed(date, hasheduserid, sourceidp, service, countrycode); + +-- -- Statistics for country logins including userid +-- CREATE TABLE IF NOT EXISTS statistics_user_country ( +-- id SERIAL PRIMARY KEY, +-- date DATE NOT NULL, +-- userid character varying(255) NOT NULL, +-- countrycode character varying(2) NOT NULL, +-- country character varying(255) NOT NULL, +-- count int NOT NULL +-- ); + +-- CREATE INDEX IF NOT EXISTS statistics_user_country_i1 ON statistics_user_country (date); +-- CREATE INDEX IF NOT EXISTS statistics_user_country_i2 ON statistics_user_country (userid); +-- CREATE INDEX IF NOT EXISTS statistics_user_country_i3 ON statistics_user_country (countrycode); +-- CREATE INDEX IF NOT EXISTS statistics_user_country_i4 ON statistics_user_country (country); +-- CREATE UNIQUE INDEX IF NOT EXISTS idx_statistics_user_country ON statistics_user_country(date, userid, countrycode); + +CREATE TABLE IF NOT EXISTS community_info ( + id SERIAL PRIMARY KEY, + name character varying(255) NOT NULL, + description character varying(255) NOT NULL, + source character varying(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS community ( + community_id INT, + created DATE NOT NULL, + PRIMARY KEY(community_id), + CONSTRAINT fk_community + FOREIGN KEY(community_id) + REFERENCES community_info(id) +); + + +CREATE TABLE IF NOT EXISTS members ( + community_id INT, + hasheduserid character varying(1024) NOT NULL, + status character varying(255) NOT NULL, + CONSTRAINT fk_community + FOREIGN KEY(community_id) + REFERENCES community_info(id) +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_members ON members(community_id,hasheduserid); +CREATE INDEX IF NOT EXISTS community_i1 ON community (created); +CREATE INDEX IF NOT EXISTS community_info_i1 ON community_info (name); +CREATE UNIQUE INDEX IF NOT EXISTS idx_community_info ON community_info(name); + +CREATE TABLE IF NOT EXISTS identityprovidersmap ( + id SERIAL PRIMARY KEY, + entityid character varying(255) NOT NULL, + name character varying(255) NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_identityprovidersmap ON identityprovidersmap(entityid); + +CREATE TABLE IF NOT EXISTS serviceprovidersmap ( + id SERIAL PRIMARY KEY, + identifier character varying(255) NOT NULL, + name character varying(255) NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_serviceprovidersmap ON serviceprovidersmap(identifier); + +CREATE TABLE IF NOT EXISTS country_codes ( + id SERIAL PRIMARY KEY, + countrycode character varying(2) NOT NULL, + country character varying(255) NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_country_codes ON country_codes(countrycode); +CREATE UNIQUE INDEX IF NOT EXISTS idxx_country_codes ON country_codes (country); + +CREATE TABLE IF NOT EXISTS users ( + hasheduserid character varying(1024) NOT NULL, + created DATE NOT NULL, + status character varying(255) NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_users ON users(hasheduserid); + +CREATE TABLE IF NOT EXISTS statistics_country_hashed ( + id SERIAL PRIMARY KEY, + date DATE NOT NULL, + hasheduserid character varying(1024) NOT NULL, + sourceidpid INT NOT NULL, + serviceid INT NOT NULL, + countryid INT NOT NULL, + count INT NOT NULL, + CONSTRAINT fk_idp + FOREIGN KEY(sourceidpid) + REFERENCES identityprovidersmap(id), + CONSTRAINT fk_service + FOREIGN KEY(serviceid) + REFERENCES serviceprovidersmap(id), + CONSTRAINT fk_country + FOREIGN KEY(countryid) + REFERENCES country_codes(id) +); + +CREATE INDEX IF NOT EXISTS statistics_country_hashed_i1 ON statistics_country_hashed (hasheduserid); +CREATE INDEX IF NOT EXISTS statistics_country_hashed_i2 ON statistics_country_hashed (sourceidpid); +CREATE INDEX IF NOT EXISTS statistics_country_hashed_i3 ON statistics_country_hashed (serviceid); +CREATE INDEX IF NOT EXISTS statistics_country_hashed_i4 ON statistics_country_hashed (countryid); diff --git a/metrics-migrate-stats/databases/empty b/metrics-migrate-stats/databases/empty new file mode 100644 index 0000000..e69de29 diff --git a/metrics-migrate-stats/migrateData.py b/metrics-migrate-stats/migrateData.py new file mode 100644 index 0000000..13a9cf7 --- /dev/null +++ b/metrics-migrate-stats/migrateData.py @@ -0,0 +1,104 @@ +#!/usr/local/bin/python3 +import os +import sys +from Controller.statisticsCountryHashedController import statisticsCountryHashedController +from Controller.usersController import usersController + +# change working directory +os.chdir(os.path.dirname(os.path.abspath(sys.argv[0]))) +from Model.vos import vos +from Model.vosInfo import vosInfo +from Model.voMemberships import voMemberships +from Model.identityProvidersMap import identityProvidersMap +from Model.serviceProvidersMap import serviceProvidersMap +from Model.tenenvs import tenenvs +from Model.tenants import tenants +from Model.environments import environments +from Controller.vosController import vosController +from Controller.voMembershipsController import voMembershipsController +from Controller.identityProvidersMapController import identityProvidersMapController +from Controller.serviceProvidersMapController import serviceProvidersMapController +from Utils import configParser +from Logger import log + + +class migrateData: + logger = log.get_logger("migrateData") + tenantName = configParser.getConfig('tenenv')['tenant'] + tenantDescription = configParser.getConfig('tenenv')['tenant_description'] + environmentName = configParser.getConfig('tenenv')['environment'] + environmentDescription = configParser.getConfig('tenenv')['environment_description'] + + environments.save(environments(environmentName, environmentDescription)) + environmentId = environments.getEnvironmentByName(environmentName)[0][0] + + tenants.save(tenants(tenantName, tenantDescription)) + tenantId = tenants.getTenantByName(tenantName)[0][0] + + tenenvs.save(tenenvs(tenantName, tenantName, tenantId, environmentId)) + tenenvId = tenenvs.getTenenvByTenantIdAndEnvId(tenantId, environmentId)[0][0] + + @classmethod + def voMembershipsMigrate(self): + voMembershipsData = voMembershipsController.getDataNotMapped(self.tenenvId) + + + @classmethod + def vosMigrate(self): + + vosData = vosController.getDataNotMapped(self.tenenvId) + mappedItems = 0 + + for item in vosData: + voItem = vosInfo(item.name, item.description, item.status, "registry", self.tenenvId) + id = vosInfo.save(voItem) + if id is not None: + self.logger.info("{0} item".format(id)) + vosInfoItem = vos(id, item.created, self.tenenvId) + vos.save(vosInfoItem) + mappedItems +=1 + + + if mappedItems > 0: + self.logger.info("{0} vos created".format(mappedItems)) + else: + self.logger.info("No new data found") + + @classmethod + def idpsMigrate(self): + mappedItems = 0 + idpsData = identityProvidersMapController.getAllData() + for item in idpsData: + idpItem = identityProvidersMap(entityid=item.entityid, name=item.name, tenenv_id=self.tenenvId) + identityProvidersMap.save(idpItem) + mappedItems +=1 + + self.logger.info("{0} Idps created".format(mappedItems)) + + @classmethod + def spsMigrate(self): + mappedItems = 0 + spsData = serviceProvidersMapController.getAllData() + for item in spsData: + spItem = serviceProvidersMap(item.identifier, item.name.replace("'", "''"), self.tenenvId) + serviceProvidersMap.save(spItem) + mappedItems +=1 + + self.logger.info("{0} Sps created".format(mappedItems)) + @classmethod + def countryStatsMigrate(self): + statisticsCountryHashedController.saveAllData(self.tenenvId) + + @classmethod + def usersMigrate(self): + usersController.saveUsers(self.tenenvId) + + +# run script +migrateData.vosMigrate() +migrateData.voMembershipsMigrate() +migrateData.idpsMigrate() +migrateData.spsMigrate() +migrateData.countryStatsMigrate() +migrateData.usersMigrate() + diff --git a/metrics-migrate-stats/migrateData_noCOmanage.py b/metrics-migrate-stats/migrateData_noCOmanage.py new file mode 100644 index 0000000..117c2da --- /dev/null +++ b/metrics-migrate-stats/migrateData_noCOmanage.py @@ -0,0 +1,104 @@ +#!/usr/local/bin/python3 +import os +import sys +from Controller.statisticsCountryHashedController import statisticsCountryHashedController +from Controller.usersController import usersController + +# change working directory +os.chdir(os.path.dirname(os.path.abspath(sys.argv[0]))) +from Model.vos import vos +from Model.vosInfo import vosInfo +from Model.voMemberships import voMemberships +from Model.identityProvidersMap import identityProvidersMap +from Model.serviceProvidersMap import serviceProvidersMap +from Model.tenenvs import tenenvs +from Model.tenants import tenants +from Model.environments import environments +from Controller.vosController import vosController +from Controller.voMembershipsController import voMembershipsController +from Controller.identityProvidersMapController import identityProvidersMapController +from Controller.serviceProvidersMapController import serviceProvidersMapController +from Utils import configParser +from Logger import log + + +class migrateData: + logger = log.get_logger("migrateData") + tenantName = configParser.getConfig('tenenv')['tenant'] + tenantDescription = configParser.getConfig('tenenv')['tenant_description'] + environmentName = configParser.getConfig('tenenv')['environment'] + environmentDescription = configParser.getConfig('tenenv')['environment_description'] + + environments.save(environments(environmentName, environmentDescription)) + environmentId = environments.getEnvironmentByName(environmentName)[0][0] + + tenants.save(tenants(tenantName, tenantDescription)) + tenantId = tenants.getTenantByName(tenantName)[0][0] + + tenenvs.save(tenenvs(tenantName, tenantName, tenantId, environmentId)) + tenenvId = tenenvs.getTenenvByTenantIdAndEnvId(tenantId, environmentId)[0][0] + + @classmethod + def voMembershipsMigrate(self): + voMembershipsData = voMembershipsController.getDataNotMapped(self.tenenvId) + + + @classmethod + def vosMigrate(self): + + vosData = vosController.getDataNotMapped(self.tenenvId) + mappedItems = 0 + + for item in vosData: + voItem = vosInfo(item.name, item.description, item.status, "registry", self.tenenvId) + id = vosInfo.save(voItem) + if id is not None: + self.logger.info("{0} item".format(id)) + vosInfoItem = vos(id, item.created, self.tenenvId) + vos.save(vosInfoItem) + mappedItems +=1 + + + if mappedItems > 0: + self.logger.info("{0} vos created".format(mappedItems)) + else: + self.logger.info("No new data found") + + @classmethod + def idpsMigrate(self): + mappedItems = 0 + idpsData = identityProvidersMapController.getAllData() + for item in idpsData: + idpItem = identityProvidersMap(entityid=item.entityid, name=item.name, tenenv_id=self.tenenvId) + identityProvidersMap.save(idpItem) + mappedItems +=1 + + self.logger.info("{0} Idps created".format(mappedItems)) + + @classmethod + def spsMigrate(self): + mappedItems = 0 + spsData = serviceProvidersMapController.getAllData() + for item in spsData: + spItem = serviceProvidersMap(item.identifier, item.name.replace("'", "''"), self.tenenvId) + serviceProvidersMap.save(spItem) + mappedItems +=1 + + self.logger.info("{0} Sps created".format(mappedItems)) + @classmethod + def countryStatsMigrate(self): + statisticsCountryHashedController.saveAllData(self.tenenvId) + + @classmethod + def usersMigrate(self): + usersController.saveUsers(self.tenenvId) + +#run script + +# migrateData.vosMigrate() +# migrateData.voMembershipsMigrate() +migrateData.idpsMigrate() +migrateData.spsMigrate() +migrateData.countryStatsMigrate() +# migrateData.usersMigrate() + diff --git a/metrics-migrate-stats/requirements.txt b/metrics-migrate-stats/requirements.txt new file mode 100644 index 0000000..bdf5123 --- /dev/null +++ b/metrics-migrate-stats/requirements.txt @@ -0,0 +1,2 @@ +psycopg2-binary==2.8.6 +geoip2 diff --git a/metrics-migrate-stats/test.py b/metrics-migrate-stats/test.py new file mode 100644 index 0000000..de6bec4 --- /dev/null +++ b/metrics-migrate-stats/test.py @@ -0,0 +1,34 @@ +import psycopg2 + +from psycopg2 import Error + + +try: + + # Connect to an existing database + connection = psycopg2.connect(user="", + password="", + host="db.rciam.noc.grnet.gr", + port="5432", + database="") + + + # Create a cursor to perform database operations + cursor = connection.cursor() + # Print PostgreSQL details + print("PostgreSQL server information") + print(connection.get_dsn_parameters(), "\n") + # Executing a SQL query + cursor.execute("SELECT version();") + # Fetch result + record = cursor.fetchone() + print("You are connected to - ", record, "\n") + +except (Exception, Error) as error: + print("Error while connecting to PostgreSQL", error) + +finally: + if (connection): + cursor.close() + connection.close() + print("PostgreSQL connection is closed") From edb472d1ff18ffef4dfb3f5a891d8ad698857e37 Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Fri, 17 Nov 2023 09:51:56 +0200 Subject: [PATCH 3/5] minor fix --- metrics-migrate-stats/README.md | 2 +- .../config-templates/pgsql_tables_old.sql | 142 ------------------ 2 files changed, 1 insertion(+), 143 deletions(-) delete mode 100644 metrics-migrate-stats/config-templates/pgsql_tables_old.sql diff --git a/metrics-migrate-stats/README.md b/metrics-migrate-stats/README.md index 5cc6f0b..369d875 100644 --- a/metrics-migrate-stats/README.md +++ b/metrics-migrate-stats/README.md @@ -18,7 +18,7 @@ virtualenv -p python3 .venv source .venv/bin/activate (venv) pip3 install -r requirements.txt (venv) python3 -m Utils.install -(venv) python3 migrateData.py +(venv) python3 migrateData.py (or migrateData_noComanage.py) 🍺 ``` diff --git a/metrics-migrate-stats/config-templates/pgsql_tables_old.sql b/metrics-migrate-stats/config-templates/pgsql_tables_old.sql deleted file mode 100644 index 5921705..0000000 --- a/metrics-migrate-stats/config-templates/pgsql_tables_old.sql +++ /dev/null @@ -1,142 +0,0 @@ --- Statistics for country logins including idp and sp --- CREATE TABLE IF NOT EXISTS statistics_country ( --- id SERIAL PRIMARY KEY, --- date DATE NOT NULL, --- sourceidp character varying(255) NOT NULL, --- service character varying(255) NOT NULL, --- countrycode character varying(2) NOT NULL, --- country character varying(255) NOT NULL, --- count int NOT NULL --- ); - --- CREATE INDEX IF NOT EXISTS statistics_country_i1 ON statistics_country (date); --- CREATE INDEX IF NOT EXISTS statistics_country_i2 ON statistics_country (sourceidp); --- CREATE INDEX IF NOT EXISTS statistics_country_i3 ON statistics_country (service); --- CREATE INDEX IF NOT EXISTS statistics_country_i4 ON statistics_country (countrycode); --- CREATE INDEX IF NOT EXISTS statistics_country_i5 ON statistics_country (country); --- CREATE UNIQUE INDEX IF NOT EXISTS idx_statistics_country --- ON statistics_country(date, sourceidp, service, countrycode); - --- -- Statistics for country logins including idp,sp and hashed userid --- CREATE TABLE IF NOT EXISTS statistics_country_hashed ( --- id SERIAL PRIMARY KEY, --- date DATE NOT NULL, --- hasheduserid character varying(255) NOT NULL, --- sourceidp character varying(255) NOT NULL, --- service character varying(255) NOT NULL, --- countrycode character varying(2) NOT NULL, --- country character varying(255) NOT NULL, --- count int NOT NULL --- ); - --- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i1 ON statistics_country_hashed (date); --- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i2 ON statistics_country_hashed (sourceidp); --- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i3 ON statistics_country_hashed (service); --- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i4 ON statistics_country_hashed (countrycode); --- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i5 ON statistics_country_hashed (country); --- CREATE INDEX IF NOT EXISTS statistics_country_hashed_i6 ON statistics_country_hashed (hasheduserid); --- CREATE UNIQUE INDEX IF NOT EXISTS idx_statistics_country_hashed --- ON statistics_country_hashed(date, hasheduserid, sourceidp, service, countrycode); - --- -- Statistics for country logins including userid --- CREATE TABLE IF NOT EXISTS statistics_user_country ( --- id SERIAL PRIMARY KEY, --- date DATE NOT NULL, --- userid character varying(255) NOT NULL, --- countrycode character varying(2) NOT NULL, --- country character varying(255) NOT NULL, --- count int NOT NULL --- ); - --- CREATE INDEX IF NOT EXISTS statistics_user_country_i1 ON statistics_user_country (date); --- CREATE INDEX IF NOT EXISTS statistics_user_country_i2 ON statistics_user_country (userid); --- CREATE INDEX IF NOT EXISTS statistics_user_country_i3 ON statistics_user_country (countrycode); --- CREATE INDEX IF NOT EXISTS statistics_user_country_i4 ON statistics_user_country (country); --- CREATE UNIQUE INDEX IF NOT EXISTS idx_statistics_user_country ON statistics_user_country(date, userid, countrycode); - -CREATE TABLE IF NOT EXISTS community_info ( - id SERIAL PRIMARY KEY, - name character varying(255) NOT NULL, - description character varying(255) NOT NULL, - source character varying(255) NOT NULL -); - -CREATE TABLE IF NOT EXISTS community ( - community_id INT, - created DATE NOT NULL, - PRIMARY KEY(community_id), - CONSTRAINT fk_community - FOREIGN KEY(community_id) - REFERENCES community_info(id) -); - - -CREATE TABLE IF NOT EXISTS members ( - community_id INT, - hasheduserid character varying(1024) NOT NULL, - status character varying(255) NOT NULL, - CONSTRAINT fk_community - FOREIGN KEY(community_id) - REFERENCES community_info(id) -); -CREATE UNIQUE INDEX IF NOT EXISTS idx_members ON members(community_id,hasheduserid); -CREATE INDEX IF NOT EXISTS community_i1 ON community (created); -CREATE INDEX IF NOT EXISTS community_info_i1 ON community_info (name); -CREATE UNIQUE INDEX IF NOT EXISTS idx_community_info ON community_info(name); - -CREATE TABLE IF NOT EXISTS identityprovidersmap ( - id SERIAL PRIMARY KEY, - entityid character varying(255) NOT NULL, - name character varying(255) NOT NULL -); - -CREATE UNIQUE INDEX IF NOT EXISTS idx_identityprovidersmap ON identityprovidersmap(entityid); - -CREATE TABLE IF NOT EXISTS serviceprovidersmap ( - id SERIAL PRIMARY KEY, - identifier character varying(255) NOT NULL, - name character varying(255) NOT NULL -); - -CREATE UNIQUE INDEX IF NOT EXISTS idx_serviceprovidersmap ON serviceprovidersmap(identifier); - -CREATE TABLE IF NOT EXISTS country_codes ( - id SERIAL PRIMARY KEY, - countrycode character varying(2) NOT NULL, - country character varying(255) NOT NULL -); - -CREATE UNIQUE INDEX IF NOT EXISTS idx_country_codes ON country_codes(countrycode); -CREATE UNIQUE INDEX IF NOT EXISTS idxx_country_codes ON country_codes (country); - -CREATE TABLE IF NOT EXISTS users ( - hasheduserid character varying(1024) NOT NULL, - created DATE NOT NULL, - status character varying(255) NOT NULL -); - -CREATE UNIQUE INDEX IF NOT EXISTS idx_users ON users(hasheduserid); - -CREATE TABLE IF NOT EXISTS statistics_country_hashed ( - id SERIAL PRIMARY KEY, - date DATE NOT NULL, - hasheduserid character varying(1024) NOT NULL, - sourceidpid INT NOT NULL, - serviceid INT NOT NULL, - countryid INT NOT NULL, - count INT NOT NULL, - CONSTRAINT fk_idp - FOREIGN KEY(sourceidpid) - REFERENCES identityprovidersmap(id), - CONSTRAINT fk_service - FOREIGN KEY(serviceid) - REFERENCES serviceprovidersmap(id), - CONSTRAINT fk_country - FOREIGN KEY(countryid) - REFERENCES country_codes(id) -); - -CREATE INDEX IF NOT EXISTS statistics_country_hashed_i1 ON statistics_country_hashed (hasheduserid); -CREATE INDEX IF NOT EXISTS statistics_country_hashed_i2 ON statistics_country_hashed (sourceidpid); -CREATE INDEX IF NOT EXISTS statistics_country_hashed_i3 ON statistics_country_hashed (serviceid); -CREATE INDEX IF NOT EXISTS statistics_country_hashed_i4 ON statistics_country_hashed (countryid); From 67f3ce9cd7eee989f4cacce11a5de9d37d728fe2 Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Fri, 17 Nov 2023 09:54:04 +0200 Subject: [PATCH 4/5] add config example --- metrics-migrate-stats/config.py.example | 49 +++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 metrics-migrate-stats/config.py.example diff --git a/metrics-migrate-stats/config.py.example b/metrics-migrate-stats/config.py.example new file mode 100644 index 0000000..e44cc43 --- /dev/null +++ b/metrics-migrate-stats/config.py.example @@ -0,0 +1,49 @@ +[source_database] + +host=localhost +database=db_name +user=db_user +password=db_password + +[source_tables] + +statistics_detail=statistics_detail +statistics=statistics +ip_table=statistics_ip +vos_comanage=cm_cous +users_comanage=cm_co_people +vo_memberships_comanage=cm_co_person_roles +statistics_country_hashed_comanage=statistics_country_hashed +identity_providers_map_proxy=identityprovidersmap +service_providers_map_proxy=serviceprovidersmap + +[destination_database] + +host=localhost +database=db_name +user=db_user +password=db_password + +[destination_tables] + +vos=community +vos_information=community_info +vo_memberships=members +users=users +identity_providers_map=identityprovidersmap +service_providers_map=serviceprovidersmap +statistics_country_hashed=statistics_country_hashed +countries=country_codes +tenants=tenant_info +environments=environment_info +projects=project_info +tenenvs=tenenv_info + +[logging] + +level = INFO +folder = log +file = metricsMigrate.log + +[ip_database_file] +db_filename = GeoLite2-Country.mmdb \ No newline at end of file From 7d5177e52abf90ec0a4480de8f8e938549ffa781 Mon Sep 17 00:00:00 2001 From: Nick Mastoris Date: Fri, 17 Nov 2023 09:55:27 +0200 Subject: [PATCH 5/5] minor refactor --- metrics-migrate-stats/Utils/install.py | 32 -------------------------- 1 file changed, 32 deletions(-) diff --git a/metrics-migrate-stats/Utils/install.py b/metrics-migrate-stats/Utils/install.py index 3648494..7abf64e 100644 --- a/metrics-migrate-stats/Utils/install.py +++ b/metrics-migrate-stats/Utils/install.py @@ -5,36 +5,4 @@ # Create tables if not exist pgConn.execute_and_commit( open("./config-templates/pgsql_tables.sql", "r").read()) - -# Check if country_statistics has data and country_statistics_hashed doesn't -# dateFrom = countryStatistics.getLastDate() -# dateFromHashed = countryStatisticsHashedUserId.getLastDate() - -# if (dateFrom[0][0] != None and dateFromHashed[0][0] == None): -# logger = log.get_logger("install") -# ipDatabaseHandler = geoip2Database() -# # We must put data at country statistics hashed table -# ipData = ipStatistics.getIpStatisticsByDate(None, dateFrom[0][0]) -# mappedItems = 0 -# countryStatsHashedList = [] - -# for item in ipData: -# # get network address -# ipaddr = ipaddress.ip_network(item.ip).network_address -# # get country code/ name -# countryData = ipDatabaseHandler.getCountryFromIp(str(ipaddr), item.ipVersion) - -# if(countryData[0] != None): -# mappedItems +=1 -# else: -# countryData[0] = 'UN' -# countryData[1] = 'Unknown' -# logger.warning("ip {0} not found at database".format(ipaddr)) -# countryStatisticsHashedItem = countryStatisticsHashedUserId(None, item.accessed, item.userid, item.sourceIdp, item.service, countryData[0], countryData[1], 1) -# countryStatsHashedList.append(countryStatisticsHashedItem) -# if countryStatsHashedList: -# countryStatisticsHashedUserId.saveAll(countryStatsHashedList) -# logger.info("{0} ips mapped to countries".format(mappedItems)) -# else: -# logger.info("No new data found") pgConn.close()