temp: Retirement refactor example

This commit shows an example of how we could use Stevedore drivers to
handle retirement extension. The entrypoint name would become the "API"
in our configuration, and each "API" extension class would provide a
"get_instance" method which would take the retirement config dict and
return a constructed instance of the API class, which would then be used
a normal.

This code is not complete, simply a real world example of how we could
abstract out the existing 3rd party APIs using this method.

scripts/user_retirement/requirements/base.in

@@ -10,4 +10,5 @@ jenkinsapi
 unicodecsv
 simplejson
 simple-salesforce
+stevedore
 google-api-python-client

scripts/user_retirement/setup.cfg

@@ -0,0 +1,12 @@
+[options]
+packages = utils.thirdparty_apis
+
+[options.entry_points]
+# `retirement_driver` is the namespace chosen by our plugin manager
+# this driver should register itself to `retirement_driver`
+retirement_driver =
+    AMPLITUDE = utils.thirdparty_apis.amplitude_api:AmplitudeApi
+    BRAZE = utils.thirdparty_apis.braze_api:BrazeApi
+    HUBSPOT = utils.thirdparty_apis.hubspot_api:HubspotApi
+    SALESFORCE = utils.thirdparty_apis.salesforce_api:SalesforceApi
+    SEGMENT = utils.thirdparty_apis.segment_api:SegmentApi

scripts/user_retirement/setup.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+from setuptools import setup
+
+setup()

scripts/user_retirement/utils/helpers.py

@@ -16,18 +16,10 @@
 
 import yaml
 from six import text_type
+from stevedore import driver
 
 from scripts.user_retirement.utils.edx_api import LmsApi  # pylint: disable=wrong-import-position
 from scripts.user_retirement.utils.edx_api import CredentialsApi, EcommerceApi, LicenseManagerApi
-from scripts.user_retirement.utils.thirdparty_apis.amplitude_api import \
-    AmplitudeApi  # pylint: disable=wrong-import-position
-from scripts.user_retirement.utils.thirdparty_apis.braze_api import BrazeApi  # pylint: disable=wrong-import-position
-from scripts.user_retirement.utils.thirdparty_apis.hubspot_api import \
-    HubspotAPI  # pylint: disable=wrong-import-position
-from scripts.user_retirement.utils.thirdparty_apis.salesforce_api import \
-    SalesforceApi  # pylint: disable=wrong-import-position
-from scripts.user_retirement.utils.thirdparty_apis.segment_api import \
-    SegmentApi  # pylint: disable=wrong-import-position
 
 
 def _log(kind, message):
@@ -152,69 +144,13 @@ def _setup_all_apis_or_exit(fail_func, fail_code, config):
         lms_base_url = config['base_urls']['lms']
         ecommerce_base_url = config['base_urls'].get('ecommerce', None)
         credentials_base_url = config['base_urls'].get('credentials', None)
-        segment_base_url = config['base_urls'].get('segment', None)
         license_manager_base_url = config['base_urls'].get('license_manager', None)
         client_id = config['client_id']
         client_secret = config['client_secret']
-        braze_api_key = config.get('braze_api_key', None)
-        braze_instance = config.get('braze_instance', None)
-        amplitude_api_key = config.get('amplitude_api_key', None)
-        amplitude_secret_key = config.get('amplitude_secret_key', None)
-        salesforce_user = config.get('salesforce_user', None)
-        salesforce_password = config.get('salesforce_password', None)
-        salesforce_token = config.get('salesforce_token', None)
-        salesforce_domain = config.get('salesforce_domain', None)
-        salesforce_assignee = config.get('salesforce_assignee', None)
-        segment_auth_token = config.get('segment_auth_token', None)
-        segment_workspace_slug = config.get('segment_workspace_slug', None)
-        hubspot_api_key = config.get('hubspot_api_key', None)
-        hubspot_aws_region = config.get('hubspot_aws_region', None)
-        hubspot_from_address = config.get('hubspot_from_address', None)
-        hubspot_alert_email = config.get('hubspot_alert_email', None)
-
-        for state in config['retirement_pipeline']:
-            for service, service_url in (
-                ('BRAZE', braze_api_key),
-                ('AMPLITUDE', amplitude_api_key),
-                ('ECOMMERCE', ecommerce_base_url),
-                ('CREDENTIALS', credentials_base_url),
-                ('SEGMENT', segment_base_url),
-                ('HUBSPOT', hubspot_api_key),
-            ):
-                if state[2] == service and service_url is None:
-                    fail_func(fail_code, 'Service URL is not configured, but required for state {}'.format(state))
 
+        # Load any Open edX service Apis that are configured
         config['LMS'] = LmsApi(lms_base_url, lms_base_url, client_id, client_secret)
 
-        if braze_api_key:
-            config['BRAZE'] = BrazeApi(
-                braze_api_key,
-                braze_instance,
-            )
-
-        if amplitude_api_key and amplitude_secret_key:
-            config['AMPLITUDE'] = AmplitudeApi(
-                amplitude_api_key,
-                amplitude_secret_key,
-            )
-
-        if salesforce_user and salesforce_password and salesforce_token:
-            config['SALESFORCE'] = SalesforceApi(
-                salesforce_user,
-                salesforce_password,
-                salesforce_token,
-                salesforce_domain,
-                salesforce_assignee
-            )
-
-        if hubspot_api_key:
-            config['HUBSPOT'] = HubspotAPI(
-                hubspot_api_key,
-                hubspot_aws_region,
-                hubspot_from_address,
-                hubspot_alert_email
-            )
-
         if ecommerce_base_url:
             config['ECOMMERCE'] = EcommerceApi(lms_base_url, ecommerce_base_url, client_id, client_secret)
 
@@ -229,11 +165,19 @@ def _setup_all_apis_or_exit(fail_func, fail_code, config):
                 client_secret,
             )
 
-        if segment_base_url:
-            config['SEGMENT'] = SegmentApi(
-                segment_base_url,
-                segment_auth_token,
-                segment_workspace_slug
+        # Load and configure any retirement plugin APIs that are installed
+        for state in config['retirement_pipeline']:
+            service_name = state[2]
+
+            # Skip any states that have already loaded APIs
+            if service_name in config:
+                continue
+
+            mgr = driver.DriverManager(
+                namespace="retirement_driver",
+                name=service_name
             )
+            config[service_name] = mgr.driver.get_instance(config)
+
     except Exception as exc:  # pylint: disable=broad-except
         fail_func(fail_code, 'Unexpected error occurred!', exc)

scripts/user_retirement/utils/thirdparty_apis/amplitude_api.py

@@ -36,6 +36,26 @@ def __init__(self, amplitude_api_key, amplitude_secret_key):
         self.base_url = "https://amplitude.com/"
         self.delete_user_path = "api/2/deletions/users"
 
+    @staticmethod
+    def get_instance(config):
+        """
+        This function is used to get instance of AmplitudeApi.
+
+        Returns:
+            AmplitudeApi: Returns instance of AmplitudeApi.
+
+        Args:
+            config (dict): Configuration dictionary.
+
+        Raises:
+            KeyError: If amplitude_api_key or amplitude_secret_key is not present in config.
+        """
+        amplitude_api_key = config.get('amplitude_api_key', None)
+        amplitude_secret_key = config.get('amplitude_secret_key', None)
+        if not amplitude_api_key or not amplitude_secret_key:
+            raise KeyError("amplitude_api_key or amplitude_secret_key is not present in config.")
+        return AmplitudeApi(amplitude_api_key, amplitude_secret_key)
+
     def auth(self):
         """
         Returns auth credentials for Amplitude authorization.

scripts/user_retirement/utils/thirdparty_apis/braze_api.py

@@ -30,6 +30,27 @@ def __init__(self, braze_api_key, braze_instance):
         # https://www.braze.com/docs/api/basics/#endpoints
         self.base_url = 'https://rest.{instance}.braze.com'.format(instance=braze_instance)
 
+    @staticmethod
+    def get_instance(config):
+        """
+        This function is used to get instance of BrazeApi.
+
+        Returns:
+            BrazeApi: Returns instance of BrazeApi.
+
+        Args:
+            config (dict): Configuration dictionary.
+
+        Raises:
+            KeyError: If braze_api_key or braze_instance is not present in config.
+        """
+        braze_api_key = config.get('braze_api_key', None)
+        braze_instance = config.get('braze_instance', None)
+        if not braze_api_key or not braze_instance:
+            raise KeyError("braze_api_key or braze_instance is not present in config.")
+
+        return BrazeApi(braze_api_key, braze_instance)
+
     def auth_headers(self):
         """Returns authorization headers suitable for passing to the requests library"""
         return {

scripts/user_retirement/utils/thirdparty_apis/hubspot_api.py

@@ -37,6 +37,30 @@ def __init__(
         self.from_address = from_address
         self.alert_email = alert_email
 
+    @staticmethod
+    def get_instance(config):
+        """
+        This function is used to get instance of HubspotAPI.
+
+        Returns:
+            HubspotAPI: Returns instance of HubspotAPI.
+
+        Args:
+            config (dict): Configuration dictionary.
+
+        Raises:
+            KeyError: If hubspot_api_key is not present in config.
+        """
+        hubspot_api_key = config.get('hubspot_api_key', None)
+        hubspot_aws_region = config.get('hubspot_aws_region', None)
+        hubspot_from_address = config.get('hubspot_from_address', None)
+        hubspot_alert_email = config.get('hubspot_alert_email', None)
+
+        if not hubspot_api_key or not hubspot_aws_region or not hubspot_from_address or not hubspot_alert_email:
+            raise KeyError("hubspot_api_key, hubspot_aws_region, hubspot_from_address, or hubspot_alert_email is not present in config.")
+
+        return HubspotAPI(hubspot_api_key, hubspot_aws_region, hubspot_from_address, hubspot_alert_email)
+
     @backoff.on_exception(
         backoff.expo,
         HubspotException,

scripts/user_retirement/utils/thirdparty_apis/salesforce_api.py

@@ -37,6 +37,31 @@ def __init__(self, username, password, security_token, domain, assignee_username
         if not self.assignee_id:
             raise Exception("Could not find Salesforce user with username " + assignee_username)
 
+    @staticmethod
+    def get_instance(config):
+        """
+        This function is used to get instance of SalesforceApi.
+
+        Returns:
+            SalesforceApi: Returns instance of SalesforceApi.
+
+        Args:
+            config (dict): Configuration dictionary.
+
+        Raises:
+            KeyError: If salesforce_username, salesforce_password, salesforce_security_token, salesforce_domain, or salesforce_assignee is not present in config.
+        """
+        salesforce_username = config.get('salesforce_username', None)
+        salesforce_password = config.get('salesforce_password', None)
+        salesforce_security_token = config.get('salesforce_security_token', None)
+        salesforce_domain = config.get('salesforce_domain', None)
+        salesforce_assignee = config.get('salesforce_assignee', None)
+
+        if not salesforce_username or not salesforce_password or not salesforce_security_token or not salesforce_domain or not salesforce_assignee:
+            raise KeyError("salesforce_username, salesforce_password, salesforce_security_token, salesforce_domain, or salesforce_assignee is not present in config.")
+
+        return SalesforceApi(salesforce_username, salesforce_password, salesforce_security_token, salesforce_domain, salesforce_assignee)
+
     @backoff.on_exception(
         backoff.expo,
         RequestsConnectionError,

scripts/user_retirement/utils/thirdparty_apis/segment_api.py

@@ -99,6 +99,29 @@ def __init__(self, base_url, auth_token, workspace_slug):
         self.auth_token = auth_token
         self.workspace_slug = workspace_slug
 
+    @staticmethod
+    def get_instance(config):
+        """
+        This function is used to get instance of SegmentApi.
+
+        Returns:
+            SegmentApi: Returns instance of SegmentApi.
+
+        Args:
+            config (dict): Configuration dictionary.
+
+        Raises:
+            KeyError: If segment_base_url, segment_auth_token, and segment_workspace_slug are not present in config.
+        """
+        segment_base_url = config.get('segment_base_url', None)
+        segment_auth_token = config.get('segment_auth_token', None)
+        segment_workspace_slug = config.get('segment_workspace_slug', None)
+
+        if not segment_base_url or not segment_auth_token or not segment_workspace_slug:
+            raise KeyError("segment_base_url, segment_auth_token or segment_workspace_slug is not present in config.")
+
+        return SegmentApi(segment_base_url, segment_auth_token, segment_workspace_slug)
+
     @_retry_segment_api()
     def _call_segment_post(self, url, params):
         """