Merge pull request #360 from silinternational/feature/IDP-1183-cron-sync

[IDP-1183, partial] Add cron-based sync of `groups_external` data from Google Sheet

application/common/components/ExternalGroupsSync.php

@@ -0,0 +1,110 @@
+<?php
+
+namespace common\components;
+
+use common\models\User;
+use Webmozart\Assert\Assert;
+use Yii;
+use yii\base\Component;
+
+class ExternalGroupsSync extends Component
+{
+    public const MAX_SYNC_SETS = 20;
+
+    public static function syncAllSets(array $syncSetsParams)
+    {
+        for ($i = 1; $i <= self::MAX_SYNC_SETS; $i++) {
+            $appPrefixKey = sprintf('set%uAppPrefix', $i);
+            $googleSheetIdKey = sprintf('set%uGoogleSheetId', $i);
+
+            if (! array_key_exists($appPrefixKey, $syncSetsParams)) {
+                Yii::warning(sprintf(
+                    'Finished syncing external groups after %s sync set(s).',
+                    ($i - 1)
+                ));
+                break;
+            }
+
+            $appPrefix = $syncSetsParams[$appPrefixKey] ?? null;
+            $googleSheetId = $syncSetsParams[$googleSheetIdKey] ?? null;
+
+            if (empty($appPrefix) || empty($googleSheetId)) {
+                Yii::error(sprintf(
+                    'Unable to do external-groups sync set %s: '
+                    . 'app-prefix (%s) or Google Sheet ID (%s) was empty.',
+                    $i,
+                    json_encode($appPrefix),
+                    json_encode($googleSheetId),
+                ));
+            } else {
+                Yii::warning(sprintf(
+                    "Syncing '%s' external groups from Google Sheet (%s)...",
+                    $appPrefix,
+                    $googleSheetId
+                ));
+                self::syncSet($appPrefix, $googleSheetId);
+            }
+        }
+    }
+
+    private static function syncSet(string $appPrefix, string $googleSheetId)
+    {
+        $desiredExternalGroups = self::getExternalGroupsFromGoogleSheet($googleSheetId);
+        $errors = User::updateUsersExternalGroups($appPrefix, $desiredExternalGroups);
+        Yii::warning(sprintf(
+            "Ran sync for '%s' external groups.",
+            $appPrefix
+        ));
+
+        if (! empty($errors)) {
+            Yii::error(sprintf(
+                'Errors that occurred while syncing %s external groups: %s',
+                $appPrefix,
+                join(" / ", $errors)
+            ));
+        }
+    }
+
+    /**
+     * Get the desired external-group values, indexed by email address, from the
+     * specified Google Sheet, from the tab named after this IDP's code name
+     * (i.e. the name used in this IDP's subdomain).
+     *
+     * @throws \Google\Service\Exception
+     */
+    private static function getExternalGroupsFromGoogleSheet(string $googleSheetId): array
+    {
+        $googleSheetsClient = new Sheets([
+            'applicationName' => Yii::$app->params['google']['applicationName'],
+            'jsonAuthFilePath' => Yii::$app->params['google']['jsonAuthFilePath'],
+            'jsonAuthString' => Yii::$app->params['google']['jsonAuthString'],
+            'spreadsheetId' => $googleSheetId,
+        ]);
+        $tabName = Yii::$app->params['idpName'];
+
+        $values = $googleSheetsClient->getValuesFromTab($tabName);
+        $columnLabels = $values[0];
+
+        Assert::eq($columnLabels[0], 'email', sprintf(
+            "The first column in the '%s' tab must be 'email'",
+            $tabName
+        ));
+        Assert::eq($columnLabels[1], 'groups', sprintf(
+            "The second column in the '%s' tab must be 'groups'",
+            $tabName
+        ));
+        Assert::eq(
+            count($columnLabels),
+            2,
+            'There should only be two columns with values'
+        );
+
+        $data = [];
+        for ($i = 1; $i < count($values); $i++) {
+            $email = trim($values[$i][0]);
+            $groups = trim($values[$i][1] ?? '');
+            $data[$email] = $groups;
+        }
+        return $data;
+    }
+}

application/common/components/Sheets.php

@@ -2,6 +2,7 @@
 
 namespace common\components;
 
+use Google\Service\Exception;
 use InvalidArgumentException;
 use yii\base\Component;
 use yii\helpers\Json;
@@ -127,6 +128,43 @@ public function getHeader()
         return $header['values'][0];
     }
 
+    /**
+     * Get all the values from the specified tab in this Google Sheet.
+     *
+     * @param string $tabName
+     * @param string $range
+     * @return array[]
+     *     Example:
+     *     ```
+     *     [
+     *         [
+     *             "A1's value",
+     *             "B1's value"
+     *         ],
+     *         [
+     *             "A2's value",
+     *             "B2's value"
+     *         ],
+     *         [
+     *             "A3's value",
+     *             "B3's value"
+     *         ]
+     *     ]
+     *     ```
+     * @throws Exception
+     */
+    public function getValuesFromTab(string $tabName, string $range = 'A:ZZ'): array
+    {
+        $client = $this->getGoogleClient();
+
+        $valueRange = $client->spreadsheets_values->get(
+            $this->spreadsheetId,
+            $tabName . '!' . $range
+        );
+
+        return $valueRange->values;
+    }
+
     /**
      * @param string[] $header
      * @param array $records

application/common/config/main.php

@@ -239,6 +239,7 @@
             ],
             Env::getArrayFromPrefix('ABANDONED_USER_')
         ),
+        'externalGroupsSyncSets' => Env::getArrayFromPrefix('EXTERNAL_GROUPS_SYNC_'),
         'googleAnalytics'               => [
             'apiSecret' => Env::get('GA_API_SECRET'),
             'measurementId' => Env::get('GA_MEASUREMENT_ID'),

application/common/models/User.php

@@ -1062,9 +1062,9 @@ public static function updateUsersExternalGroups(
             $successful = $user->updateExternalGroups($appPrefix, $groupsForPrefix);
             if (! $successful) {
                 $errors[] = sprintf(
-                    "Failed to update external groups for %s: \n%s",
+                    'Failed to update external groups for %s: %s',
                     $email,
-                    json_encode($user->getFirstErrors(), JSON_PRETTY_PRINT)
+                    join(' / ', $user->getFirstErrors())
                 );
             }
         }
@@ -1084,9 +1084,9 @@ public function updateExternalGroups(string $appPrefix, string $csvAppExternalGr
             if (! str_starts_with($appExternalGroup, $appPrefix . '-')) {
                 $this->addErrors([
                     'groups_external' => sprintf(
-                        'The given group %s does not start with the given prefix (%s)',
-                        json_encode($appExternalGroup),
-                        json_encode($appPrefix)
+                        'The given group (%s) does not start with the given prefix (%s)',
+                        $appExternalGroup,
+                        $appPrefix
                     ),
                 ]);
                 return false;
@@ -1096,17 +1096,7 @@ public function updateExternalGroups(string $appPrefix, string $csvAppExternalGr
         $this->addInMemoryExternalGroups($appExternalGroups);
 
         $this->scenario = self::SCENARIO_UPDATE_USER;
-        $saved = $this->save(true, ['groups_external']);
-        if ($saved) {
-            return true;
-        } else {
-            Yii::warning(sprintf(
-                'Failed to update external groups for %s: %s',
-                $this->email,
-                join(', ', $this->getFirstErrors())
-            ));
-            return false;
-        }
+        return $this->save(true, ['groups_external']);
     }
 
     /**

application/console/controllers/CronController.php

@@ -2,6 +2,7 @@
 
 namespace console\controllers;
 
+use common\components\ExternalGroupsSync;
 use common\helpers\Utils;
 use common\models\Invite;
 use common\models\Method;
@@ -137,6 +138,16 @@ public function actionExportToSheets()
         User::exportToSheets();
     }
 
+    /**
+     * Sync external groups from Google Sheets
+     */
+    public function actionSyncExternalGroups()
+    {
+        ExternalGroupsSync::syncAllSets(
+            \Yii::$app->params['externalGroupsSyncSets'] ?? []
+        );
+    }
+
     /**
      * Run all cron tasks, catching and logging errors to allow remaining tasks to run after an exception
      */

local.env.dist

@@ -337,3 +337,11 @@ GOOGLE_delegatedAdmin=admin@example.com
 
 # spreadsheet ID
 GOOGLE_spreadsheetId=putAnActualSheetIDHerejD70xAjqPnOCHlDK3YomH
+
+# === external groups sync sets === #
+
+# EXTERNAL_GROUPS_SYNC_set1AppPrefix=
+# EXTERNAL_GROUPS_SYNC_set1GoogleSheetId=
+# EXTERNAL_GROUPS_SYNC_set2AppPrefix=
+# EXTERNAL_GROUPS_SYNC_set2GoogleSheetId=
+# ... with as many sets as you want.