From 930e932d5d77cb7ed8e8d083d126eebd01cc166e Mon Sep 17 00:00:00 2001 From: Matt H Date: Wed, 4 Sep 2024 15:17:27 -0400 Subject: [PATCH 1/5] Add partial implementation of cron-based external-groups sync --- .../common/components/ExternalGroupsSync.php | 66 +++++++++++++++++++ application/common/config/main.php | 1 + .../console/controllers/CronController.php | 11 ++++ local.env.dist | 8 +++ 4 files changed, 86 insertions(+) create mode 100644 application/common/components/ExternalGroupsSync.php diff --git a/application/common/components/ExternalGroupsSync.php b/application/common/components/ExternalGroupsSync.php new file mode 100644 index 00000000..2fc33bc7 --- /dev/null +++ b/application/common/components/ExternalGroupsSync.php @@ -0,0 +1,66 @@ + Env::getArrayFromPrefix('EXTERNAL_GROUPS_SYNC_'), 'googleAnalytics' => [ 'apiSecret' => Env::get('GA_API_SECRET'), 'measurementId' => Env::get('GA_MEASUREMENT_ID'), diff --git a/application/console/controllers/CronController.php b/application/console/controllers/CronController.php index 22f15390..01c4648c 100644 --- a/application/console/controllers/CronController.php +++ b/application/console/controllers/CronController.php @@ -2,6 +2,7 @@ namespace console\controllers; +use common\components\ExternalGroupsSync; use common\helpers\Utils; use common\models\Invite; use common\models\Method; @@ -137,6 +138,16 @@ public function actionExportToSheets() User::exportToSheets(); } + /** + * Sync external groups from Google Sheets + */ + public function actionSyncExternalGroups() + { + ExternalGroupsSync::syncAllSets( + \Yii::$app->params['externalGroupsSyncSets'] ?? [] + ); + } + /** * Run all cron tasks, catching and logging errors to allow remaining tasks to run after an exception */ diff --git a/local.env.dist b/local.env.dist index 6d2604aa..c460df7a 100644 --- a/local.env.dist +++ b/local.env.dist @@ -337,3 +337,11 @@ GOOGLE_delegatedAdmin=admin@example.com # spreadsheet ID GOOGLE_spreadsheetId=putAnActualSheetIDHerejD70xAjqPnOCHlDK3YomH + +# === external groups sync sets === # + +# EXTERNAL_GROUPS_SYNC_set1AppPrefix= +# EXTERNAL_GROUPS_SYNC_set1GoogleSheetId= +# EXTERNAL_GROUPS_SYNC_set2AppPrefix= +# EXTERNAL_GROUPS_SYNC_set2GoogleSheetId= +# ... with as many sets as you want. From aa2056781303af979ac440970de98d763ebf0764 Mon Sep 17 00:00:00 2001 From: Matt H Date: Wed, 4 Sep 2024 16:40:41 -0400 Subject: [PATCH 2/5] Combine external-groups sync error message to a single line --- application/common/components/ExternalGroupsSync.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/common/components/ExternalGroupsSync.php b/application/common/components/ExternalGroupsSync.php index 2fc33bc7..c420525c 100644 --- a/application/common/components/ExternalGroupsSync.php +++ b/application/common/components/ExternalGroupsSync.php @@ -57,9 +57,9 @@ private static function syncSet(string $appPrefix, string $googleSheetId) if (! empty($errors)) { Yii::error(sprintf( - "Errors that occurred while syncing %s external groups: \n%s", + 'Errors that occurred while syncing %s external groups: %s', $appPrefix, - join("\n", $errors) + join(" / ", $errors) )); } } From cbc532fc6eb5fe365034a096e2966b4fc1ff12e3 Mon Sep 17 00:00:00 2001 From: Matt H Date: Wed, 4 Sep 2024 16:56:48 -0400 Subject: [PATCH 3/5] Clean up some external-groups log message formatting --- application/common/components/ExternalGroupsSync.php | 6 +++--- application/common/models/User.php | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/application/common/components/ExternalGroupsSync.php b/application/common/components/ExternalGroupsSync.php index c420525c..213ec761 100644 --- a/application/common/components/ExternalGroupsSync.php +++ b/application/common/components/ExternalGroupsSync.php @@ -37,8 +37,8 @@ public static function syncAllSets(array $syncSetsParams) )); } else { Yii::warning(sprintf( - 'Syncing %s external groups from Google Sheet (%s)...', - json_encode($appPrefix), + "Syncing '%s' external groups from Google Sheet (%s)...", + $appPrefix, $googleSheetId )); self::syncSet($appPrefix, $googleSheetId); @@ -51,7 +51,7 @@ private static function syncSet(string $appPrefix, string $googleSheetId) $desiredExternalGroups = self::getExternalGroupsFromGoogleSheet($googleSheetId); $errors = User::syncExternalGroups($appPrefix, $desiredExternalGroups); Yii::warning(sprintf( - 'Ran sync for %s external groups.', + "Ran sync for '%s' external groups.", $appPrefix )); diff --git a/application/common/models/User.php b/application/common/models/User.php index b9cb9e0f..2bcaf259 100644 --- a/application/common/models/User.php +++ b/application/common/models/User.php @@ -1060,9 +1060,9 @@ public static function syncExternalGroups(string $appPrefix, array $desiredExter $successful = $user->updateExternalGroups($appPrefix, $groupsForPrefix); if (! $successful) { $errors[] = sprintf( - "Failed to update external groups for %s: \n%s", + 'Failed to update external groups for %s: %s', $email, - json_encode($user->getFirstErrors(), JSON_PRETTY_PRINT) + join(' / ', $user->getFirstErrors()) ); } } @@ -1082,9 +1082,9 @@ public function updateExternalGroups(string $appPrefix, string $csvAppExternalGr if (! str_starts_with($appExternalGroup, $appPrefix . '-')) { $this->addErrors([ 'groups_external' => sprintf( - 'The given group %s does not start with the given prefix (%s)', - json_encode($appExternalGroup), - json_encode($appPrefix) + 'The given group (%s) does not start with the given prefix (%s)', + $appExternalGroup, + $appPrefix ), ]); return false; From ca040adacf83f0c602c5e9db0212e99da89463b7 Mon Sep 17 00:00:00 2001 From: Matt H Date: Wed, 4 Sep 2024 16:57:17 -0400 Subject: [PATCH 4/5] Remove redundant log message from `$user->updateExternalGroups()` --- application/common/models/User.php | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/application/common/models/User.php b/application/common/models/User.php index 2bcaf259..7e866966 100644 --- a/application/common/models/User.php +++ b/application/common/models/User.php @@ -1094,17 +1094,7 @@ public function updateExternalGroups(string $appPrefix, string $csvAppExternalGr $this->addInMemoryExternalGroups($appExternalGroups); $this->scenario = self::SCENARIO_UPDATE_USER; - $saved = $this->save(true, ['groups_external']); - if ($saved) { - return true; - } else { - Yii::warning(sprintf( - 'Failed to update external groups for %s: %s', - $this->email, - join(', ', $this->getFirstErrors()) - )); - return false; - } + return $this->save(true, ['groups_external']); } /** From ccace395f7c17c7bf28e40e72e551a9e3519c382 Mon Sep 17 00:00:00 2001 From: Matt H Date: Thu, 5 Sep 2024 11:57:16 -0400 Subject: [PATCH 5/5] Use the data from the appropriate tab of the specified Google Sheet --- .../common/components/ExternalGroupsSync.php | 46 ++++++++++++++++++- application/common/components/Sheets.php | 38 +++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/application/common/components/ExternalGroupsSync.php b/application/common/components/ExternalGroupsSync.php index 213ec761..4e59d667 100644 --- a/application/common/components/ExternalGroupsSync.php +++ b/application/common/components/ExternalGroupsSync.php @@ -3,6 +3,7 @@ namespace common\components; use common\models\User; +use Webmozart\Assert\Assert; use Yii; use yii\base\Component; @@ -49,7 +50,7 @@ public static function syncAllSets(array $syncSetsParams) private static function syncSet(string $appPrefix, string $googleSheetId) { $desiredExternalGroups = self::getExternalGroupsFromGoogleSheet($googleSheetId); - $errors = User::syncExternalGroups($appPrefix, $desiredExternalGroups); + $errors = User::updateUsersExternalGroups($appPrefix, $desiredExternalGroups); Yii::warning(sprintf( "Ran sync for '%s' external groups.", $appPrefix @@ -63,4 +64,47 @@ private static function syncSet(string $appPrefix, string $googleSheetId) )); } } + + /** + * Get the desired external-group values, indexed by email address, from the + * specified Google Sheet, from the tab named after this IDP's code name + * (i.e. the name used in this IDP's subdomain). + * + * @throws \Google\Service\Exception + */ + private static function getExternalGroupsFromGoogleSheet(string $googleSheetId): array + { + $googleSheetsClient = new Sheets([ + 'applicationName' => Yii::$app->params['google']['applicationName'], + 'jsonAuthFilePath' => Yii::$app->params['google']['jsonAuthFilePath'], + 'jsonAuthString' => Yii::$app->params['google']['jsonAuthString'], + 'spreadsheetId' => $googleSheetId, + ]); + $tabName = Yii::$app->params['idpName']; + + $values = $googleSheetsClient->getValuesFromTab($tabName); + $columnLabels = $values[0]; + + Assert::eq($columnLabels[0], 'email', sprintf( + "The first column in the '%s' tab must be 'email'", + $tabName + )); + Assert::eq($columnLabels[1], 'groups', sprintf( + "The second column in the '%s' tab must be 'groups'", + $tabName + )); + Assert::eq( + count($columnLabels), + 2, + 'There should only be two columns with values' + ); + + $data = []; + for ($i = 1; $i < count($values); $i++) { + $email = trim($values[$i][0]); + $groups = trim($values[$i][1] ?? ''); + $data[$email] = $groups; + } + return $data; + } } diff --git a/application/common/components/Sheets.php b/application/common/components/Sheets.php index 3c989847..848e21dd 100644 --- a/application/common/components/Sheets.php +++ b/application/common/components/Sheets.php @@ -2,6 +2,7 @@ namespace common\components; +use Google\Service\Exception; use InvalidArgumentException; use yii\base\Component; use yii\helpers\Json; @@ -127,6 +128,43 @@ public function getHeader() return $header['values'][0]; } + /** + * Get all the values from the specified tab in this Google Sheet. + * + * @param string $tabName + * @param string $range + * @return array[] + * Example: + * ``` + * [ + * [ + * "A1's value", + * "B1's value" + * ], + * [ + * "A2's value", + * "B2's value" + * ], + * [ + * "A3's value", + * "B3's value" + * ] + * ] + * ``` + * @throws Exception + */ + public function getValuesFromTab(string $tabName, string $range = 'A:ZZ'): array + { + $client = $this->getGoogleClient(); + + $valueRange = $client->spreadsheets_values->get( + $this->spreadsheetId, + $tabName . '!' . $range + ); + + return $valueRange->values; + } + /** * @param string[] $header * @param array $records