Merge pull request #114 from fivetran/bug/agent-work-time-schedule-ho…

…liday-3

.buildkite/hooks/pre-command

@@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C
 export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
 export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
+export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")

.buildkite/pipeline.yml

@@ -69,5 +69,6 @@ steps:
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
+            - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
       bash .buildkite/scripts/run_models.sh databricks

.gitignore

@@ -1,4 +1,4 @@
-
+env/
 target/
 dbt_modules/
 logs/

CHANGELOG.md

@@ -1,3 +1,30 @@
+# dbt_zendesk v0.12.0
+
+This release includes fixes to issues introduced in v0.11.0-v0.11.1 surrounding the incorporation of schedule holidays.
+
+Special thanks to [@cth84](https://github.com/cth84) and [@nschimmoller](https://github.com/nschimmoller) for working with us to figure out some seriously tricky bugs!
+
+## Bug Fix
+- Adjusted the gap-merging logic in `int_zendesk__schedule_spine` to look forward in time instead of backward. This allows the model to take Daylight Savings Time into account when merging gaps. Previously, schedule periods with different `start_time_utc`s (because of DST) were getting merged together ([PR #114](https://github.com/fivetran/dbt_zendesk/pull/114)).
+  - Also removed the `double_gap` logic as it was rendered unnecessary by the above change.
+- In all of our intermediate business hour models, adjusted the join logic in the `intercepted_periods` CTE, where we associate ticket weekly periods with the appropriate business schedule period. Previously, we did so by comparing the ticket's `status_valid_starting_at` and `status_valid_ending_at` fields to the schedule's `valid_from` and `valid_until` dates. This was causing fanout in certain cases, as we need to take the ticket-status's `week_number` into account because it is part of the grain of the CTE we are joining ([PR #114](https://github.com/fivetran/dbt_zendesk/pull/114)).
+- Adjusted the way we calculate the end of holidays in `int_zendesk__schedule_spine`. Previously, we calculated the end of holiday day by adding `24*60*60-1` seconds (making the end the last second of the same day) to the start of the holiday. This previously worked because our downstream joins for calculating business metrics were inclusive (ie `>=` instead of `>`). We've updated these joins to be exclusive (ie `>` or `<`), so we've set the end of the holiday to truly be the end of the day instead of a second prior ([PR #114](https://github.com/fivetran/dbt_zendesk/pull/114)).
+- Updated `int_zendesk__requester_wait_time_filtered_statuses` to include the `hold` status, as zendesk updated `on-hold` to just `hold` ([PR #114](https://github.com/fivetran/dbt_zendesk/pull/114)).
+- Updates the logic in `int_zendesk__reply_time_combined` to bring through the correct `sla_event_id` records to the end `zendesk__sla_policies` model. ([PR #108](https://github.com/fivetran/dbt_zendesk/issues/108))
+   - Originally, duplicate `sla_event_id` records were being persisted because the upstream `filtered_reply_times` CTE did not include for all scenarios. With this update, the CTE will filter for the following scenarios:
+       - Ticket is replied to between a schedule window
+       - Ticket is replied to before a schedule window and no business minutes have been spent on it
+       - Ticket is not replied to and therefore active. But only bring through the active SLA record that is most recent (after the last SLA schedule starts but before the next)
+- Updated the ordering within the `int_zendesk__comments_enriched` model logic to also take into account when two comments are posted at the exact same time. Previously, the next comment would be picked arbitrarily. However, we now use the `commenter_role` as the tie breaker giving preference to the `end-user` as they will likely be the first commenter when two comments are posted at the exact same time. ([PR #114](https://github.com/fivetran/dbt_zendesk/pull/114))
+- Modified the requester and agent wait time `sla_elapsed_time` metric calculations within the `zendesk__sla_policies` to capture the max `running_total_scheduled_minutes` record as opposed to the cumulative sum. Max more accurately represents the upstream data as it is presented in a rolling sum in the previous intermediate models. ([PR #114](https://github.com/fivetran/dbt_zendesk/pull/114))
+
+## Dependency Updates
+- The `dbt-date` dependency has been updated to reflect the recommended latest range, [">=0.9.0", "<1.0.0"]. This will help to avoid upstream dependency conflicts. ([PR #113](https://github.com/fivetran/dbt_zendesk/pull/113))
+
+## Contributors: 
+- [@nschimmoller](https://github.com/nschimmoller) ([#108](https://github.com/fivetran/dbt_zendesk/issues/108))
+- [@cth84](https://github.com/cth84) ([#107](https://github.com/fivetran/dbt_zendesk/issues/107))
+
 # dbt_zendesk v0.11.2
 
 ## Rollback
@@ -187,4 +214,4 @@ Tiny release ahead!
 - [csaroff](https://github.com/csaroff) ([#47](https://github.com/fivetran/dbt_zendesk/pull/47))
 - [jackiexsun](https://github.com/jackiexsun) ([#42](https://github.com/fivetran/dbt_zendesk/pull/42))
 - [emiliedecherney](https://github.com/emiliedecherney) ([#50](https://github.com/fivetran/dbt_zendesk/pull/50))
-- [gareginordyan](https://github.com/gareginordyan) ([#44](https://github.com/fivetran/dbt_zendesk/pull/44))
+- [gareginordyan](https://github.com/gareginordyan) ([#44](https://github.com/fivetran/dbt_zendesk/pull/44))

README.md

@@ -59,7 +59,7 @@ Include the following zendesk package version in your `packages.yml` file:
 ```yml
 packages:
   - package: fivetran/zendesk
-    version: [">=0.11.0", "<0.12.0"]
+    version: [">=0.12.0", "<0.13.0"]
 
 ```
 > **Note**: Do not include the Zendesk source package. The Zendesk transform package already has a dependency on the source in its own `packages.yml` file.
@@ -74,7 +74,7 @@ vars:
 ```
 
 ## Step 4: Disable models for non-existent sources
-This package takes into consideration that not every Zendesk account utilizes the `schedule`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable:
+This package takes into consideration that not every Zendesk account utilizes the `schedule`, `schedule_holiday`, `ticket_schedule` `daylight_time`, `time_zone`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. Add variables for only the tables you want to disable:
 ```yml
 vars:
     using_schedules:            False         #Disable if you are not using schedules
@@ -85,7 +85,6 @@ vars:
 ```
 
 ## (Optional) Step 5: Additional configurations
-<details><summary>Expand for configurations</summary>
 
 ### Adding passthrough columns
 This package includes all source columns defined in the staging models. However, the `stg_zendesk__ticket` model allows for additional columns to be added using a pass-through column variable. This is extremely useful if you'd like to include custom fields to the package.
@@ -161,7 +160,6 @@ If an individual source table has a different name than the package expects, add
 vars:
     zendesk_<default_source_table_name>_identifier: your_table_name 
 ```
-</details>
 
 ## (Optional) Step 6: Orchestrate your models with Fivetran Transformations for dbt Core™
 <details><summary>Expand for details</summary>
@@ -177,7 +175,7 @@ This dbt package is dependent on the following dbt packages. Please be aware tha
 ```yml
 packages:
     - package: fivetran/zendesk_source
-      version: [">=0.8.0", "<0.9.0"]
+      version: [">=0.9.0", "<0.10.0"]
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]
@@ -189,7 +187,7 @@ packages:
       version: [">=0.3.0", "<0.4.0"]
 
     - package: calogica/dbt_date
-      version: [">=0.7.0", "<0.8.0"]
+      version: [">=0.9.0", "<1.0.0"]
 ```
 # 🙌 How is this package maintained and can I contribute?
 ## Package Maintenance

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'zendesk'
-version: '0.11.2'
+version: '0.12.0'
 
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
@@ -35,6 +35,7 @@ vars:
     organization_tag: "{{ ref('stg_zendesk__organization_tag') }}"
     organization: "{{ ref('stg_zendesk__organization') }}"
     schedule: "{{ ref('stg_zendesk__schedule') }}"
+    schedule_holiday: "{{ ref('stg_zendesk__schedule_holiday') }}"
     ticket: "{{ ref('stg_zendesk__ticket') }}"
     ticket_form_history: "{{ ref('stg_zendesk__ticket_form_history') }}"
     ticket_comment: "{{ ref('stg_zendesk__ticket_comment') }}"

integration_tests/ci/sample.profiles.yml

@@ -45,10 +45,10 @@ integration_tests:
       schema: zendesk_integration_tests_2
       threads: 8
     databricks:
-      catalog: null
+      catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
       schema: zendesk_integration_tests_2
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
-      type: databricks
+      type: databricks

integration_tests/dbt_project.yml

@@ -1,8 +1,7 @@
 config-version: 2
 
 name: 'zendesk_integration_tests'
-version: '0.11.2'
-
+version: '0.12.0'
 
 profile: 'integration_tests'
 
@@ -11,6 +10,7 @@ vars:
   zendesk_source:
     zendesk_organization_identifier:           "organization_data"
     zendesk_schedule_identifier:               "schedule_data"
+    zendesk_schedule_holiday_identifier:       "schedule_holiday_data"
     zendesk_ticket_identifier:                 "ticket_data"
     zendesk_ticket_comment_identifier:         "ticket_comment_data"
     zendesk_ticket_field_history_identifier:   "ticket_field_history_data"
@@ -49,6 +49,10 @@ seeds:
         end_time_utc: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
         start_time_utc: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
         created_at: timestamp
+    schedule_holiday_data:
+      +column_types:
+        id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
+        schedule_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
     ticket_comment_data:
       +column_types:
         id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}"
@@ -125,6 +129,11 @@ seeds:
         created_at: timestamp
         last_login_at: timestamp
       +enabled: "{{ true if target.type == 'snowflake' else false }}"
+    user_tag_data_snowflake:
+      +enabled: "{{ true if target.type == 'snowflake' else false }}"
+    organization_tag_data_snowflake:
+      +enabled: "{{ true if target.type == 'snowflake' else false }}"
+
 
 dispatch:
   - macro_namespace: dbt_utils

integration_tests/requirements.txt

@@ -4,4 +4,6 @@ dbt-redshift>=1.3.0,<2.0.0
 dbt-postgres>=1.3.0,<2.0.0
 dbt-spark>=1.3.0,<2.0.0
 dbt-spark[PyHive]>=1.3.0,<2.0.0
-dbt-databricks>=1.3.0,<2.0.0
+dbt-databricks>=1.3.0,<2.0.0
+
+oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437

integration_tests/seeds/schedule_holiday_data.csv

@@ -0,0 +1,3 @@
+id,schedule_id,_fivetran_deleted,_fivetran_synced,end_date,name,start_date
+1163028,360000,FALSE,2023-01-19 21:53:06.281,2022-12-27,Test Holiday,2022-12-26
+1163027,360000,FALSE,2023-01-19 21:53:06.281,2022-12-29,Test Holiday 2,2022-12-29

models/agent_work_time/int_zendesk__ticket_work_time_business.sql

@@ -33,6 +33,7 @@ with ticket_historical_status as (
     from ticket_historical_status
     left join ticket_schedules
       on ticket_historical_status.ticket_id = ticket_schedules.ticket_id
+      -- making sure there is indeed real overlap
       where {{ dbt.datediff('greatest(valid_starting_at, schedule_created_at)', 'least(valid_ending_at, schedule_invalidated_at)', 'second') }} > 0
 
 ), ticket_full_solved_time as (
@@ -54,7 +55,9 @@ with ticket_historical_status as (
               'ticket_status_crossed_with_schedule.status_schedule_start',
               'ticket_status_crossed_with_schedule.status_schedule_end',
               'second') }} /60
-            ) as raw_delta_in_minutes
+            ) as raw_delta_in_minutes,
+    {{ dbt_date.week_start('ticket_status_crossed_with_schedule.status_schedule_start','UTC') }} as start_week_date
+
     from ticket_status_crossed_with_schedule
     {{ dbt_utils.group_by(n=7) }}
 
@@ -66,7 +69,7 @@ with ticket_historical_status as (
     -- because time is reported in minutes since the beginning of the week, we have to split up time spent on the ticket into calendar weeks
     select 
       ticket_full_solved_time.*,
-      generated_number - 1 as week_number
+      cast(generated_number - 1 as {{ dbt.type_int() }}) as week_number
     from ticket_full_solved_time
     cross join weeks
     where floor((start_time_in_minutes_from_week + raw_delta_in_minutes) / (7*24*60)) >= generated_number -1
@@ -76,8 +79,10 @@ with ticket_historical_status as (
     select
 
       weeks_cross_ticket_full_solved_time.*,
-      greatest(0, start_time_in_minutes_from_week - week_number * (7*24*60)) as ticket_week_start_time,
-      least(start_time_in_minutes_from_week + raw_delta_in_minutes - week_number * (7*24*60), (7*24*60)) as ticket_week_end_time
+      -- for each week, at what minute do we start counting?
+      cast(greatest(0, start_time_in_minutes_from_week - week_number * (7*24*60)) as {{ dbt.type_int() }}) as ticket_week_start_time,
+      -- for each week, at what minute do we stop counting?
+      cast(least(start_time_in_minutes_from_week + raw_delta_in_minutes - week_number * (7*24*60), (7*24*60)) as {{ dbt.type_int() }}) as ticket_week_end_time
 
     from weeks_cross_ticket_full_solved_time
 
@@ -94,12 +99,14 @@ with ticket_historical_status as (
       schedule.end_time_utc as schedule_end_time,
       least(ticket_week_end_time, schedule.end_time_utc) - greatest(weekly_periods.ticket_week_start_time, schedule.start_time_utc) as scheduled_minutes
     from weekly_periods
-    join schedule on ticket_week_start_time <= schedule.end_time_utc 
+    join schedule on 
+      ticket_week_start_time <= schedule.end_time_utc 
       and ticket_week_end_time >= schedule.start_time_utc
       and weekly_periods.schedule_id = schedule.schedule_id
       -- this chooses the Daylight Savings Time or Standard Time version of the schedule
-      and weekly_periods.status_valid_ending_at >= cast(schedule.valid_from as {{ dbt.type_timestamp() }})
-      and weekly_periods.status_valid_starting_at < cast(schedule.valid_until as {{ dbt.type_timestamp() }}) 
+      -- We have everything calculated within a week, so take us to the appropriate week first by adding the week_number * minutes-in-a-week to the minute-mark where we start and stop counting for the week
+      and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_end_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) > cast(schedule.valid_from as {{ dbt.type_timestamp() }})
+      and cast( {{ dbt.dateadd(datepart='minute', interval='week_number * (7*24*60) + ticket_week_start_time', from_date_or_timestamp='start_week_date') }} as {{ dbt.type_timestamp() }}) < cast(schedule.valid_until as {{ dbt.type_timestamp() }})
 
 ), business_minutes as (
 
@@ -113,7 +120,11 @@ with ticket_historical_status as (
       case when ticket_status in ('new', 'open') then scheduled_minutes
           else 0 end as agent_work_time_in_minutes,
       case when ticket_status in ('hold') then scheduled_minutes
-          else 0 end as on_hold_time_in_minutes
+          else 0 end as on_hold_time_in_minutes,
+      case when ticket_status = 'new' then scheduled_minutes
+          else 0 end as new_status_duration_minutes,
+      case when ticket_status = 'open' then scheduled_minutes
+          else 0 end as open_status_duration_minutes
     from intercepted_periods
 
 )
@@ -123,6 +134,8 @@ with ticket_historical_status as (
       sum(agent_wait_time_in_minutes) as agent_wait_time_in_business_minutes,
       sum(requester_wait_time_in_minutes) as requester_wait_time_in_business_minutes,
       sum(agent_work_time_in_minutes) as agent_work_time_in_business_minutes,
-      sum(on_hold_time_in_minutes) as on_hold_time_in_business_minutes
+      sum(on_hold_time_in_minutes) as on_hold_time_in_business_minutes,
+      sum(new_status_duration_minutes) as new_status_duration_in_business_minutes,
+      sum(open_status_duration_minutes) as open_status_duration_in_business_minutes
     from business_minutes
     group by 1

models/intermediate/int_zendesk__comment_metrics.sql

@@ -31,7 +31,11 @@ comment_counts as (
         count(*) as total_comments,
         count(distinct case when commenter_role = 'internal_comment'
             then user_id
-                end) as count_ticket_handoffs
+                end) as count_ticket_handoffs,
+        sum(case when commenter_role = 'internal_comment' and is_public = true and previous_commenter_role != 'first_comment'
+            then 1
+            else 0
+                end) as count_agent_replies
     from ticket_comments
 
     group by 1, 2