feat(insights): Add cumulative retention (#24240)

frontend/src/queries/nodes/InsightQuery/utils/filtersToQueryNode.ts

@@ -436,6 +436,7 @@ export const retentionFilterToQuery = (filters: Partial<RetentionFilterType>): R
         targetEntity: sanitizeRetentionEntity(filters.target_entity),
         period: filters.period,
         showMean: filters.show_mean,
+        cumulative: filters.cumulative,
     })
     // TODO: query.aggregation_group_type_index
 }

frontend/src/queries/nodes/InsightQuery/utils/queryNodeToFilter.ts

@@ -272,12 +272,14 @@ export const queryNodeToFilter = (query: InsightQueryNode): Partial<FilterType>
         camelCasedRetentionProps.target_entity = queryCopy.retentionFilter?.targetEntity
         camelCasedRetentionProps.total_intervals = queryCopy.retentionFilter?.totalIntervals
         camelCasedRetentionProps.show_mean = queryCopy.retentionFilter?.showMean
+        camelCasedRetentionProps.cumulative = queryCopy.retentionFilter?.cumulative
         delete queryCopy.retentionFilter?.retentionReference
         delete queryCopy.retentionFilter?.retentionType
         delete queryCopy.retentionFilter?.returningEntity
         delete queryCopy.retentionFilter?.targetEntity
         delete queryCopy.retentionFilter?.totalIntervals
         delete queryCopy.retentionFilter?.showMean
+        delete queryCopy.retentionFilter?.cumulative
     } else if (isPathsQuery(queryCopy)) {
         camelCasedPathsProps.edge_limit = queryCopy.pathsFilter?.edgeLimit
         camelCasedPathsProps.paths_hogql_expression = queryCopy.pathsFilter?.pathsHogQLExpression

frontend/src/queries/nodes/InsightViz/InsightDisplayConfig.tsx

@@ -17,6 +17,7 @@ import { ScalePicker } from 'scenes/insights/EditorFilters/ScalePicker'
 import { ShowLegendFilter } from 'scenes/insights/EditorFilters/ShowLegendFilter'
 import { ValueOnSeriesFilter } from 'scenes/insights/EditorFilters/ValueOnSeriesFilter'
 import { InsightDateFilter } from 'scenes/insights/filters/InsightDateFilter'
+import { RetentionCumulativeCheckbox } from 'scenes/insights/filters/RetentionCumulativeCheckbox'
 import { RetentionMeanCheckbox } from 'scenes/insights/filters/RetentionMeanCheckbox'
 import { RetentionReferencePicker } from 'scenes/insights/filters/RetentionReferencePicker'
 import { insightLogic } from 'scenes/insights/insightLogic'
@@ -146,6 +147,7 @@ export function InsightDisplayConfig(): JSX.Element {
                         <RetentionDatePicker />
                         <RetentionReferencePicker />
                         <RetentionMeanCheckbox />
+                        <RetentionCumulativeCheckbox />
                     </ConfigFilter>
                 )}
 

frontend/src/queries/schema.json

@@ -8090,6 +8090,9 @@
         "RetentionFilter": {
             "additionalProperties": false,
             "properties": {
+                "cumulative": {
+                    "type": "boolean"
+                },
                 "period": {
                     "$ref": "#/definitions/RetentionPeriod",
                     "default": "Day"
@@ -8121,6 +8124,9 @@
             "additionalProperties": false,
             "description": "`RetentionFilterType` minus everything inherited from `FilterType`",
             "properties": {
+                "cumulative": {
+                    "type": "boolean"
+                },
                 "period": {
                     "$ref": "#/definitions/RetentionPeriod"
                 },

frontend/src/queries/schema.ts

@@ -868,6 +868,7 @@ export type RetentionFilter = {
     /** @default Day */
     period?: RetentionFilterLegacy['period']
     showMean?: RetentionFilterLegacy['show_mean']
+    cumulative?: RetentionFilterLegacy['cumulative']
 }
 
 export interface RetentionValue {

frontend/src/scenes/insights/RetentionDatePicker.tsx

@@ -26,7 +26,7 @@ export function RetentionDatePicker(): JSX.Element {
                 clearable
                 buttonProps={{
                     tooltip: 'Cohorts up to this end date',
-                    type: 'tertiary',
+                    type: 'secondary',
                     sideIcon: null,
                     size: 'small',
                 }}

frontend/src/scenes/insights/filters/RetentionCumulativeCheckbox.tsx

@@ -0,0 +1,45 @@
+import { IconInfo } from '@posthog/icons'
+import { LemonSwitch, Tooltip } from '@posthog/lemon-ui'
+import { useActions, useValues } from 'kea'
+import { insightLogic } from 'scenes/insights/insightLogic'
+import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic'
+
+export function RetentionCumulativeCheckbox(): JSX.Element | null {
+    const { insightProps, canEditInsight } = useValues(insightLogic)
+
+    const { retentionFilter } = useValues(insightVizDataLogic(insightProps))
+    const { updateInsightFilter } = useActions(insightVizDataLogic(insightProps))
+
+    const cumulativeRetention = retentionFilter?.cumulative || false
+
+    if (!canEditInsight) {
+        return null
+    }
+
+    return (
+        <LemonSwitch
+            onChange={(cumulative: boolean) => {
+                updateInsightFilter({ cumulative })
+            }}
+            checked={cumulativeRetention}
+            label={
+                <span className="font-normal">
+                    Rolling retention
+                    <Tooltip
+                        title={
+                            <>
+                                Rolling, or unbounded, retention includes any subsequent time period, instead of only
+                                the next period. For example, if a user is comes back on day 7, they are counted in all
+                                previous retention periods.
+                            </>
+                        }
+                    >
+                        <IconInfo className="w-4 info-indicator" />
+                    </Tooltip>
+                </span>
+            }
+            bordered
+            size="small"
+        />
+    )
+}

frontend/src/scenes/insights/filters/RetentionMeanCheckbox.tsx

@@ -1,4 +1,4 @@
-import { LemonCheckbox } from '@posthog/lemon-ui'
+import { LemonSwitch } from '@posthog/lemon-ui'
 import { useActions, useValues } from 'kea'
 import { insightLogic } from 'scenes/insights/insightLogic'
 import { insightVizDataLogic } from 'scenes/insights/insightVizDataLogic'
@@ -16,7 +16,7 @@ export function RetentionMeanCheckbox(): JSX.Element | null {
     }
 
     return (
-        <LemonCheckbox
+        <LemonSwitch
             onChange={(showMean: boolean) => {
                 updateInsightFilter({ showMean })
             }}

frontend/src/scenes/insights/utils/cleanFilters.ts

@@ -306,6 +306,7 @@ export function cleanFilters(
             breakdown_type: filters.breakdown_type,
             retention_reference: filters.retention_reference,
             show_mean: filters.show_mean,
+            cumulative: filters.cumulative,
             total_intervals: Math.min(Math.max(filters.total_intervals ?? 11, 0), 100),
             ...(filters.aggregation_group_type_index != undefined
                 ? { aggregation_group_type_index: filters.aggregation_group_type_index }

frontend/src/scenes/insights/utils/compareInsightQuery.ts

@@ -61,6 +61,7 @@ const cleanInsightQuery = (query: InsightQueryNode, ignoreVisualizationOnlyChang
             toggledLifecycles: undefined,
             showLabelsOnSeries: undefined,
             showMean: undefined,
+            cumulative: undefined,
             yAxisScaleType: undefined,
             hiddenLegendIndexes: undefined,
             hiddenLegendBreakdowns: undefined,

frontend/src/scenes/retention/RetentionModal.tsx

@@ -27,7 +27,7 @@ export function RetentionModal(): JSX.Element | null {
     const { results } = useValues(retentionLogic(insightProps))
     const { people, peopleLoading, peopleLoadingMore } = useValues(retentionPeopleLogic(insightProps))
     const { loadMorePeople } = useActions(retentionPeopleLogic(insightProps))
-    const { aggregationTargetLabel, selectedInterval, exploreUrl, actorsQuery } = useValues(
+    const { aggregationTargetLabel, selectedInterval, exploreUrl, actorsQuery, retentionFilter } = useValues(
         retentionModalLogic(insightProps)
     )
     const { closeModal } = useActions(retentionModalLogic(insightProps))
@@ -111,20 +111,32 @@ export function RetentionModal(): JSX.Element | null {
                             <tbody>
                                 <tr>
                                     <th>{capitalizeFirstLetter(aggregationTargetLabel.singular)}</th>
-                                    {row.values?.map((data: any, index: number) => (
-                                        <th key={index}>
-                                            <div>{results[index].label}</div>
-                                            <div>
-                                                {data.count}
-                                                &nbsp;
-                                                {data.count > 0 && (
-                                                    <span className="text-muted">
-                                                        ({percentage(data.count / row?.values[0]['count'])})
-                                                    </span>
-                                                )}
-                                            </div>
-                                        </th>
-                                    ))}
+                                    {row.values?.map((data: any, index: number) => {
+                                        let cumulativeCount = data.count
+                                        if (retentionFilter?.cumulative) {
+                                            for (let i = index + 1; i < row.values.length; i++) {
+                                                cumulativeCount += row.values[i].count
+                                            }
+                                            cumulativeCount = Math.min(cumulativeCount, row.values[0].count)
+                                        }
+                                        const percentageValue =
+                                            row.values[0].count > 0 ? cumulativeCount / row.values[0].count : 0
+
+                                        return (
+                                            <th key={index}>
+                                                <div>{results[index].label}</div>
+                                                <div>
+                                                    {cumulativeCount}
+                                                    &nbsp;
+                                                    {cumulativeCount > 0 && (
+                                                        <span className="text-muted">
+                                                            ({percentage(percentageValue)})
+                                                        </span>
+                                                    )}
+                                                </div>
+                                            </th>
+                                        )
+                                    })}
                                 </tr>
                                 {people.result &&
                                     people.result.map((personAppearances: RetentionTableAppearanceType) => (

frontend/src/scenes/retention/RetentionTable.tsx

@@ -35,7 +35,7 @@ export function RetentionTable({ inCardView = false }: { inCardView?: boolean })
 
                 {showMean && tableRows.length > 0 ? (
                     <tr className="border-b" key={-1}>
-                        {range(0, tableRows[0].length - 1).map((columnIndex) => (
+                        {range(0, tableRows[0].length).map((columnIndex) => (
                             <td key={columnIndex} className="pb-2">
                                 {columnIndex <= (hideSizeColumn ? 0 : 1) ? (
                                     columnIndex == 0 ? (
@@ -47,23 +47,23 @@ export function RetentionTable({ inCardView = false }: { inCardView?: boolean })
                                             mean(
                                                 tableRows.map((row) => {
                                                     // Stop before the last item in a row, which is an incomplete time period
-                                                    if (columnIndex < row.length - 1) {
-                                                        return row[columnIndex].percentage
+                                                    if (
+                                                        (columnIndex >= row.length - 1 && isLatestPeriod) ||
+                                                        !row[columnIndex]
+                                                    ) {
+                                                        return null
                                                     }
-                                                    return null
+                                                    return row[columnIndex].percentage
                                                 })
                                             ) || 0
                                         }
-                                        latest={columnIndex == tableRows[0].length - 1}
+                                        latest={isLatestPeriod && columnIndex == tableRows[0].length - 1}
                                         clickable={false}
                                         backgroundColor={PURPLE}
                                     />
                                 )}
                             </td>
                         ))}
-                        <td className="pb-2">
-                            <CohortDay percentage={0} latest={true} clickable={false} />
-                        </td>
                     </tr>
                 ) : undefined}
 

frontend/src/scenes/retention/retentionLineGraphLogic.ts

@@ -29,7 +29,7 @@ export const retentionLineGraphLogic = kea<retentionLineGraphLogicType>([
         trendSeries: [
             (s) => [s.results, s.retentionFilter],
             (results, retentionFilter): RetentionTrendPayload[] => {
-                const { period, retentionReference } = retentionFilter || {}
+                const { period, retentionReference, cumulative } = retentionFilter || {}
                 // If the retention reference option is specified as previous,
                 // then translate retention rates to relative to previous,
                 // otherwise, just use what the result was originally.
@@ -45,12 +45,22 @@ export const retentionLineGraphLogic = kea<retentionLineGraphLogicType>([
                 // further and translate these numbers into percentage of the
                 // previous value so we get some idea for the rate of
                 // convergence.
+
                 return results.map((cohortRetention, datasetIndex) => {
-                    const retentionPercentages = cohortRetention.values
+                    let retentionPercentages = cohortRetention.values
                         .map((value) => value.count / cohortRetention.values[0].count)
-                        // Make them display in the right scale
                         .map((value) => (isNaN(value) ? 0 : 100 * value))
 
+                    if (cumulative) {
+                        retentionPercentages = retentionPercentages.map((value, valueIndex, arr) => {
+                            let cumulativeValue = value
+                            for (let i = valueIndex + 1; i < arr.length; i++) {
+                                cumulativeValue += arr[i]
+                            }
+                            return Math.min(cumulativeValue, 100)
+                        })
+                    }
+
                     // To calculate relative percentages, we take for instance Cohort 1 as percentages
                     // of the cohort size and create another series that has a 100 at prepended so we have
                     //

frontend/src/scenes/retention/retentionModalLogic.ts

@@ -22,7 +22,7 @@ export const retentionModalLogic = kea<retentionModalLogicType>([
     connect((props: InsightLogicProps) => ({
         values: [
             insightVizDataLogic(props),
-            ['querySource'],
+            ['querySource', 'retentionFilter'],
             groupsModel,
             ['aggregationLabel'],
             featureFlagLogic,

frontend/src/scenes/retention/retentionTableLogic.ts

@@ -69,7 +69,7 @@ export const retentionTableLogic = kea<retentionTableLogicType>([
         tableRows: [
             (s) => [s.results, s.maxIntervalsCount, s.retentionFilter, s.breakdownFilter, s.hideSizeColumn],
             (results, maxIntervalsCount, retentionFilter, breakdownFilter, hideSizeColumn) => {
-                const { period } = retentionFilter || {}
+                const { period, cumulative } = retentionFilter || {}
                 const { breakdowns } = breakdownFilter || {}
 
                 return range(maxIntervalsCount).map((index: number) => {
@@ -99,12 +99,21 @@ export const retentionTableLogic = kea<retentionTableLogicType>([
 
                     const secondColumn = hideSizeColumn ? [] : [currentResult.values[0].count]
 
-                    const otherColumns = currentResult.values.map((value) => {
+                    const otherColumns = currentResult.values.map((value, valueIndex) => {
                         const totalCount = currentResult.values[0]['count']
-                        const percentage = totalCount > 0 ? (value['count'] / totalCount) * 100 : 0
+                        let count = value['count']
+
+                        if (cumulative && valueIndex > 0) {
+                            for (let i = valueIndex + 1; i < currentResult.values.length; i++) {
+                                count += currentResult.values[i]['count']
+                            }
+                            count = Math.min(count, totalCount)
+                        }
+
+                        const percentage = totalCount > 0 ? (count / totalCount) * 100 : 0
 
                         return {
-                            count: value['count'],
+                            count,
                             percentage,
                         }
                     })

frontend/src/types.ts

@@ -2269,6 +2269,7 @@ export interface RetentionFilterType extends FilterType {
 
     //frontend only
     show_mean?: boolean
+    cumulative?: boolean
 }
 export interface LifecycleFilterType extends FilterType {
     /** @deprecated */

posthog/hogql_queries/legacy_compatibility/filter_to_query.py

@@ -456,6 +456,7 @@ def _insight_filter(filter: dict):
                 ),
                 period=filter.get("period"),
                 showMean=filter.get("show_mean"),
+                cumulative=filter.get("cumulative"),
             )
         }
     elif _insight_type(filter) == "PATHS":

posthog/hogql_queries/legacy_compatibility/test/test_filter_to_query.py

@@ -1504,6 +1504,7 @@ def test_retention_filter(self):
             "target_entity": {"id": "$pageview", "name": "$pageview", "type": "events"},
             "period": "Week",
             "show_mean": True,
+            "cumulative": True,
         }
 
         query = filter_to_query(filter)
@@ -1530,6 +1531,7 @@ def test_retention_filter(self):
                     "order": None,
                 },
                 showMean=True,
+                cumulative=True,
             ),
         )
 

posthog/schema.py

@@ -3083,6 +3083,7 @@ class RetentionFilter(BaseModel):
     model_config = ConfigDict(
         extra="forbid",
     )
+    cumulative: Optional[bool] = None
     period: Optional[RetentionPeriod] = RetentionPeriod.DAY
     retentionReference: Optional[RetentionReference] = None
     retentionType: Optional[RetentionType] = None
@@ -3096,6 +3097,7 @@ class RetentionFilterLegacy(BaseModel):
     model_config = ConfigDict(
         extra="forbid",
     )
+    cumulative: Optional[bool] = None
     period: Optional[RetentionPeriod] = None
     retention_reference: Optional[RetentionReference] = None
     retention_type: Optional[RetentionType] = None

posthog/schema_helpers.py

@@ -70,6 +70,7 @@ def serialize_query(self, next_serializer):
                                 "toggledLifecycles",
                                 "showLabelsOnSeries",
                                 "showMean",
+                                "cumulative",
                                 "yAxisScaleType",
                             ]
                         }