Store all order statuses (#1285)

* Store all orders regardless of status in wc_order_stats

* Only query select statuses if status_is or status_is_not is not set

* Fix order stats example query

* Filter category store by default order statuses

* Add table_name to interval functions to avoid ambiguity

* Filter coupon store by default order statuses

* Fix interval stats by adding table_name to interval function

* Filter products store by default order statuses

* Remove unused order_status in coupons data store

* Store products with any order status

* Fix ambiguous order by call in products data store

* Apply status filter after other filters to avoid logical OR matching

* Store all coupon data regardless of order status

* Filter taxes by default order statuses

* Filter customer stats by default order status

* Filter default order statuses in variations data store

* Use excluded statuses to filter reports by default

* DRY up default excluded statuses query

* Fix up new orders data store after rebase

includes/api/class-wc-admin-rest-reports-orders-stats-controller.php

@@ -52,8 +52,8 @@ protected function prepare_reports_query( $request ) {
 		$args['status_is_not']    = (array) $request['status_is_not'];
 		$args['product_includes'] = (array) $request['product_includes'];
 		$args['product_excludes'] = (array) $request['product_excludes'];
-		$args['coupon_includes']    = (array) $request['coupon_includes'];
-		$args['coupon_excludes']    = (array) $request['coupon_excludes'];
+		$args['coupon_includes']  = (array) $request['coupon_includes'];
+		$args['coupon_excludes']  = (array) $request['coupon_excludes'];
 		$args['customer']         = $request['customer'];
 		$args['categories']       = (array) $request['categories'];
 
@@ -319,6 +319,7 @@ public function get_collection_params() {
 			'type'              => 'array',
 			'sanitize_callback' => 'wp_parse_slug_list',
 			'validate_callback' => 'rest_validate_request_arg',
+			'default'           => null,
 			'items'             => array(
 				'enum' => $this->get_order_statuses(),
 				'type' => 'string',

includes/class-wc-admin-reports-interval.php

@@ -30,28 +30,29 @@ class WC_Admin_Reports_Interval {
 	 * Returns date format to be used as grouping clause in SQL.
 	 *
 	 * @param string $time_interval Time interval.
+	 * @param string $table_name Name of the db table relevant for the date constraint.
 	 * @return mixed
 	 */
-	public static function db_datetime_format( $time_interval ) {
+	public static function db_datetime_format( $time_interval, $table_name ) {
 		$first_day_of_week = absint( get_option( 'start_of_week' ) );
 
 		if ( 1 === $first_day_of_week ) {
 			// Week begins on Monday, ISO 8601.
-			$week_format = "DATE_FORMAT(date_created, '%x-%v')";
+			$week_format = "DATE_FORMAT({$table_name}.date_created, '%x-%v')";
 		} else {
 			// Week begins on day other than specified by ISO 8601, needs to be in sync with function simple_week_number.
-			$week_format = "CONCAT(YEAR(date_created), '-', LPAD( FLOOR( ( DAYOFYEAR(date_created) + ( ( DATE_FORMAT(MAKEDATE(YEAR(date_created),1), '%w') - $first_day_of_week + 7 ) % 7 ) - 1 ) / 7  ) + 1 , 2, '0'))";
+			$week_format = "CONCAT(YEAR({$table_name}.date_created), '-', LPAD( FLOOR( ( DAYOFYEAR({$table_name}.date_created) + ( ( DATE_FORMAT(MAKEDATE(YEAR({$table_name}.date_created),1), '%w') - $first_day_of_week + 7 ) % 7 ) - 1 ) / 7  ) + 1 , 2, '0'))";
 
 		}
 
 		// Whenever this is changed, double check method time_interval_id to make sure they are in sync.
 		$mysql_date_format_mapping = array(
-			'hour'    => "DATE_FORMAT(date_created, '%Y-%m-%d %H')",
-			'day'     => "DATE_FORMAT(date_created, '%Y-%m-%d')",
+			'hour'    => "DATE_FORMAT({$table_name}.date_created, '%Y-%m-%d %H')",
+			'day'     => "DATE_FORMAT({$table_name}.date_created, '%Y-%m-%d')",
 			'week'    => $week_format,
-			'month'   => "DATE_FORMAT(date_created, '%Y-%m')",
-			'quarter' => "CONCAT(YEAR(date_created), '-', QUARTER(date_created))",
-			'year'    => 'YEAR(date_created)',
+			'month'   => "DATE_FORMAT({$table_name}.date_created, '%Y-%m')",
+			'quarter' => "CONCAT(YEAR({$table_name}.date_created), '-', QUARTER({$table_name}.date_created))",
+			'year'    => "YEAR({$table_name}.date_created)",
 
 		);
 

includes/class-wc-admin-reports-orders-stats-query.php

@@ -9,7 +9,7 @@
  *          'interval'     => 'week',
  *          'categories'   => array(15, 18),
  *          'coupons'      => array(138),
- *          'order_status' => array('completed'),
+ *          'status_in'    => array('completed'),
  *         );
  * $report = new WC_Admin_Reports_Orders_Stats_Query( $args );
  * $mydata = $report->get_data();

includes/data-stores/class-wc-admin-reports-categories-data-store.php

@@ -59,6 +59,16 @@ class WC_Admin_Reports_Categories_Data_Store extends WC_Admin_Reports_Data_Store
 		'products_count' => 'COUNT(DISTINCT product_id) as products_count',
 	);
 
+	/**
+	 * Constructor
+	 */
+	public function __construct() {
+		global $wpdb;
+		$table_name = $wpdb->prefix . self::TABLE_NAME;
+		// Avoid ambigious column order_id in SQL query.
+		$this->report_columns['orders_count'] = str_replace( 'order_id', $table_name . '.order_id', $this->report_columns['orders_count'] );
+	}
+
 	/**
 	 * Return the database query with parameters used for Categories report: time span and order status.
 	 *
@@ -93,7 +103,7 @@ protected function get_sql_query_params( $query_args ) {
 
 		$order_status_filter = $this->get_status_subquery( $query_args );
 		if ( $order_status_filter ) {
-			$sql_query_params['from_clause']  .= " JOIN {$wpdb->prefix}posts ON {$order_product_lookup_table}.order_id = {$wpdb->prefix}posts.ID";
+			$sql_query_params['from_clause']  .= " JOIN {$wpdb->prefix}wc_order_stats ON {$order_product_lookup_table}.order_id = {$wpdb->prefix}wc_order_stats.order_id";
 			$sql_query_params['where_clause'] .= " AND ( {$order_status_filter} )";
 		}
 
@@ -216,9 +226,6 @@ public function get_data( $query_args ) {
 			'fields'        => '*',
 			'categories'    => array(),
 			'extended_info' => false,
-			// This is not a parameter for products reports per se, but maybe we should restricts order statuses here, too?
-			'order_status'  => parent::get_report_order_statuses(),
-
 		);
 		$query_args = wp_parse_args( $query_args, $defaults );
 

includes/data-stores/class-wc-admin-reports-coupons-data-store.php

@@ -41,6 +41,16 @@ class WC_Admin_Reports_Coupons_Data_Store extends WC_Admin_Reports_Data_Store im
 		'orders_count' => 'COUNT(DISTINCT order_id) as orders_count',
 	);
 
+	/**
+	 * Constructor
+	 */
+	public function __construct() {
+		global $wpdb;
+		$table_name = $wpdb->prefix . self::TABLE_NAME;
+		// Avoid ambigious column order_id in SQL query.
+		$this->report_columns['orders_count'] = str_replace( 'order_id', $table_name . '.order_id', $this->report_columns['orders_count'] );
+	}
+
 	/**
 	 * Set up all the hooks for maintaining and populating table data.
 	 */
@@ -84,10 +94,9 @@ protected function get_sql_query_params( $query_args ) {
 			$sql_query_params['where_clause'] .= " AND {$order_coupon_lookup_table}.coupon_id IN ({$included_coupons})";
 		}
 
-		// TODO: questionable, I think we need order status filters, even though it's not specified.
 		$order_status_filter = $this->get_status_subquery( $query_args );
 		if ( $order_status_filter ) {
-			$sql_query_params['from_clause']  .= " JOIN {$wpdb->prefix}posts ON {$order_coupon_lookup_table}.order_id = {$wpdb->prefix}posts.ID";
+			$sql_query_params['from_clause']  .= " JOIN {$wpdb->prefix}wc_order_stats ON {$order_coupon_lookup_table}.order_id = {$wpdb->prefix}wc_order_stats.order_id";
 			$sql_query_params['where_clause'] .= " AND ( {$order_status_filter} )";
 		}
 
@@ -214,9 +223,6 @@ public function get_data( $query_args ) {
 			'fields'        => '*',
 			'coupons'       => array(),
 			'extended_info' => false,
-			// This is not a parameter for coupons reports per se, but we want to only take into account selected order types.
-			'order_status'  => parent::get_report_order_statuses(),
-
 		);
 		$query_args = wp_parse_args( $query_args, $defaults );
 
@@ -319,15 +325,6 @@ public static function sync_order_coupons( $order_id ) {
 			return;
 		}
 
-		if ( ! in_array( $order->get_status(), parent::get_report_order_statuses(), true ) ) {
-			$wpdb->delete(
-				$wpdb->prefix . self::TABLE_NAME,
-				array( 'order_id' => $order->get_id() ),
-				array( '%d' )
-			);
-			return;
-		}
-
 		$coupon_items = $order->get_items( 'coupon' );
 		foreach ( $coupon_items as $coupon_item ) {
 			$wpdb->replace(

includes/data-stores/class-wc-admin-reports-coupons-stats-data-store.php

@@ -39,6 +39,16 @@ class WC_Admin_Reports_Coupons_Stats_Data_Store extends WC_Admin_Reports_Coupons
 		'orders_count'  => 'COUNT(DISTINCT order_id) as orders_count',
 	);
 
+	/**
+	 * Constructor
+	 */
+	public function __construct() {
+		global $wpdb;
+		$table_name = $wpdb->prefix . self::TABLE_NAME;
+		// Avoid ambigious column order_id in SQL query.
+		$this->report_columns['orders_count'] = str_replace( 'order_id', $table_name . '.order_id', $this->report_columns['orders_count'] );
+	}
+
 	/**
 	 * Updates the database query with parameters used for Products Stats report: categories and order status.
 	 *
@@ -61,7 +71,7 @@ protected function update_sql_query_params( $query_args, &$totals_params, &$inte
 
 		$order_status_filter = $this->get_status_subquery( $query_args );
 		if ( $order_status_filter ) {
-			$coupons_from_clause  .= " JOIN {$wpdb->prefix}posts ON {$order_coupon_lookup_table}.order_id = {$wpdb->prefix}posts.ID";
+			$coupons_from_clause  .= " JOIN {$wpdb->prefix}wc_order_stats ON {$order_coupon_lookup_table}.order_id = {$wpdb->prefix}wc_order_stats.order_id";
 			$coupons_where_clause .= " AND ( {$order_status_filter} )";
 		}
 
@@ -99,8 +109,6 @@ public function get_data( $query_args ) {
 			'fields'       => '*',
 			'interval'     => 'week',
 			'coupons'      => array(),
-			// This is not a parameter for products reports per se, but we should probably restricts order statuses here, too.
-			'order_status' => parent::get_report_order_statuses(),
 		);
 		$query_args = wp_parse_args( $query_args, $defaults );
 
@@ -160,15 +168,15 @@ public function get_data( $query_args ) {
 			$totals = (object) $this->cast_numbers( $totals[0] );
 
 			// Intervals.
-			$this->update_intervals_sql_params( $intervals_query, $query_args, $db_interval_count, $expected_interval_count );
+			$this->update_intervals_sql_params( $intervals_query, $query_args, $db_interval_count, $expected_interval_count, $table_name );
 
 			if ( '' !== $selections ) {
 				$selections = ', ' . $selections;
 			}
 
 			$intervals = $wpdb->get_results(
 				"SELECT
-							MAX(date_created) AS datetime_anchor,
+							MAX({$table_name}.date_created) AS datetime_anchor,
 							{$intervals_query['select_clause']} AS time_interval
 							{$selections}
 						FROM

includes/data-stores/class-wc-admin-reports-customers-data-store.php

@@ -203,11 +203,13 @@ protected function get_time_period_sql_params( $query_args, $table_name ) {
 	 */
 	protected function get_sql_query_params( $query_args ) {
 		global $wpdb;
-		$customer_lookup_table = $wpdb->prefix . self::TABLE_NAME;
+		$customer_lookup_table  = $wpdb->prefix . self::TABLE_NAME;
+		$order_stats_table_name = $wpdb->prefix . 'wc_order_stats';
 
-		$sql_query_params = $this->get_time_period_sql_params( $query_args, $customer_lookup_table );
-		$sql_query_params = array_merge( $sql_query_params, $this->get_limit_sql_params( $query_args ) );
-		$sql_query_params = array_merge( $sql_query_params, $this->get_order_by_sql_params( $query_args ) );
+		$sql_query_params                = $this->get_time_period_sql_params( $query_args, $customer_lookup_table );
+		$sql_query_params                = array_merge( $sql_query_params, $this->get_limit_sql_params( $query_args ) );
+		$sql_query_params                = array_merge( $sql_query_params, $this->get_order_by_sql_params( $query_args ) );
+		$sql_query_params['from_clause'] = " LEFT JOIN {$order_stats_table_name} ON {$customer_lookup_table}.customer_id = {$order_stats_table_name}.customer_id";
 
 		$match_operator = $this->get_match_operator( $query_args );
 		$where_clauses  = array();
@@ -276,6 +278,11 @@ protected function get_sql_query_params( $query_args ) {
 			$sql_query_params['where_clause'] = $preceding_match . implode( " {$match_operator} ", $where_clauses );
 		}
 
+		$order_status_filter = $this->get_status_subquery( $query_args );
+		if ( $order_status_filter ) {
+			$sql_query_params['from_clause'] .= " AND ( {$order_status_filter} )";
+		}
+
 		if ( $having_clauses ) {
 			$preceding_match = empty( $sql_query_params['having_clause'] ) ? ' AND ' : " {$match_operator} ";
 			$sql_query_params['having_clause'] .= $preceding_match . implode( " {$match_operator} ", $having_clauses );
@@ -293,7 +300,7 @@ protected function get_sql_query_params( $query_args ) {
 	public function get_data( $query_args ) {
 		global $wpdb;
 
-		$customers_table_name = $wpdb->prefix . self::TABLE_NAME;
+		$customers_table_name   = $wpdb->prefix . self::TABLE_NAME;
 		$order_stats_table_name = $wpdb->prefix . 'wc_order_stats';
 
 		// These defaults are only partially applied when used via REST API, as that has its own defaults.
@@ -325,10 +332,7 @@ public function get_data( $query_args ) {
 					SELECT {$customers_table_name}.customer_id
 					FROM
 						{$customers_table_name}
-					LEFT JOIN
-						{$order_stats_table_name}
-					ON
-						{$customers_table_name}.customer_id = {$order_stats_table_name}.customer_id
+						{$sql_query_params['from_clause']}
 					WHERE
 						1=1
 						{$sql_query_params['where_time_clause']}
@@ -352,10 +356,7 @@ public function get_data( $query_args ) {
 						{$selections}
 					FROM
 						{$customers_table_name}
-					LEFT JOIN
-						{$order_stats_table_name}
-					ON
-						{$customers_table_name}.customer_id = {$order_stats_table_name}.customer_id
+						{$sql_query_params['from_clause']}
 					WHERE
 						1=1
 						{$sql_query_params['where_time_clause']}

includes/data-stores/class-wc-admin-reports-data-store.php

@@ -251,12 +251,13 @@ protected function intervals_missing( $expected_interval_count, $db_records, $it
 	 * If there are less records in the database than time intervals, then we need to remap offset in SQL query
 	 * to fetch correct records.
 	 *
-	 * @param array $intervals_query Array with clauses for the Intervals SQL query.
-	 * @param array $query_args Query arguements.
-	 * @param int   $db_interval_count Database interval count.
-	 * @param int   $expected_interval_count Expected interval count on the output.
+	 * @param array  $intervals_query Array with clauses for the Intervals SQL query.
+	 * @param array  $query_args Query arguements.
+	 * @param int    $db_interval_count Database interval count.
+	 * @param int    $expected_interval_count Expected interval count on the output.
+	 * @param string $table_name Name of the db table relevant for the date constraint.
 	 */
-	protected function update_intervals_sql_params( &$intervals_query, &$query_args, $db_interval_count, $expected_interval_count ) {
+	protected function update_intervals_sql_params( &$intervals_query, &$query_args, $db_interval_count, $expected_interval_count, $table_name ) {
 		if ( $db_interval_count === $expected_interval_count ) {
 			return;
 		}
@@ -328,8 +329,8 @@ protected function update_intervals_sql_params( &$intervals_query, &$query_args,
 			$query_args['adj_after']               = $new_start_date->format( WC_Admin_Reports_Interval::$iso_datetime_format );
 			$query_args['adj_before']              = $new_end_date->format( WC_Admin_Reports_Interval::$iso_datetime_format );
 			$intervals_query['where_time_clause']  = '';
-			$intervals_query['where_time_clause'] .= " AND date_created <= '{$query_args['adj_before']}'";
-			$intervals_query['where_time_clause'] .= " AND date_created >= '{$query_args['adj_after']}'";
+			$intervals_query['where_time_clause'] .= " AND {$table_name}.date_created <= '{$query_args['adj_before']}'";
+			$intervals_query['where_time_clause'] .= " AND {$table_name}.date_created >= '{$query_args['adj_after']}'";
 			$intervals_query['limit']              = 'LIMIT 0,' . $intervals_query['per_page'];
 		} else {
 			if ( 'asc' === $query_args['order'] ) {
@@ -396,12 +397,12 @@ protected function selected_columns( $query_args ) {
 	}
 
 	/**
-	 * Get the order statuses used when calculating reports.
+	 * Get the excluded order statuses used when calculating reports.
 	 *
 	 * @return array
 	 */
-	protected static function get_report_order_statuses() {
-		return apply_filters( 'woocommerce_reports_order_statuses', array( 'completed', 'processing', 'on-hold' ) );
+	protected static function get_excluded_report_order_statuses() {
+		return apply_filters( 'woocommerce_reports_excluded_order_statuses', array( 'refunded', 'pending', 'failed', 'cancelled' ) );
 	}
 
 	/**
@@ -585,7 +586,7 @@ protected function get_intervals_sql_params( $query_args, $table_name ) {
 
 		if ( isset( $query_args['interval'] ) && '' !== $query_args['interval'] ) {
 			$interval                         = $query_args['interval'];
-			$intervals_query['select_clause'] = WC_Admin_Reports_Interval::db_datetime_format( $interval );
+			$intervals_query['select_clause'] = WC_Admin_Reports_Interval::db_datetime_format( $interval, $table_name );
 		}
 
 		$intervals_query = array_merge( $intervals_query, $this->get_limit_sql_params( $query_args ) );
@@ -769,7 +770,8 @@ protected function get_excluded_users( $query_args ) {
 	protected function get_status_subquery( $query_args, $operator = 'AND' ) {
 		global $wpdb;
 
-		$subqueries = array();
+		$subqueries        = array();
+		$excluded_statuses = array();
 		if ( isset( $query_args['status_is'] ) && is_array( $query_args['status_is'] ) && count( $query_args['status_is'] ) > 0 ) {
 			$allowed_statuses = array_map( array( $this, 'normalize_order_status' ), $query_args['status_is'] );
 			if ( $allowed_statuses ) {
@@ -778,10 +780,17 @@ protected function get_status_subquery( $query_args, $operator = 'AND' ) {
 		}
 
 		if ( isset( $query_args['status_is_not'] ) && is_array( $query_args['status_is_not'] ) && count( $query_args['status_is_not'] ) > 0 ) {
-			$forbidden_statuses = array_map( array( $this, 'normalize_order_status' ), $query_args['status_is_not'] );
-			if ( $forbidden_statuses ) {
-				$subqueries[] = "{$wpdb->prefix}wc_order_stats.status NOT IN ( '" . implode( "','", $forbidden_statuses ) . "' )";
-			}
+			$excluded_statuses = array_map( array( $this, 'normalize_order_status' ), $query_args['status_is_not'] );
+		}
+
+		if ( ( ! isset( $query_args['status_is'] ) || empty( $query_args['status_is'] ) )
+			&& ( ! isset( $query_args['status_is_not'] ) || empty( $query_args['status_is_not'] ) )
+		) {
+			$excluded_statuses = array_map( array( $this, 'normalize_order_status' ), $this->get_excluded_report_order_statuses() );
+		}
+
+		if ( $excluded_statuses ) {
+			$subqueries[] = "{$wpdb->prefix}wc_order_stats.status NOT IN ( '" . implode( "','", $excluded_statuses ) . "' )";
 		}
 
 		return implode( " $operator ", $subqueries );

includes/data-stores/class-wc-admin-reports-downloads-stats-data-store.php

@@ -107,7 +107,7 @@ public function get_data( $query_args ) {
 				return array();
 			}
 
-			$this->update_intervals_sql_params( $intervals_query, $query_args, $db_records_count, $expected_interval_count );
+			$this->update_intervals_sql_params( $intervals_query, $query_args, $db_records_count, $expected_interval_count, $table_name );
 			$intervals_query['where_time_clause'] = str_replace( 'date_created', 'timestamp', $intervals_query['where_time_clause'] );
 
 			$totals = $wpdb->get_results(