From afddde039abad89edc2ddacb73efaa38476cb150 Mon Sep 17 00:00:00 2001
From: "Dr. Hannah De los Santos" <HDELOSSANTOS@MITRE.ORG>
Date: Tue, 1 Aug 2023 19:04:46 -0400
Subject: [PATCH] Clean up, add factor levels, and fix tiny SDe bug

---
 R/growth.R        | 22 ++++++++++++++++++++++
 R/infants_clean.R | 32 ++++++++++++++++++++++++++++----
 2 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/R/growth.R b/R/growth.R
index 56e6c37..b192c40 100644
--- a/R/growth.R
+++ b/R/growth.R
@@ -244,6 +244,26 @@ cleangrowth <- function(subjid,
     'Exclude-SDE-All-Extreme',
     'Exclude-SDE-EWMA',
     'Exclude-SDE-One-Day',
+    "Exclude-EWMA2-middle",
+    "Exclude-EWMA2-birth-WT",
+    "Exclude-EWMA-2-birth-WT-ext",
+    "Exclude-EWMA-2-first",
+    "Exclude-EWMA-2-first-ext",
+    "Exclude-EWMA2-last",
+    "Exclude-EWMA2-last-high",
+    "Exclude-EWMA2-last-ext",
+    "Exclude-EWMA2-last-ext-high",
+    "Exclude-EWMA2-birth-HT-HC",
+    "Exclude-EWMA2-birth-HT-HC-ext",
+    "Exclude-Min-diff",
+    "Exclude-Max-diff",
+    "Exclude-2-meas->1-year",
+    "Exclude-2-meas-<1-year",
+    "Exclude-1-meas",
+    "Exclude-Error-load",
+
+    # old
+
     'Exclude-EWMA-8',
     'Exclude-EWMA-9',
     'Exclude-EWMA-10',
@@ -259,6 +279,8 @@ cleangrowth <- function(subjid,
     'Exclude-Single-Outlier',
     'Exclude-Too-Many-Errors',
     'Exclude-Too-Many-Errors-Other-Parameter',
+
+    #new
     "Exclude-Absolute-BIV",
     "Exclude-Standardized-BIV",
     "Exclude-Evil-Twin",
diff --git a/R/infants_clean.R b/R/infants_clean.R
index 1d26e0a..0944a9c 100644
--- a/R/infants_clean.R
+++ b/R/infants_clean.R
@@ -834,7 +834,7 @@ cleanbatch_infants <- function(data.df,
             if (s_df$agedays[1]%%2 == 0){
               which.min(s_df$v)
             } else {
-              which.max(s_df$v == max(s_df$v[order(med_diff)[1:2]]))
+              which.max(s_df$v)
             }
             rem_ids <- c(rem_ids, s_df$index[-keep_val])
           }
@@ -855,7 +855,11 @@ cleanbatch_infants <- function(data.df,
 
   # 15: moderate EWMA ----
 
-  # THINK ABOUT NNTE FULL HERE
+  if (!quietly)
+    cat(sprintf(
+      "[%s] Exclude moderate EWMA...\n",
+      Sys.time()
+    ))
 
   # create the valid set
   # we only running carried forwards on valid values, non NNTE values,
@@ -1113,6 +1117,12 @@ cleanbatch_infants <- function(data.df,
 
   # 16: moderate EWMA for birth HT and HC ----
 
+  if (!quietly)
+    cat(sprintf(
+      "[%s] Exclude moderate EWMA for birth height and head circumference...\n",
+      Sys.time()
+    ))
+
   # create the valid set
   # we only running carried forwards on valid values, non NNTE values,
   # and non single values, and non weight
@@ -1260,7 +1270,11 @@ cleanbatch_infants <- function(data.df,
 
   # 17: raw differences ----
 
-  # NOTE: ADD STEP OUTPUT PRINT
+  if (!quietly)
+    cat(sprintf(
+      "[%s] Exclude raw differences...\n",
+      Sys.time()
+    ))
 
   # read in tanner data
   tanner_ht_vel_rev_path <- ifelse(
@@ -1568,7 +1582,11 @@ cleanbatch_infants <- function(data.df,
 
   # 19: 1 or 2 measurements ----
 
-  # NOTE: ADD STEP OUTPUT PRINT
+  if (!quietly)
+    cat(sprintf(
+      "[%s] Exclude 1 or 2 measurements...\n",
+      Sys.time()
+    ))
 
   # create the valid set
   # we only running carried forwards on valid values, non NNTE values,
@@ -1667,6 +1685,12 @@ cleanbatch_infants <- function(data.df,
 
   # 21: error load ----
 
+  if (!quietly)
+    cat(sprintf(
+      "[%s] Exclude error load...\n",
+      Sys.time()
+    ))
+
   valid_set <- !data.df$nnte_full
 
   data.df[valid_set,