diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2cd20b3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,118 @@ +name: Implementation Testing + +on: + push: + branches: + - '*' + pull_request: + branches: + - '*' + +env: + CACHE_NUMBER: 0 # increase to reset cache manually + +jobs: + foundation: + + strategy: + matrix: + python-version: ["3.10"] + defaults: + run: + shell: bash -l {0} + name: linux-64-py${{ matrix.python-version }} + runs-on: ubuntu-latest + steps: + # checkout the code in this repository + - uses: actions/checkout@v4 + with: + path: 'arc-activitysim' + + # checkout the main branch of ActivitySim itself + - uses: actions/checkout@v4 + with: + repository: 'ActivitySim/activitysim' + ref: main + path: 'activitysim' + fetch-depth: 0 # get all tags, lets setuptools_scm do its thing + + - name: Setup Miniforge + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + activate-environment: asim-test + python-version: ${{ matrix.python-version }} + + - name: Set cache date for year and month + run: echo "DATE=$(date +'%Y%m')" >> $GITHUB_ENV + + - uses: actions/cache@v4 + with: + path: ~/conda_pkgs_dir + key: linux-64-conda-${{ hashFiles('activitysim/conda-environments/github-actions-tests.yml') }}-${{ env.DATE }}-${{ env.CACHE_NUMBER }} + id: cache + + - name: Update environment + run: | + conda env update -n asim-test -f activitysim/conda-environments/github-actions-tests.yml + + - name: Install activitysim + # installing without dependencies is faster, we trust that all needed dependencies + # are in the conda environment defined above. Also, this avoids pip getting + # confused and reinstalling tables (pytables). + run: | + python -m pip install ./activitysim --no-deps + + - name: Conda checkup + run: | + conda info -a + conda list + + - name: Get the Fulton data + run: | + cd arc-activitysim + python scripts/fetch-fulton.py + + - name: Run progressive tests + run: | + cd arc-activitysim + python -m pytest tests/test_activitysim.py + + - name: Run without Sharrow + run: | + cd arc-activitysim + python scripts/run-fulton.py + + - name: Upload legacy artifacts + uses: actions/upload-artifact@v4 + with: + name: legacy-outputs + path: | + ${{ github.workspace }}/arc-activitysim/output-fulton-legacy/final_*.csv + ${{ github.workspace }}/arc-activitysim/output-fulton-legacy/*.log + ${{ github.workspace }}/arc-activitysim/output-fulton-legacy/timing_log.csv + + - name: Check legacy outputs + run: | + cd arc-activitysim + python scripts/check-fulton.py --check-dir ${{ github.workspace }}/arc-activitysim/output-fulton-legacy + + - name: Run with Sharrow + run: | + cd arc-activitysim + python scripts/run-fulton.py --sharrow + + - name: Upload Sharrow artifacts + uses: actions/upload-artifact@v4 + with: + name: sharrow-outputs + path: | + ${{ github.workspace }}/arc-activitysim/output-fulton-sharrow/final_*.csv + ${{ github.workspace }}/arc-activitysim/output-fulton-sharrow/*.log + ${{ github.workspace }}/arc-activitysim/output-fulton-sharrow/timing_log.csv + + - name: Check sharrow outputs + run: | + cd arc-activitysim + python scripts/check-fulton.py --check-dir ${{ github.workspace }}/arc-activitysim/output-fulton-sharrow + diff --git a/README.md b/README.md index 95408da..157d7ef 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,118 @@ # arc-activitysim The standalone activitysim implementation for ARC travel demand model. + +## Installation + +To install the ARC ActivitySim model, simply clone this repository: + +```bash +git clone https://github.com/atlregional/arc-activitysim.git +cd arc-activitysim +``` + +Using this model requires ActivitySim 1.3 or later. This is most easily +accomplished using the `activitysim` conda package, which first requires the +installation of the `conda` package manager. For most systems, the Miniforge +distribution is recommended, which can be downloaded from +[conda-forge](https://github.com/conda-forge/miniforge?tab=readme-ov-file#miniforge3). + +Once `conda` is installed, the `activitysim` package can be installed from the +MiniForge Prompt (or the terminal on Linux/Mac): + +```bash +conda create -n ARC-ASIM activitysim -c conda-forge --override-channels +``` + +This will create a new conda environment named `ARC-ASIM` with the `activitysim` +package installed. To activate the environment, use: + +```bash +conda activate ARC-ASIM +``` + +## Running the Model + +The ARC ActivitySim model can be run using the `activitysim` command line tool. +The model is configured using the `configs` directory, which contains the +configuration files for the model. From the directory where this repository +has been cloned, the model can be run using the following command: + +```bash +activitysim run -c configs -d data_dir -o output_dir +``` + +Where `data_dir` is the directory containing the input data for the model, and +`output_dir` is the directory where the model output will be written. The data +directory should contain the necessary input files (houeholds, persons, land use, +and skims), which can be the full scale ARC data or a smaller test data set (see +instructions to access the Fulton County test data below). The output directory +will be created if it does not exist, and the model output will be written to +subdirectories of this directory. + +## Running the Model with Sharrow + +The ARC ActivitySim model can also be run with the sharrow enabled. This is +done by adding the relevant sharrow configs directory to the command. For +example, to run the model with the sharrow in compile-test mode, use the +following command: + +```bash +activitysim run -c configs -c configs_sh_compile -d data_dir -o output_dir +``` + +This will run the model with the sharrow enabled, and will compile the numba +code and run tests to ensure the results match between sharrow and legacy modes. +Once the sharrow compiling is complete, the model can subsequently be run in +sharrow's "production" mode, which will be much faster: + +```bash +activitysim run -c configs -c configs_sh -d data_dir -o output_dir +``` + +## Testing Dataset (Fulton County) + +This model is built to run with the data that simulates the full-scale +model of the ARC region, but this scale of data can be overwhelming +for testing the operation of the model, especially on more limited +platforms. + +To facilitate testing, data for a smaller slice of the region is available. +This test data includes just Fulton County, which has 1,296 zones; this is +a small enough area to run the model on a laptop or within the CI testing +infrastructure, as it will require only about 6GB of RAM to to store the +skims in memory, and another 1 or 2 GB for the rest of the model. But this +area is still large enough to provide a meaningful test of the model, with +enough zones to exercise the model's capabilities and complexity. The Fulton +County data can be downloaded with this Python script (also available +as [fetch-fulton.py](./scripts/fetch-fulton.py)): + +```python +from pathlib import Path +from activitysim.examples.external import download_external_example + +example_dir = download_external_example( + name=".", + working_dir=Path.cwd(), + assets={ + "arc-fulton-data.tar.zst": { + "url": "https://github.com/atlregional/arc-activitysim/releases/download/v1.3.0/arc-fulton-data.tar.zst", + "sha256": "402c3cf1fdd96ae0342f17146453b713602ca8454b78f1e8ff8cbc403e03441e", + "unpack": "arc-fulton-data", + }, + } +) +``` + +## Continuous Integration Testing + +This repository is configured to run continuous integration testing +using GitHub Actions. The tests are run on a small subset of the data +for Fulton County, and the results are uploaded to the `Actions` tab +of the repository. The tests are configured in the `.github/workflows` +directory, and use the scripts in the `scripts` directory. + +Note that the tests are run in a clean environment every time, so the +first sharrow test includes the overhead of compiling all the numba code. +This will make it appear that this sharrow test is *much* slower than the +comparable legacy test; this is normal an not an indication that sharrow is +slower than the legacy code for production runs. diff --git a/configs_sh/settings.yaml b/configs_sh/settings.yaml new file mode 100644 index 0000000..6da741b --- /dev/null +++ b/configs_sh/settings.yaml @@ -0,0 +1,212 @@ +households_sample_size: 0 + +# input tables +input_table_list: + - tablename: households + filename: households.csv + # The index column is set before keep_columns, + # so don't put index in keep columns + index_col: household_id + recode_columns: + home_zone_id: land_use.zone_id + rename_columns: + maz: home_zone_id + np: hhsize + nwrkrs_esr: num_workers + keep_columns: + - home_zone_id + - hhsize + - num_workers + - hincp + - hht + - tablename: persons + filename: persons.csv + # The index column is set before keep_columns, + # so don't put index in keep columns + index_col: person_id + rename_columns: + maz: home_zone_id + sporder: PNUM + keep_columns: + - household_id + - home_zone_id + - PNUM + - agep + - pecasOcc + - sex + - esr + - wkw + - wkhp + - schg + - tablename: land_use + filename: land_use.csv + # The index column is set before keep_columns, + # so don't put index in keep columns + index_col: zone_id + recode_columns: + zone_id: zero-based + keep_columns: + - retail + - service + - emp + - pop + - hshld + - univ + - acres + - PARKTOT + - PARKLNG + - PROPFREE + - PARKRATE + - areatype + - CBDFlag + - N11 + - N21 + - N22 + - N23 + - N313233 + - N42 + - N4445 + - N4849 + - N51 + - N52 + - N53 + - N54 + - N55 + - N56 + - N61 + - N62 + - N71 + - N72 + - N81 + - N92 + - EnrollDS + - EnrollPD + - I_PCTLT10K + - I_PCT10TO20 + - I_PCT20TO40 + - I_PCTGT40 + - RetailEmp30 + - PARKING_ZONE + +#input data store and skims +#input_store: arc_asim.h5 +skims_file: skims.omx + +# - shadow pricing global switches + +# turn shadow_pricing on and off for all models (e.g. school and work) +# shadow pricing is deprecated for less than full samples +use_shadow_pricing: True + +# chunk_size: +chunk_method: hybrid_uss +chunk_training_mode: explicit + +models: + - initialize_landuse + - compute_accessibility + - initialize_households + - school_location + - workplace_location + - auto_ownership_simulate + - free_parking + - cdap_simulate + - mandatory_tour_frequency + - mandatory_tour_scheduling + - joint_tour_frequency + - joint_tour_composition + - joint_tour_participation + - joint_tour_destination + - joint_tour_scheduling + - non_mandatory_tour_frequency + - non_mandatory_tour_destination + - non_mandatory_tour_scheduling + - tour_mode_choice_simulate + - atwork_subtour_frequency + - atwork_subtour_destination + - atwork_subtour_scheduling + - atwork_subtour_mode_choice + - stop_frequency + - trip_purpose + - trip_destination + - trip_purpose_and_destination + - trip_scheduling_choice + - trip_departure_choice + - trip_mode_choice + - parking_location + - write_data_dictionary + - track_skim_usage + - write_trip_matrices + - write_tables + +# resume_after: trip_departure_choice + +multiprocess: True +fail_fast: True +num_processes: 28 + +multiprocess_steps: + - name: mp_initialize_landuse + begin: initialize_landuse + - name: mp_accessibility + begin: compute_accessibility + slice: + tables: + - accessibility + exclude: True + - name: mp_initialize_households + begin: initialize_households + - name: mp_households + begin: school_location + slice: + tables: + - households + - persons + - name: mp_summarize + begin: write_trip_matrices + + +output_tables: + h5_store: False + action: include + prefix: final_ + tables: + - checkpoints + - tablename: accessibility + decode_columns: + zone_id: land_use.zone_id + - tablename: land_use + decode_columns: + zone_id: land_use.zone_id + - tablename: households + decode_columns: + home_zone_id: land_use.zone_id + - tablename: persons + decode_columns: + home_zone_id: land_use.zone_id + school_zone_id: nonnegative | land_use.zone_id + workplace_zone_id: nonnegative | land_use.zone_id + - tablename: tours + decode_columns: + origin: land_use.zone_id + destination: land_use.zone_id + - tablename: trips + decode_columns: + origin: land_use.zone_id + destination: land_use.zone_id + parking_zone_id: nonnegative | land_use.zone_id + - joint_tour_participants + +# trace_od: [100,10] +# trace_hh_id: 292105 #2720 +# trace_hh_id: 2 +# trace_hh_id: 1643904 + +min_value_of_time: 1 +max_value_of_time: 50 +distributed_vot_mu: 0.684 +distributed_vot_sigma: 0.85 + +# when running sharrow +sharrow: require +recode_pipeline_columns: True diff --git a/configs_sh_compile/settings_sh.yaml b/configs_sh_compile/settings.yaml similarity index 98% rename from configs_sh_compile/settings_sh.yaml rename to configs_sh_compile/settings.yaml index 6ff0d9a..4b7befc 100644 --- a/configs_sh_compile/settings_sh.yaml +++ b/configs_sh_compile/settings.yaml @@ -172,7 +172,9 @@ output_tables: prefix: final_ tables: - checkpoints - - accessibility + - tablename: accessibility + decode_columns: + zone_id: land_use.zone_id - tablename: land_use decode_columns: zone_id: land_use.zone_id diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..b578b53 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,14 @@ +# ARC ActivitySim Scripts + +This directory contains scripts for running ActivitySim with the ARC travel demand model. + +- *fetch-fulton.py*: A script for downloading the Fulton County dataset, which is + a subset of the full ARC dataset. This will download and extract the necessary + data files into the `arc-fulton-data` directory (creating it if needed). +- *run-fulton.py*: A script for running the ARC ActivitySim model using the + Fulton County dataset. +- *check-fulton.py*: A script for checking the output of the ARC ActivitySim + model using the Fulton County dataset. +- *fulton-reference-outputs.zip*: A zip file containing reference outputs for the + ARC ActivitySim model using the Fulton County dataset. This is used by the + `check-fulton.py` script to verify the model output. diff --git a/scripts/check-fulton.py b/scripts/check-fulton.py new file mode 100644 index 0000000..c528a20 --- /dev/null +++ b/scripts/check-fulton.py @@ -0,0 +1,139 @@ +import pandas as pd +import zipfile +import argparse +import os +import pathlib +from pandas.api.types import is_float_dtype +import numpy as np + +reference_outputs = pathlib.Path(__file__).parent.joinpath( + "fulton-reference-outputs.zip" +) + +DEFAULT_TOL = dict(rtol=1.0e-3, atol=1.0e-6) + + +def assert_frame_equal(df1, df2, sort_col=None, ignore_cols=None): + ignore_cols = ignore_cols or [] + if sort_col is not None: + df1 = df1.sort_values(sort_col).set_index(sort_col) + df2 = df2.sort_values(sort_col).set_index(sort_col) + pd.testing.assert_index_equal(df1.index, df2.index) + error_cols = [ + "household_id", + "person_id", + "tour_id", + "trip_id", + "zone_id", + "participant_id", + ] + error_cols = [c for c in error_cols if c in df1.columns] + + for k in df1.columns: + if k in ignore_cols: + continue + s1 = df1[k] + if k not in df2.columns: + raise AssertionError(f"Column {k!r} not found in second DataFrame") + s2 = df2[k] + + # print(f"---------- {k} ---------") + # print("s1=", s1.iloc[:3]) + # print("s2=", s2.iloc[:3]) + + if is_float_dtype(s1): + y = np.isclose(s1.values, s2.values, equal_nan=True, **DEFAULT_TOL) + # print("$$$\n", y) + misses = np.where(~y) + else: + y = ~((s1.values == s2.values) | (s1.isna() & s2.isna())) + misses = np.where(y) + misses = misses[0] + if len(misses) > 0: + print(f"- Column {k!r} has {len(misses)} discrepancies: {misses}") + print( + f"- Column {k!r} left failures=\n{df1[error_cols+[k]].iloc[misses[:3]]}" + ) + print( + f"- Column {k!r} right failures=\n{df2[error_cols+[k]].iloc[misses[:3]]}" + ) + if len(misses) > len(df1) * 0.01: + pd.testing.assert_series_equal(s1, s2, check_dtype=False, **DEFAULT_TOL) + raise AssertionError(f"Column {k!r} has {len(misses)} discrepancies") + + +def compare_table_artifact( + filename="final_households.csv", + artifact_name="legacy-outputs.zip", + sort_col=None, + **kwargs, +): + with zipfile.ZipFile(reference_outputs) as z: + with z.open(filename) as f: + ref_tab = pd.read_csv(f) + with zipfile.ZipFile(artifact_name) as z: + with z.open(filename) as f: + check_tab = pd.read_csv(f) + try: + assert_frame_equal(ref_tab, check_tab, sort_col, **kwargs) + except AssertionError: + print(f"ERROR: {filename} from {artifact_name} does not match") + raise + else: + print(f"OK: {filename} from {artifact_name} matches") + + +def compare_table_raw( + filename="final_households.csv", check_dir=".", sort_col=None, **kwargs +): + with zipfile.ZipFile(reference_outputs) as z: + with z.open(filename) as f: + ref_tab = pd.read_csv(f) + with open(os.path.join(check_dir, filename)) as f: + check_tab = pd.read_csv(f) + try: + assert_frame_equal(ref_tab, check_tab, sort_col, **kwargs) + except AssertionError: + print(f"ERROR: {check_dir}/{filename} does not match") + raise + else: + print(f"OK: {check_dir}/{filename} matches") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--check-dir", help="directory containing output files to check", type=str + ) + args = parser.parse_args() + + if args.check_dir: + compare_table_raw("final_households.csv", args.check_dir, "household_id") + compare_table_raw("final_persons.csv", args.check_dir, "person_id") + compare_table_raw("final_tours.csv", args.check_dir, "tour_id") + compare_table_raw("final_trips.csv", args.check_dir, "trip_id") + compare_table_raw( + "final_joint_tour_participants.csv", + args.check_dir, + sort_col="participant_id", + ) + compare_table_raw("final_accessibility.csv", args.check_dir, sort_col="zone_id") + return + + compare_table_artifact("final_households.csv", "legacy-outputs.zip") + compare_table_artifact("final_persons.csv", "legacy-outputs.zip") + compare_table_artifact("final_tours.csv", "legacy-outputs.zip") + compare_table_artifact("final_trips.csv", "legacy-outputs.zip") + compare_table_artifact("final_accessibility.csv", "legacy-outputs.zip") + compare_table_artifact("final_joint_tour_participants.csv", "legacy-outputs.zip") + + compare_table_artifact("final_households.csv", "sharrow-outputs.zip") + compare_table_artifact("final_persons.csv", "sharrow-outputs.zip") + compare_table_artifact("final_tours.csv", "sharrow-outputs.zip") + compare_table_artifact("final_trips.csv", "sharrow-outputs.zip") + compare_table_artifact("final_accessibility.csv", "sharrow-outputs.zip") + compare_table_artifact("final_joint_tour_participants.csv", "sharrow-outputs.zip") + + +if __name__ == "__main__": + main() diff --git a/scripts/fetch-fulton.py b/scripts/fetch-fulton.py new file mode 100644 index 0000000..fa90c37 --- /dev/null +++ b/scripts/fetch-fulton.py @@ -0,0 +1,19 @@ +from pathlib import Path +from activitysim.examples.external import download_external_example + +if __name__ == "__main__": + example_dir = download_external_example( + name=".", + working_dir=Path.cwd(), + assets={ + "arc-fulton-data.tar.zst": { + "url": "https://github.com/atlregional/arc-activitysim/releases/download/v1.3.0/arc-fulton-data.tar.zst", + "sha256": "402c3cf1fdd96ae0342f17146453b713602ca8454b78f1e8ff8cbc403e03441e", + "unpack": "arc-fulton-data", + }, + }, + ) + if example_dir.joinpath("arc-fulton-data").exists(): + example_dir.joinpath("arc-fulton-data").joinpath(".gitignore").write_text( + "**\n" + ) diff --git a/scripts/fulton-reference-outputs.zip b/scripts/fulton-reference-outputs.zip new file mode 100644 index 0000000..a639d25 Binary files /dev/null and b/scripts/fulton-reference-outputs.zip differ diff --git a/scripts/run-fulton.py b/scripts/run-fulton.py new file mode 100644 index 0000000..11ebb5a --- /dev/null +++ b/scripts/run-fulton.py @@ -0,0 +1,94 @@ +# ruff: noqa: E402 +import os + +# These environment variables are set to 1 to ensure that the model runs single-threaded +# This needs to be done before ActivitySim or any other dependency is imported +os.environ["MKL_NUM_THREADS"] = "1" +os.environ["NUMEXPR_NUM_THREADS"] = "1" +os.environ["OMP_NUM_THREADS"] = "1" +os.environ["OPENBLAS_NUM_THREADS"] = "1" +os.environ["VECLIB_MAXIMUM_THREADS"] = "1" +os.environ["NUMBA_NUM_THREADS"] = "1" + +import argparse +import activitysim.abm # register components # noqa: F401 +from pathlib import Path +import sharrow + +print("ActivitySim Version:", activitysim.__version__) +print("Sharrow Version:", sharrow.__version__) + +from activitysim.core import workflow + + +def local_path(dirname) -> str: + return str(Path(__file__).parents[1].joinpath(dirname)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--sharrow-compile", help="compile sharrow", action="store_true") + parser.add_argument("--sharrow", help="use sharrow", action="store_true") + parser.add_argument("--households-sample-size", type=int, default=1000) + args = parser.parse_args() + + # These things change based on whether we are using sharrow or not. + if args.sharrow_compile: + print("sharrow compile activated") + configs_dir = ( + local_path("configs_sh_compile"), + local_path("configs"), + ) + out_dir = local_path("output-fulton-sharrow-compile") + elif args.sharrow: + print("sharrow run activated") + configs_dir = ( + local_path("configs_sh"), + local_path("configs"), + ) + out_dir = local_path("output-fulton-sharrow") + else: + print("legacy run activated") + configs_dir = local_path("configs") + out_dir = local_path("output-fulton-legacy") + + # ensure the output directory exists, and do not commit any output to git + Path(out_dir).mkdir(exist_ok=True) + Path(out_dir).joinpath(".gitignore").write_text("**\n") + + # check that the arc-fulton-data has been downloaded + if not Path(local_path("arc-fulton-data")).exists(): + raise FileNotFoundError( + "arc-fulton-data directory not found. Please download the fulton data" + ) + + # for this test script, we will sample a limited number of households, + # and run the model with only 2 processes. + settings = { + "households_sample_size": args.households_sample_size, + # multiprocess is set to False as the trip scheduling choice model's + # reproducible random number generator is not MP-safe + # see https://github.com/ActivitySim/activitysim/issues/891 + "multiprocess": False, + # user can modify other settings here if desired + } + + # set `resume_after` to None to start from the beginning even if a partial + # result exists, otherwise use underscore to resume from where it left off + resume_after = "_" + + # create a state object and run the model + state = workflow.State.make_default( + configs_dir=configs_dir, + data_dir=local_path("arc-fulton-data"), + output_dir=out_dir, + settings=settings, + cache_dir=Path(out_dir).joinpath("cache"), + ) + state.import_extensions([]) + state.logging.config_logger() + state.run.all(resume_after=resume_after) + + +if __name__ == "__main__": + main() diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/accessibility/compute_accessibility.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/accessibility/compute_accessibility.parquet new file mode 100644 index 0000000..b47b3aa Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/accessibility/compute_accessibility.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/accessibility/initialize_landuse.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/accessibility/initialize_landuse.parquet new file mode 100644 index 0000000..ebb0ad7 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/accessibility/initialize_landuse.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/checkpoints.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/checkpoints.parquet new file mode 100644 index 0000000..cb17eaa Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/checkpoints.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/auto_ownership_simulate.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/auto_ownership_simulate.parquet new file mode 100644 index 0000000..cdbd385 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/auto_ownership_simulate.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/cdap_simulate.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/cdap_simulate.parquet new file mode 100644 index 0000000..bfee63a Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/cdap_simulate.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/initialize_households.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/initialize_households.parquet new file mode 100644 index 0000000..0eef566 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/initialize_households.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/joint_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/joint_tour_frequency.parquet new file mode 100644 index 0000000..240b412 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/joint_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/workplace_location.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/workplace_location.parquet new file mode 100644 index 0000000..f4dd8b2 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/households/workplace_location.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/joint_tour_participants/joint_tour_participation.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/joint_tour_participants/joint_tour_participation.parquet new file mode 100644 index 0000000..b50999f Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/joint_tour_participants/joint_tour_participation.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/land_use/initialize_landuse.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/land_use/initialize_landuse.parquet new file mode 100644 index 0000000..8b9b480 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/land_use/initialize_landuse.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/initialize_households.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/initialize_households.parquet new file mode 100644 index 0000000..283b9d8 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/initialize_households.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/joint_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/joint_tour_scheduling.parquet new file mode 100644 index 0000000..e6de26f Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/joint_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/mandatory_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/mandatory_tour_scheduling.parquet new file mode 100644 index 0000000..f04dc30 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/mandatory_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/non_mandatory_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/non_mandatory_tour_scheduling.parquet new file mode 100644 index 0000000..405ad6e Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/person_windows/non_mandatory_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/cdap_simulate.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/cdap_simulate.parquet new file mode 100644 index 0000000..5b8990b Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/cdap_simulate.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/free_parking.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/free_parking.parquet new file mode 100644 index 0000000..c43cbc2 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/free_parking.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/initialize_households.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/initialize_households.parquet new file mode 100644 index 0000000..083b945 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/initialize_households.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/joint_tour_participation.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/joint_tour_participation.parquet new file mode 100644 index 0000000..7570885 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/joint_tour_participation.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/mandatory_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/mandatory_tour_frequency.parquet new file mode 100644 index 0000000..d634ab6 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/mandatory_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/non_mandatory_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/non_mandatory_tour_frequency.parquet new file mode 100644 index 0000000..35416be Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/non_mandatory_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/school_location.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/school_location.parquet new file mode 100644 index 0000000..3d0bd20 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/school_location.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/workplace_location.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/workplace_location.parquet new file mode 100644 index 0000000..610a257 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/persons/workplace_location.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/school_destination_size/initialize_households.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/school_destination_size/initialize_households.parquet new file mode 100644 index 0000000..f6c12a9 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/school_destination_size/initialize_households.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/school_modeled_size/school_location.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/school_modeled_size/school_location.parquet new file mode 100644 index 0000000..862cff9 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/school_modeled_size/school_location.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_destination.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_destination.parquet new file mode 100644 index 0000000..243029b Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_destination.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_frequency.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_frequency.parquet new file mode 100644 index 0000000..ed21735 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_mode_choice.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_mode_choice.parquet new file mode 100644 index 0000000..a1f68c1 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_mode_choice.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_scheduling.parquet new file mode 100644 index 0000000..5a72f53 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/atwork_subtour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_composition.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_composition.parquet new file mode 100644 index 0000000..b3a6792 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_composition.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_destination.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_destination.parquet new file mode 100644 index 0000000..519a548 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_destination.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_frequency.parquet new file mode 100644 index 0000000..9820bef Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_participation.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_participation.parquet new file mode 100644 index 0000000..ef04930 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_participation.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_scheduling.parquet new file mode 100644 index 0000000..d6ce04b Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/joint_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/mandatory_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/mandatory_tour_frequency.parquet new file mode 100644 index 0000000..ad7497f Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/mandatory_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/mandatory_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/mandatory_tour_scheduling.parquet new file mode 100644 index 0000000..aaa7f54 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/mandatory_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/non_mandatory_tour_destination.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/non_mandatory_tour_destination.parquet new file mode 100644 index 0000000..354f0b9 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/non_mandatory_tour_destination.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/non_mandatory_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/non_mandatory_tour_frequency.parquet new file mode 100644 index 0000000..8a9b852 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/non_mandatory_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/non_mandatory_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/non_mandatory_tour_scheduling.parquet new file mode 100644 index 0000000..3addd8e Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/non_mandatory_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/stop_frequency.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/stop_frequency.parquet new file mode 100644 index 0000000..59fad3a Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/stop_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/tour_mode_choice_simulate.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/tour_mode_choice_simulate.parquet new file mode 100644 index 0000000..cdc817a Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/tour_mode_choice_simulate.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/trip_scheduling_choice.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/trip_scheduling_choice.parquet new file mode 100644 index 0000000..71924a4 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/tours/trip_scheduling_choice.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/parking_location.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/parking_location.parquet new file mode 100644 index 0000000..12de770 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/parking_location.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/stop_frequency.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/stop_frequency.parquet new file mode 100644 index 0000000..945358a Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/stop_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_departure_choice.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_departure_choice.parquet new file mode 100644 index 0000000..95ba18f Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_departure_choice.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_destination.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_destination.parquet new file mode 100644 index 0000000..aabd4f2 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_destination.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_mode_choice.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_mode_choice.parquet new file mode 100644 index 0000000..7de5bb9 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_mode_choice.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_purpose.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_purpose.parquet new file mode 100644 index 0000000..150f928 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_purpose.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_purpose_and_destination.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_purpose_and_destination.parquet new file mode 100644 index 0000000..fcab843 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/trips/trip_purpose_and_destination.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/workplace_destination_size/initialize_households.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/workplace_destination_size/initialize_households.parquet new file mode 100644 index 0000000..63a68e0 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/workplace_destination_size/initialize_households.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/workplace_modeled_size/workplace_location.parquet b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/workplace_modeled_size/workplace_location.parquet new file mode 100644 index 0000000..93055ae Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000-recode.parquetpipeline/workplace_modeled_size/workplace_location.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/accessibility/compute_accessibility.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/accessibility/compute_accessibility.parquet new file mode 100644 index 0000000..b8c9edc Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/accessibility/compute_accessibility.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/accessibility/initialize_landuse.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/accessibility/initialize_landuse.parquet new file mode 100644 index 0000000..48b1c6b Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/accessibility/initialize_landuse.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/checkpoints.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/checkpoints.parquet new file mode 100644 index 0000000..09823b0 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/checkpoints.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/auto_ownership_simulate.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/auto_ownership_simulate.parquet new file mode 100644 index 0000000..c116919 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/auto_ownership_simulate.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/cdap_simulate.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/cdap_simulate.parquet new file mode 100644 index 0000000..0434f07 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/cdap_simulate.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/initialize_households.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/initialize_households.parquet new file mode 100644 index 0000000..7c44988 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/initialize_households.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/joint_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/joint_tour_frequency.parquet new file mode 100644 index 0000000..81033c6 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/joint_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/workplace_location.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/workplace_location.parquet new file mode 100644 index 0000000..030125c Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/households/workplace_location.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/joint_tour_participants/joint_tour_participation.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/joint_tour_participants/joint_tour_participation.parquet new file mode 100644 index 0000000..b50999f Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/joint_tour_participants/joint_tour_participation.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/land_use/initialize_landuse.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/land_use/initialize_landuse.parquet new file mode 100644 index 0000000..8252981 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/land_use/initialize_landuse.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/initialize_households.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/initialize_households.parquet new file mode 100644 index 0000000..283b9d8 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/initialize_households.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/joint_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/joint_tour_scheduling.parquet new file mode 100644 index 0000000..e6de26f Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/joint_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/mandatory_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/mandatory_tour_scheduling.parquet new file mode 100644 index 0000000..f04dc30 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/mandatory_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/non_mandatory_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/non_mandatory_tour_scheduling.parquet new file mode 100644 index 0000000..405ad6e Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/person_windows/non_mandatory_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/cdap_simulate.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/cdap_simulate.parquet new file mode 100644 index 0000000..3aca942 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/cdap_simulate.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/free_parking.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/free_parking.parquet new file mode 100644 index 0000000..be15bde Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/free_parking.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/initialize_households.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/initialize_households.parquet new file mode 100644 index 0000000..ff4aa86 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/initialize_households.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/joint_tour_participation.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/joint_tour_participation.parquet new file mode 100644 index 0000000..7037ae2 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/joint_tour_participation.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/mandatory_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/mandatory_tour_frequency.parquet new file mode 100644 index 0000000..ec16ea5 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/mandatory_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/non_mandatory_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/non_mandatory_tour_frequency.parquet new file mode 100644 index 0000000..00deedd Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/non_mandatory_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/school_location.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/school_location.parquet new file mode 100644 index 0000000..3b8219b Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/school_location.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/workplace_location.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/workplace_location.parquet new file mode 100644 index 0000000..450a487 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/persons/workplace_location.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/school_destination_size/initialize_households.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/school_destination_size/initialize_households.parquet new file mode 100644 index 0000000..ec369b9 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/school_destination_size/initialize_households.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/school_modeled_size/school_location.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/school_modeled_size/school_location.parquet new file mode 100644 index 0000000..77a6f03 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/school_modeled_size/school_location.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_destination.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_destination.parquet new file mode 100644 index 0000000..04ffb11 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_destination.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_frequency.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_frequency.parquet new file mode 100644 index 0000000..df4d442 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_mode_choice.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_mode_choice.parquet new file mode 100644 index 0000000..d85beac Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_mode_choice.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_scheduling.parquet new file mode 100644 index 0000000..0956dcd Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/atwork_subtour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_composition.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_composition.parquet new file mode 100644 index 0000000..017121d Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_composition.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_destination.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_destination.parquet new file mode 100644 index 0000000..6f9b561 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_destination.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_frequency.parquet new file mode 100644 index 0000000..3996d4a Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_participation.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_participation.parquet new file mode 100644 index 0000000..2003dca Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_participation.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_scheduling.parquet new file mode 100644 index 0000000..518d449 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/joint_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/mandatory_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/mandatory_tour_frequency.parquet new file mode 100644 index 0000000..3ef44c4 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/mandatory_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/mandatory_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/mandatory_tour_scheduling.parquet new file mode 100644 index 0000000..bf1dfb6 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/mandatory_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/non_mandatory_tour_destination.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/non_mandatory_tour_destination.parquet new file mode 100644 index 0000000..324f764 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/non_mandatory_tour_destination.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/non_mandatory_tour_frequency.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/non_mandatory_tour_frequency.parquet new file mode 100644 index 0000000..ccfe0cd Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/non_mandatory_tour_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/non_mandatory_tour_scheduling.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/non_mandatory_tour_scheduling.parquet new file mode 100644 index 0000000..e970391 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/non_mandatory_tour_scheduling.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/stop_frequency.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/stop_frequency.parquet new file mode 100644 index 0000000..4673a81 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/stop_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/tour_mode_choice_simulate.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/tour_mode_choice_simulate.parquet new file mode 100644 index 0000000..9e2a239 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/tour_mode_choice_simulate.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/trip_scheduling_choice.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/trip_scheduling_choice.parquet new file mode 100644 index 0000000..c6503c3 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/tours/trip_scheduling_choice.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/parking_location.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/parking_location.parquet new file mode 100644 index 0000000..14de561 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/parking_location.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/stop_frequency.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/stop_frequency.parquet new file mode 100644 index 0000000..55a9f32 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/stop_frequency.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_departure_choice.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_departure_choice.parquet new file mode 100644 index 0000000..ea66b75 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_departure_choice.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_destination.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_destination.parquet new file mode 100644 index 0000000..20eba49 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_destination.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_mode_choice.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_mode_choice.parquet new file mode 100644 index 0000000..a9e1a49 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_mode_choice.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_purpose.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_purpose.parquet new file mode 100644 index 0000000..3ad50fb Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_purpose.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_purpose_and_destination.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_purpose_and_destination.parquet new file mode 100644 index 0000000..e055c41 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/trips/trip_purpose_and_destination.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/workplace_destination_size/initialize_households.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/workplace_destination_size/initialize_households.parquet new file mode 100644 index 0000000..dbc4e9a Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/workplace_destination_size/initialize_households.parquet differ diff --git a/tests/regress/reference-pipeline-hh1000.parquetpipeline/workplace_modeled_size/workplace_location.parquet b/tests/regress/reference-pipeline-hh1000.parquetpipeline/workplace_modeled_size/workplace_location.parquet new file mode 100644 index 0000000..ae70154 Binary files /dev/null and b/tests/regress/reference-pipeline-hh1000.parquetpipeline/workplace_modeled_size/workplace_location.parquet differ diff --git a/tests/test_activitysim.py b/tests/test_activitysim.py new file mode 100644 index 0000000..c17b55a --- /dev/null +++ b/tests/test_activitysim.py @@ -0,0 +1,133 @@ +import os +from pathlib import Path + +import pytest + +from activitysim.core import workflow + + +def _example_path(dirname): + """Paths to things in the top-level directory of this repository.""" + return os.path.normpath(os.path.join(os.path.dirname(__file__), "..", dirname)) + + +def _test_path(dirname) -> Path: + """Paths to things in the `tests` directory.""" + return Path(__file__).parent.joinpath(dirname) + + +EXPECTED_MODELS = [ + "initialize_landuse", + "compute_accessibility", + "initialize_households", + "school_location", + "workplace_location", + "auto_ownership_simulate", + "free_parking", + "cdap_simulate", + "mandatory_tour_frequency", + "mandatory_tour_scheduling", + "joint_tour_frequency", + "joint_tour_composition", + "joint_tour_participation", + "joint_tour_destination", + "joint_tour_scheduling", + "non_mandatory_tour_frequency", + "non_mandatory_tour_destination", + "non_mandatory_tour_scheduling", + "tour_mode_choice_simulate", + "atwork_subtour_frequency", + "atwork_subtour_destination", + "atwork_subtour_scheduling", + "atwork_subtour_mode_choice", + "stop_frequency", + "trip_purpose", + "trip_destination", + "trip_purpose_and_destination", + "trip_scheduling_choice", + "trip_departure_choice", + "trip_mode_choice", + "parking_location", + "write_data_dictionary", + "track_skim_usage", + "write_trip_matrices", + "write_tables", +] + + +@pytest.mark.parametrize("use_sharrow", [False, True]) +def test_arc_progressive(use_sharrow): + import activitysim.abm # register components # noqa: F401 + + out_dir = _test_path( + "output-progressive-sharrow" if use_sharrow else "output-progressive" + ) + out_dir.mkdir(exist_ok=True) + out_dir.joinpath(".gitignore").write_text("**\n") + + settings = dict( + cleanup_pipeline_after_run=False, + treat_warnings_as_errors=True, + households_sample_size=1000, + chunk_size=0, + use_shadow_pricing=True, + multiprocess=False, + ) + tags = ["-hh1000"] + + if use_sharrow: + settings["sharrow"] = "test" + settings["recode_pipeline_columns"] = True + tags.append("-recode") + + state = workflow.State.make_default( + configs_dir=(_example_path(r"configs"),), + data_dir=_example_path("arc-fulton-data"), + output_dir=out_dir, + settings=settings, + ) + state.logging.config_logger() + + assert state.settings.models == EXPECTED_MODELS + assert state.settings.chunk_size == 0 + if not use_sharrow: + assert not state.settings.sharrow + + tags = "".join(tags) + ref_pipeline = Path(__file__).parent.joinpath( + f"regress/reference-pipeline{tags}.parquetpipeline" + ) + + for step_name in EXPECTED_MODELS: + state.run.by_name(step_name) + if ref_pipeline.exists(): + try: + # The usual default rtol=1e-5 is too strict for cross-platform testing + state.checkpoint.check_against( + ref_pipeline, checkpoint_name=step_name, rtol=3.3e-5 + ) + except Exception: + print(f"> {step_name}: ERROR") + raise + else: + print(f"> {step_name}: ok") + else: + print(f"> {step_name}: ran, not checked (no reference pipeline)") + + if not ref_pipeline.exists(): + # make new reference pipeline file if it is missing + import shutil + + if ref_pipeline.suffix == ".zip": + shutil.make_archive( + ref_pipeline.with_suffix(""), "zip", state.checkpoint.store.filename + ) + else: + shutil.copytree(state.checkpoint.store.filename, ref_pipeline) + if (ref_pipeline / ".gitignore").exists(): + os.remove(ref_pipeline / ".gitignore") + + +if __name__ == "__main__": + test_arc_progressive(use_sharrow=False) + test_arc_progressive(use_sharrow=True)