Update dependencies for the nightly build (#6926)

* Update nightly build to use recent nightly packages
* Add test constraints for unit-tests with nightly TFX libraries

.github/workflows/ci-test.yml

@@ -52,7 +52,10 @@ jobs:
         python -m pip install --upgrade pip wheel
         # TODO(b/232490018): Cython need to be installed separately to build pycocotools.
         python -m pip install Cython -c ./test_constraints.txt
-        pip install -c ./test_constraints.txt --extra-index-url https://pypi-nightly.tensorflow.org/simple --pre .[all]
+        pip install \
+          -c ./${{ matrix.dependency-selector == 'NIGHTLY' && 'nightly_test_constraints.txt' || 'test_constraints.txt' }} \
+          --extra-index-url https://pypi-nightly.tensorflow.org/simple --pre .[all]
+
       env:
         TFX_DEPENDENCY_SELECTOR: ${{ matrix.dependency-selector }}
 

nightly_test_constraints.txt

@@ -0,0 +1,378 @@
+# nightly_test_constraints.txt
+# This file specifies the constraints for the test environment of tfx.
+# Unlike library dependency which aims to specify the widest version range
+# possible, it is okay to specify exact version here.
+#
+# constraints.txt file is similar to requirements.txt except it does not tell
+# to really "install" the specified target; it only specifies the version
+# constraint if it is installed either directly or transitively by the
+# dependencies.
+
+# TODO(b/321609768): Remove pinned Flask-session version after resolving the issue.
+Flask-session<0.6.0
+
+#TODO(b/329181965): Remove once we migrate TFX to 2.16.
+tensorflow==2.15.1
+tensorflow-text==2.15.0
+
+absl-py==1.4.0
+aiohappyeyeballs==2.4.3
+aiohttp==3.10.9
+aiosignal==1.3.1
+alembic==1.13.3
+annotated-types==0.7.0
+anyio==4.6.0
+apache-airflow==2.10.2
+apache-airflow-providers-common-compat==1.2.1rc1
+apache-airflow-providers-common-io==1.4.2rc1
+apache-airflow-providers-common-sql==1.18.0rc1
+apache-airflow-providers-fab==1.4.1rc1
+apache-airflow-providers-ftp==3.11.1
+apache-airflow-providers-http==4.13.1
+apache-airflow-providers-imap==3.7.0
+apache-airflow-providers-mysql==5.7.2rc1
+apache-airflow-providers-smtp==1.8.0
+apache-airflow-providers-sqlite==3.9.0
+apache-beam==2.59.0
+apispec==6.6.1
+argcomplete==3.5.1
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+array_record==0.5.1
+arrow==1.3.0
+asgiref==3.8.1
+astunparse==1.6.3
+async-lru==2.0.4
+async-timeout==4.0.3
+attrs==23.2.0
+babel==2.16.0
+backcall==0.2.0
+beautifulsoup4==4.12.3
+bleach==6.1.0
+blinker==1.8.2
+cachelib==0.9.0
+cachetools==5.5.0
+certifi==2024.8.30
+cffi==1.17.1
+cfgv==3.4.0
+charset-normalizer==3.4.0
+chex==0.1.86
+click==8.1.7
+clickclick==20.10.2
+cloudpickle==2.2.1
+colorama==0.4.6
+colorlog==6.8.2
+comm==0.2.2
+ConfigUpdater==3.2
+connexion==2.14.2
+cramjam==2.8.4
+crcmod==1.7
+cron-descriptor==1.4.5
+croniter==3.0.3
+cryptography==43.0.1
+Cython==3.0.11
+debugpy==1.8.7
+decorator==5.1.1
+defusedxml==0.7.1
+Deprecated==1.2.14
+dill==0.3.1.1
+distlib==0.3.9
+dm-tree==0.1.8
+dnspython==2.7.0
+docker==7.1.0
+docopt==0.6.2
+docstring_parser==0.16
+docutils==0.21.2
+email_validator==2.2.0
+etils==1.5.2
+exceptiongroup==1.2.2
+fastavro==1.9.7
+fasteners==0.19
+fastjsonschema==2.20.0
+filelock==3.16.1
+Flask==2.2.5
+Flask-AppBuilder==4.5.0
+Flask-Babel==2.0.0
+Flask-Caching==2.3.0
+Flask-JWT-Extended==4.6.0
+Flask-Limiter==3.8.0
+Flask-Login==0.6.3
+Flask-Session==0.5.0
+Flask-SQLAlchemy==2.5.1
+Flask-WTF==1.2.1
+flatbuffers==24.3.25
+flax==0.8.4
+fqdn==1.5.1
+frozenlist==1.4.1
+fsspec==2024.9.0
+gast==0.6.0
+google-api-core==2.21.0
+google-api-python-client==1.12.11
+google-apitools==0.5.31
+google-auth==2.35.0
+google-auth-httplib2==0.2.0
+google-auth-oauthlib==1.2.1
+google-cloud-aiplatform==1.70.0
+google-cloud-bigquery==3.26.0
+google-cloud-bigquery-storage==2.26.0
+google-cloud-bigtable==2.26.0
+google-cloud-core==2.4.1
+google-cloud-datastore==2.20.1
+google-cloud-dlp==3.23.0
+google-cloud-language==2.14.0
+google-cloud-pubsub==2.26.0
+google-cloud-pubsublite==1.11.1
+google-cloud-recommendations-ai==0.10.12
+google-cloud-resource-manager==1.12.5
+google-cloud-spanner==3.49.1
+google-cloud-storage==2.18.2
+google-cloud-videointelligence==2.13.5
+google-cloud-vision==3.7.4
+google-crc32c==1.6.0
+google-pasta==0.2.0
+google-re2==1.1.20240702
+google-resumable-media==2.7.2
+googleapis-common-protos==1.65.0
+greenlet==3.1.1
+grpc-google-iam-v1==0.13.1
+grpc-interceptor==0.15.4
+grpcio==1.66.2
+grpcio-status==1.48.2
+gunicorn==23.0.0
+h11==0.14.0
+h5py==3.12.1
+hdfs==2.7.3
+httpcore==1.0.6
+httplib2==0.22.0
+httpx==0.27.2
+identify==2.6.1
+idna==3.10
+importlib_metadata==8.4.0
+importlib_resources==6.4.5
+inflection==0.5.1
+iniconfig==2.0.0
+ipykernel==6.29.5
+ipython==7.34.0
+ipython-genutils==0.2.0
+ipywidgets==7.8.4
+isoduration==20.11.0
+itsdangerous==2.2.0
+jax==0.4.23
+jaxlib==0.4.23
+jedi==0.19.1
+Jinja2==3.1.4
+jmespath==1.0.1
+joblib==1.4.2
+Js2Py==0.74
+json5==0.9.25
+jsonpickle==3.3.0
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter-events==0.10.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter_server==2.13.0
+jupyter_server_terminals==0.5.3
+jupyterlab==4.2.5
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+jupyterlab_widgets==1.1.10
+keras==2.15.0
+keras-tuner==1.4.7
+kfp==2.5.0
+kfp-pipeline-spec==0.2.2
+kfp-server-api==2.0.5
+kt-legacy==1.0.5
+kubernetes==26.1.0
+lazy-object-proxy==1.10.0
+libclang==18.1.1
+limits==3.13.0
+linkify-it-py==2.0.3
+lockfile==0.12.2
+lxml==5.3.0
+Mako==1.3.5
+Markdown==3.7
+markdown-it-py==3.0.0
+MarkupSafe==3.0.1
+marshmallow==3.22.0
+marshmallow-oneofschema==3.1.1
+marshmallow-sqlalchemy==0.28.2
+matplotlib-inline==0.1.7
+mdit-py-plugins==0.4.2
+mdurl==0.1.2
+methodtools==0.4.7
+mistune==3.0.2
+ml-dtypes==0.3.2
+ml-metadata>=1.17.0.dev20241016
+mmh==2.2
+more-itertools==10.5.0
+msgpack==1.1.0
+multidict==6.1.0
+mysql-connector-python==9.0.0
+mysqlclient==2.2.4
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+nltk==3.9.1
+nodeenv==1.9.1
+notebook==7.2.2
+notebook_shim==0.2.4
+numpy==1.26.4
+oauth2client==4.1.3
+oauthlib==3.2.2
+objsize==0.7.0
+opentelemetry-api==1.27.0
+opentelemetry-exporter-otlp==1.27.0
+opentelemetry-exporter-otlp-proto-common==1.27.0
+opentelemetry-exporter-otlp-proto-grpc==1.27.0
+opentelemetry-exporter-otlp-proto-http==1.27.0
+opentelemetry-proto==1.27.0
+opentelemetry-sdk==1.27.0
+opentelemetry-semantic-conventions==0.48b0
+opt_einsum==3.4.0
+optax==0.2.2
+orbax-checkpoint==0.5.16
+ordered-set==4.1.0
+orjson==3.10.6
+overrides==7.7.0
+packaging==23.2
+pandas==1.5.3
+pandocfilters==1.5.1
+parso==0.8.4
+pathspec==0.12.1
+pendulum==3.0.0
+pexpect==4.9.0
+pickleshare==0.7.5
+pillow==10.4.0
+platformdirs==4.3.6
+pluggy==1.5.0
+portalocker==2.10.1
+portpicker==1.6.0
+pre_commit==4.0.1
+presto-python-client==0.7.0
+prison==0.2.1
+prometheus_client==0.21.0
+promise==2.3
+prompt_toolkit==3.0.48
+propcache==0.2.0
+proto-plus==1.24.0
+protobuf==3.20.3
+psutil==6.0.0
+ptyprocess==0.7.0
+pyarrow==10.0.1
+pyarrow-hotfix==0.6
+pyasn1==0.6.1
+pyasn1_modules==0.4.1
+pybind11==2.13.6
+pycparser==2.22
+pydantic==2.9.2
+pydantic_core==2.23.4
+pydot==1.4.2
+pyfarmhash==0.3.2
+Pygments==2.18.0
+pyjsparser==2.7.1
+PyJWT==2.9.0
+pymongo==4.10.1
+pyparsing==3.1.4
+pytest==8.0.0
+pytest-subtests==0.13.1
+python-daemon==3.0.1
+python-dateutil==2.9.0.post0
+python-json-logger==2.0.7
+python-nvd3==0.16.0
+python-slugify==8.0.4
+python-snappy==0.7.3
+pytz==2024.2
+PyYAML==6.0.2
+pyzmq==26.2.0
+redis==5.1.1
+referencing==0.35.1
+regex==2024.9.11
+requests==2.32.3
+requests-oauthlib==2.0.0
+requests-toolbelt==0.10.1
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.9.2
+rich-argparse==1.5.2
+rouge_score==0.1.2
+rpds-py==0.20.0
+rsa==4.9
+sacrebleu==2.4.3
+scikit-learn==1.5.1
+scipy==1.12.0
+Send2Trash==1.8.3
+setproctitle==1.3.3
+shapely==2.0.6
+six==1.16.0
+slackclient==2.9.4
+sniffio==1.3.1
+sounddevice==0.5.0
+soupsieve==2.6
+SQLAlchemy==1.4.54
+SQLAlchemy-JSONField==1.0.2
+SQLAlchemy-Utils==0.41.2
+sqlparse==0.5.1
+struct2tensor>=0.47.0.dev20240430; extra == "all"
+tabulate==0.9.0
+tenacity==9.0.0
+tensorboard==2.15.2
+tensorboard-data-server==0.7.2
+tensorflow==2.15.1
+tensorflow-cloud==0.1.16
+tensorflow-data-validation>=1.16.0.dev20240508
+tensorflow-datasets==4.9.3
+tensorflow-decision-forests==1.8.1
+tensorflow-estimator==2.15.0
+tensorflow-hub==0.15.0
+tensorflow-io==0.24.0
+tensorflow-io-gcs-filesystem==0.24.0
+tensorflow-metadata>=1.17.0.dev20241016
+tensorflow-ranking==0.5.5
+tensorflow-serving-api==2.15.1
+tensorflow-text==2.15.0
+tensorflow-transform>=1.16.0.dev20240430
+tensorflow_model_analysis>=0.47.0.dev20240617
+tensorflowjs==4.17.0
+tensorstore==0.1.66
+termcolor==2.5.0
+terminado==0.18.1
+text-unidecode==1.3
+tflite-support==0.4.4
+tfx-bsl>=1.16.0.dev20240430
+threadpoolctl==3.5.0
+time-machine==2.16.0
+tinycss2==1.3.0
+toml==0.10.2
+tomli==2.0.2
+toolz==1.0.0
+tornado==6.4.1
+tqdm==4.66.5
+traitlets==5.14.3
+types-python-dateutil==2.9.0.20241003
+typing_extensions==4.12.2
+tzdata==2024.2
+tzlocal==5.2
+uc-micro-py==1.0.3
+unicodecsv==0.14.1
+universal_pathlib==0.2.5
+uri-template==1.3.0
+uritemplate==3.0.1
+urllib3==1.26.20
+virtualenv==20.26.6
+wcwidth==0.2.13
+webcolors==24.8.0
+webencodings==0.5.1
+websocket-client==0.59.0
+Werkzeug==2.2.3
+widgetsnbextension==3.6.9
+wirerope==0.4.7
+wrapt==1.14.1
+WTForms==3.1.2
+wurlitzer==3.1.1
+yarl==1.14.0
+zipp==3.20.2
+zstandard==0.23.0