Update Docker files to support new toml project files (#6928)

Update the Dockerfile to use individual pyproject.toml files for ml-pipelines-sdk and tfx and add a requirements file for building the Docker image.

tfx/dependencies.py

@@ -171,7 +171,7 @@ def make_extra_packages_docker_image():
         "kfp>=2",
         "kfp-pipeline-spec>=0.2.2",
         "mmh>=2.2,<3",
-        "python-snappy>=0.5,<0.6",
+        "python-snappy>=0.7",
         # Required for tfx/examples/penguin/penguin_utils_cloud_tuner.py
         "tensorflow-cloud>=0.1,<0.2",
         "tensorflow-io>=0.9.0, <=0.24.0",

tfx/tools/docker/Dockerfile

@@ -27,18 +27,21 @@ WORKDIR ${TFX_DIR}
 ARG TFX_DEPENDENCY_SELECTOR
 ENV TFX_DEPENDENCY_SELECTOR=${TFX_DEPENDENCY_SELECTOR}
 
-RUN python -m pip install --upgrade pip
+RUN python -m pip install --upgrade pip wheel
 RUN python -m pip install tomli
 
 # TODO(b/175089240): clean up conditional checks on whether ml-pipelines-sdk is
 # built after TFX versions <= 0.25 are no longer eligible for cherry-picks.
 RUN cd ${TFX_DIR}/src; \
     if [ -e "package_build" ]; then \
       bash -x package_build/initialize.sh; \
+      cd package_build/ml-pipelines-sdk; \
       CFLAGS=$(/usr/bin/python-config --cflags) \
-        python package_build/ml-pipelines-sdk/setup.py bdist_wheel; \
+        python setup.py bdist_wheel; \
+      cd ../../package_build/tfx; \
       CFLAGS=$(/usr/bin/python-config --cflags) \
-        python package_build/tfx/setup.py bdist_wheel; \
+        python setup.py bdist_wheel; \
+      cd ../..; \
       MLSDK_WHEEL=$(find dist -name "ml_pipelines_sdk-*.whl"); \
       TFX_WHEEL=$(find dist -name "tfx-*.whl"); \
     else \
@@ -51,10 +54,10 @@ RUN cd ${TFX_DIR}/src; \
       CFLAGS=$(/usr/bin/python-config --cflags) \
         python -m pip install \
           --extra-index-url https://pypi-nightly.tensorflow.org/simple \
-          ${MLSDK_WHEEL} ${TFX_WHEEL}[docker-image] ; \
+          ${MLSDK_WHEEL} ${TFX_WHEEL}[docker-image] -c tfx/tools/docker/requirements.txt; \
     else \
       CFLAGS=$(/usr/bin/python-config --cflags) \
-        python -m pip install ${MLSDK_WHEEL} ${TFX_WHEEL}[docker-image]; \
+        python -m pip install ${MLSDK_WHEEL} ${TFX_WHEEL}[docker-image] -c tfx/tools/docker/requirements.txt; \
     fi;
 
 # We need to name this step for the next COPY --from command.