Merge pull request #6907 from aktech/introduce-pyproject.toml

Add basic `pyproject.toml`

.github/workflows/ci-test.yml

@@ -47,7 +47,7 @@ jobs:
       with:
         # This path is specific to Ubuntu
         path: ~/.cache/pip
-        # Look to see if there is a cache hit for the corresponding setup.py + TFX version 
+        # Look to see if there is a cache hit for the corresponding setup.py + TFX version
         key: ${{ runner.os }}-pip-${{ hashFiles('tfx/dependencies.py') }}-
         restore-keys: |
           ${{ runner.os }}-pip-
@@ -57,7 +57,7 @@ jobs:
         python -m pip install --upgrade pip wheel
         # TODO(b/232490018): Cython need to be installed separately to build pycocotools.
         python -m pip install Cython -c ./test_constraints.txt
-        TFX_DEPENDENCY_SELECTOR=NIGHTLY pip install -c ./test_constraints.txt --extra-index-url https://pypi-nightly.tensorflow.org/simple --pre --editable .[all]
+        TFX_DEPENDENCY_SELECTOR=NIGHTLY pip install -c ./test_constraints.txt --extra-index-url https://pypi-nightly.tensorflow.org/simple --pre .[all]
 
     - name: Run unit tests
       shell: bash

.github/workflows/wheels.yml

@@ -80,11 +80,14 @@ jobs:
         bazel info
         bazel version
 
+    - name: Install build
+      run: python -m pip install --upgrade pip build
+
     - name: Build wheels
       run: |
         package_build/initialize.sh
-        python package_build/tfx/setup.py bdist_wheel
-        python package_build/ml-pipelines-sdk/setup.py bdist_wheel
+        python -m build --wheel package_build/tfx/
+        python -m build --wheel package_build/ml-pipelines-sdk/
         mkdir wheelhouse
         mv dist/*.whl wheelhouse/
 

package_build/initialize.sh

@@ -28,6 +28,7 @@ do
   ln -sf $BASEDIR/dist $BASEDIR/package_build/$CONFIG_NAME/
   ln -sf $BASEDIR/tfx $BASEDIR/package_build/$CONFIG_NAME/
   ln -sf $BASEDIR/README*.md $BASEDIR/package_build/$CONFIG_NAME/
+  ln -sf $BASEDIR/LICENSE $BASEDIR/package_build/$CONFIG_NAME/
 
   rm -rf $BASEDIR/package_build/$CONFIG_NAME/build
   mkdir $BASEDIR/package_build/$CONFIG_NAME/build

package_build/ml-pipelines-sdk/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=72", "wheel", "tomli"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ml-pipelines-sdk"
+dynamic = ["version", "dependencies", "optional-dependencies", "scripts"]
+description = "A dependency-light distribution of the core pipeline authoring functionality of TensorFlow Extended (TFX)."
+readme = "README.md"
+license = { file = "LICENSE" }
+authors = [
+    { name = "Google LLC", email = "tensorflow-extended-dev@googlegroups.com" }
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Education",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Scientific/Engineering :: Mathematics",
+    "Topic :: Software Development",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Software Development :: Libraries :: Python Modules"
+]
+keywords = ["tensorflow", "tfx"]
+requires-python = ">=3.9,<3.11"
+[project.urls]
+Homepage = "https://www.tensorflow.org/tfx"
+Repository = "https://github.com/tensorflow/tfx"

package_build/tfx/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=72", "wheel", "tomli"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tfx"
+dynamic = ["version", "dependencies", "optional-dependencies", "scripts"]
+description = "TensorFlow Extended (TFX) is a TensorFlow-based general-purpose machine learning platform implemented at Google."
+readme = "README.md"
+license = { file = "LICENSE" }
+authors = [
+    { name = "Google LLC", email = "tensorflow-extended-dev@googlegroups.com" }
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Education",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Scientific/Engineering :: Mathematics",
+    "Topic :: Software Development",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Software Development :: Libraries :: Python Modules"
+]
+keywords = ["tensorflow", "tfx"]
+requires-python = ">=3.9,<3.11"
+[project.urls]
+Homepage = "https://www.tensorflow.org/tfx"
+Repository = "https://github.com/tensorflow/tfx"

pyproject.toml

@@ -0,0 +1,46 @@
+[build-system]
+requires = ["setuptools>=72", "wheel", "tomli"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tfx-dev"
+dynamic = ["version", "dependencies", "optional-dependencies", "scripts"]
+description = "TensorFlow Extended (TFX) is a TensorFlow-based general-purpose machine learning platform implemented at Google."
+readme = "README.md"
+license = { file = "LICENSE" }
+authors = [
+    { name = "Google LLC", email = "tensorflow-extended-dev@googlegroups.com" }
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Education",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Scientific/Engineering :: Mathematics",
+    "Topic :: Software Development",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Software Development :: Libraries :: Python Modules"
+]
+keywords = ["tensorflow", "tfx"]
+requires-python = ">=3.9,<3.11"
+[project.urls]
+Homepage = "https://www.tensorflow.org/tfx"
+Repository = "https://github.com/tensorflow/tfx"
+
+[tool.pytest.ini_options]
+addopts = "--verbose -m 'not end_to_end'"
+python_files = "*_test.py"
+norecursedirs = ["custom_components", ".*", "*.egg"]
+markers = [
+    "end_to_end: end-to-end tests which are slow and require more dependencies (deselect with '-m \"not end_to_end\"')",
+    "serial: mark tests that should not run in parallel"
+]

setup.py

@@ -30,8 +30,6 @@
 from distutils.command import build
 # pylint: enable=g-bad-import-order
 
-from tfx import dependencies
-from tfx import version
 from wheel import bdist_wheel
 
 # Prefer to import `package_config` from the setup.py script's directory. The
@@ -40,9 +38,16 @@
 # package build README at `package_build/README.md`.
 sys.path.insert(0, os.path.dirname(__file__))
 # pylint: disable=g-bad-import-order,g-import-not-at-top
-import package_config
+
+from tfx import dependencies
+from tfx import version
 # pylint: enable=g-bad-import-order,g-import-not-at-top
 
+import tomli
+
+pyproject_toml = tomli.load(open('pyproject.toml', 'rb'))
+package_name = pyproject_toml['project']['name']
+
 
 class _BdistWheelCommand(bdist_wheel.bdist_wheel):
   """Overrided bdist_wheel command.
@@ -74,20 +79,14 @@ class _UnsupportedDevBuildWheelCommand(_BdistWheelCommand):
 
   def finalize_options(self):
     if not os.environ.get('UNSUPPORTED_BUILD_TFX_DEV_WHEEL'):
-      raise Exception(
-          'Starting in version 0.26.0, pip package build for TFX has changed,'
-          'and `python setup.py bdist_wheel` can no longer be invoked '
-          'directly.\n\nFor instructions on how to build wheels for TFX, see '
-          'https://github.com/tensorflow/tfx/blob/master/package_build/'
-          'README.md.\n\nEditable pip installation for development is still '
-          'supported through `pip install -e`.')
+        logging.info("UNSUPPORTED_BUILD_TFX_DEV_WHEEL is not set, so we're not building a wheel.")
     super().finalize_options()
 
 
 class _BuildCommand(build.build):
   """Build everything that is needed to install.
 
-  This overrides the original distutils "build" command to to run gen_proto
+  This overrides the original distutils "build" command to run gen_proto
   command before any sub_commands.
 
   build command is also invoked from bdist_wheel and install command, therefore
@@ -190,7 +189,6 @@ def run(self):
 with open('README.ml-pipelines-sdk.md') as fp:
   _PIPELINES_SDK_LONG_DESCRIPTION = fp.read()
 
-package_name = package_config.PACKAGE_NAME
 tfx_extras_requires = {
     # In order to use 'docker-image' or 'all', system libraries specified
     # under 'tfx/tools/docker/Dockerfile' are required
@@ -257,20 +255,19 @@ def run(self):
 # that should be generated, the second part is the import path followed by a
 # colon (:) with the Click command group. After installation, the user can
 # invoke the CLI using "tfx <command_group> <sub_command> <flags>"
-TFX_ENTRY_POINTS = """
-    [console_scripts]
-    tfx=tfx.tools.cli.cli_main:cli_group
-"""
+TFX_ENTRY_POINTS = {
+    "console_scripts": ["tfx=tfx.tools.cli.cli_main:cli_group"]
+}
 ML_PIPELINES_SDK_ENTRY_POINTS = None
 
 # This `setup.py` file can be used to build packages in 3 configurations. See
 # the discussion in `package_build/README.md` for an overview. The `tfx` and
 # `ml-pipelines-sdk` pip packages can be built for distribution using the
-# selectable `package_config.PACKAGE_NAME` specifier. Additionally, for
+# selectable `package_name` specifier. Additionally, for
 # development convenience, the `tfx-dev` package containing the union of the
 # the `tfx` and `ml-pipelines-sdk` package can be installed as an editable
 # package using `pip install -e .`, but should not be built for distribution.
-if package_config.PACKAGE_NAME == 'tfx-dev':
+if package_name == 'tfx-dev':
   # Monolithic development package with the entirety of `tfx.*` and the full
   # set of dependencies. Functionally equivalent to the union of the "tfx" and
   # "tfx-pipeline-sdk" packages.
@@ -284,7 +281,7 @@ def run(self):
   build_wheel_command = _UnsupportedDevBuildWheelCommand  # pylint: disable=invalid-name
   # Include TFX entrypoints.
   entry_points = TFX_ENTRY_POINTS
-elif package_config.PACKAGE_NAME == 'ml-pipelines-sdk':
+elif package_name == 'ml-pipelines-sdk':
   # Core TFX pipeline authoring SDK, without dependency on component-specific
   # packages like "tensorflow" and "apache-beam".
   install_requires = dependencies.make_pipeline_sdk_required_install_packages()
@@ -297,7 +294,7 @@ def run(self):
   build_wheel_command = bdist_wheel.bdist_wheel  # pylint: disable=invalid-name
   # Include ML Pipelines SDK entrypoints.
   entry_points = ML_PIPELINES_SDK_ENTRY_POINTS
-elif package_config.PACKAGE_NAME == 'tfx':
+elif package_name == 'tfx':
   # Recommended installation package for TFX. This package builds on top of
   # the "ml-pipelines-sdk" pipeline authoring SDK package and adds first-party
   # TFX components and additional functionality.
@@ -314,49 +311,20 @@ def run(self):
   # Include TFX entrypoints.
   entry_points = TFX_ENTRY_POINTS
 else:
-  raise ValueError('Invalid package config: %r.' % package_config.PACKAGE_NAME)
+  raise ValueError('Invalid package config: %r.' % package_name)
 
 logging.info('Executing build for package %r.', package_name)
-
 setup(
-    name=package_name,
     version=version.__version__,
-    author='Google LLC',
-    author_email='tensorflow-extended-dev@googlegroups.com',
-    license='Apache 2.0',
-    classifiers=[
-        'Development Status :: 5 - Production/Stable',
-        'Intended Audience :: Developers',
-        'Intended Audience :: Education',
-        'Intended Audience :: Science/Research',
-        'License :: OSI Approved :: Apache Software License',
-        'Operating System :: OS Independent',
-        'Programming Language :: Python',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3 :: Only',
-        'Topic :: Scientific/Engineering',
-        'Topic :: Scientific/Engineering :: Artificial Intelligence',
-        'Topic :: Scientific/Engineering :: Mathematics',
-        'Topic :: Software Development',
-        'Topic :: Software Development :: Libraries',
-        'Topic :: Software Development :: Libraries :: Python Modules',
-    ],
     namespace_packages=[],
     install_requires=install_requires,
     extras_require=extras_require,
-    # TODO(b/158761800): Move to [build-system] requires in pyproject.toml.
-    setup_requires=[
-        'pytest-runner',
-    ],
     cmdclass={
         'bdist_wheel': build_wheel_command,
         'build': _BuildCommand,
         'develop': _DevelopCommand,
         'gen_proto': _GenProtoCommand,
     },
-    python_requires='>=3.9,<3.11',
     packages=packages,
     include_package_data=True,
     description=description,