Merge pull request #2727 from ROCm/develop-upstream-sync-241021

Develop upstream sync 241021

RELEASE.md

@@ -79,6 +79,11 @@ This release contains contributions from many people at Google, as well as:
 
   Hermetic CUDA uses a specific downloadable version of CUDA instead of the user’s locally installed CUDA. Bazel will download CUDA, CUDNN and NCCL distributions, and then use CUDA libraries and tools as dependencies in various Bazel targets. This enables more reproducible builds for Google ML projects and supported CUDA versions.
 
+* Remove the `EnumNamesXNNPackFlags` function in `tensorflow/lite/acceleration/configuration/configuration_generated.h`.
+
+  This change is a bug fix in the automatically generated code. This change is automatically generated by the new flatbuffer generator. The flatbuffers library is updated to 24.3.25 in https://github.com/tensorflow/tensorflow/commit/c17d64df85a83c1bd0fd7dcc0b1230812b0d3d48. The new flatbuffers library includes the following change https://github.com/google/flatbuffers/pull/7813 which fixed a underlying flatbuffer code generator bug.
+
+
 ### Known Caveats
 
 ### Major Features and Improvements

ci/official/containers/ml_build/Dockerfile

@@ -18,7 +18,6 @@ COPY builder.devtoolset/glibc2.17-inline.patch /glibc2.17-inline.patch
 RUN /build_devtoolset.sh devtoolset-9 /dt9
 
 # Setup Python
-# TODO(quoct): Add python 3.13 when it's available on Oct 7th.
 COPY setup.python.sh /setup.python.sh
 COPY builder.requirements.txt /builder.requirements.txt
 RUN /setup.python.sh python3.9 builder.requirements.txt
@@ -27,6 +26,14 @@ RUN /setup.python.sh python3.11 builder.requirements.txt
 RUN /setup.python.sh python3.12 builder.requirements.txt
 RUN /setup.python.sh python3.13 builder.requirements.txt
 
+# Setup links for TensorFlow to compile.
+# Referenced in devel.usertools/*.bazelrc.
+# Set python3.12 as the default python version.
+# TF does not support python3.13.
+RUN ln -sf /usr/bin/python3.12 /usr/bin/python3
+RUN ln -sf /usr/bin/python3.12 /usr/bin/python
+RUN ln -sf /usr/lib/python3.12 /usr/lib/tf_python
+
 # Make sure clang is on the path
 RUN ln -s /usr/lib/llvm-18/bin/clang /usr/bin/clang
 

ci/official/containers/ml_build/setup.python.sh

@@ -51,12 +51,6 @@ for f in $(ls | grep python); do
 done
 popd
 
-# Setup links for TensorFlow to compile.
-# Referenced in devel.usertools/*.bazelrc
-ln -sf /usr/bin/$VERSION /usr/bin/python3
-ln -sf /usr/bin/$VERSION /usr/bin/python
-ln -sf /usr/lib/$VERSION /usr/lib/tf_python
-
 # Python 3.10 include headers fix:
 # sysconfig.get_path('include') incorrectly points to /usr/local/include/python
 # map /usr/include/python3.10 to /usr/local/include/python3.10

ci/official/envs/linux_arm64

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_arm64_linux"
+TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_arm64_linux"
 TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_arm64
 # Note: this is not set to "--cpu", because that changes the package name
 # to tensorflow_cpu. These ARM builds are supposed to have the name "tensorflow"

ci/official/envs/linux_arm64_cross_compile

@@ -13,5 +13,5 @@
 # limitations under the License.
 # ==============================================================================
 source ci/official/envs/linux_arm64
-TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config  cross_compile_linux_arm64"
+TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config  cross_compile_linux_arm64"
 TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=cross_compile_linux_arm64

ci/official/envs/linux_x86

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_cpu_linux"
+TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_cpu_linux"
 TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_cpu
 TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow_cpu"
 TFCI_DOCKER_ENABLE=1

ci/official/envs/linux_x86_build

@@ -0,0 +1,29 @@
+# Copyright 2024 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_cpu_linux"
+TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_cpu
+TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow_cpu"
+TFCI_DOCKER_ENABLE=1
+TFCI_DOCKER_IMAGE=tensorflow/build:2.18-python${TFCI_PYTHON_VERSION}
+TFCI_DOCKER_PULL_ENABLE=1
+TFCI_DOCKER_REBUILD_ARGS="--build-arg PYTHON_VERSION=python$TFCI_PYTHON_VERSION --target=devel tensorflow/tools/tf_sig_build_dockerfiles"
+TFCI_INDEX_HTML_ENABLE=1
+TFCI_LIB_SUFFIX="-cpu-linux-x86_64"
+TFCI_OUTPUT_DIR=build_output
+TFCI_WHL_AUDIT_ENABLE=1
+TFCI_WHL_AUDIT_PLAT=manylinux2014_x86_64
+TFCI_WHL_BAZEL_TEST_ENABLE=1
+TFCI_WHL_SIZE_LIMIT=240M
+TFCI_WHL_SIZE_LIMIT_ENABLE=1

ci/official/envs/linux_x86_cuda

@@ -13,7 +13,7 @@
 # limitations under the License.
 # ==============================================================================
 source ci/official/envs/linux_x86
-TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_gpu_linux"
+TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_gpu_linux"
 TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_cuda
 TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow"
 TFCI_DOCKER_ARGS="--gpus all"

ci/official/envs/linux_x86_cuda_build

@@ -0,0 +1,21 @@
+# Copyright 2024 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+source ci/official/envs/linux_x86
+TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_gpu_linux"
+TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_cuda
+TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow"
+TFCI_DOCKER_ARGS="--gpus all"
+TFCI_LIB_SUFFIX="-gpu-linux-x86_64"
+TFCI_WHL_SIZE_LIMIT=610M

ci/official/envs/linux_x86_tpu

@@ -13,7 +13,7 @@
 # limitations under the License.
 # ==============================================================================
 source ci/official/envs/linux_x86
-TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_cpu_linux --config=tpu"
+TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_cpu_linux --config=tpu"
 TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_tpu
 TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow_tpu"
 TFCI_LIB_SUFFIX="-tpu-linux-x86_64"

ci/official/envs/macos_arm64

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_macos_arm64"
+TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_macos_arm64"
 TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=macos_arm64
 TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow"
 TFCI_INDEX_HTML_ENABLE=1

ci/official/envs/windows_x86

@@ -15,6 +15,6 @@
 TFCI_DOCKER_ENABLE=1
 TFCI_DOCKER_PULL_ENABLE=1
 TFCI_DOCKER_IMAGE="gcr.io/tensorflow-testing/tf-win2019-rbe@sha256:1082ef4299a72e44a84388f192ecefc81ec9091c146f507bc36070c089c0edcc"
-TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION"
+TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION"
 TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=windows_x86_cpu
 TFCI_OUTPUT_DIR=build_output

ci/official/pycpp.sh

@@ -29,9 +29,13 @@ if [[ "$TFCI_WHL_NUMPY_VERSION" == 1 ]]; then
 fi
 
 if [[ $TFCI_PYCPP_SWAP_TO_BUILD_ENABLE == 1 ]]; then
-   tfrun bazel build $TFCI_BAZEL_COMMON_ARGS --profile "$PROFILE_JSON_PATH" --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_pycpp_test"
+  tfrun bazel build $TFCI_BAZEL_COMMON_ARGS --profile "$PROFILE_JSON_PATH" --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_pycpp_test"
 else
-   tfrun bazel test $TFCI_BAZEL_COMMON_ARGS --profile "$PROFILE_JSON_PATH"  --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_pycpp_test"
+  tfrun bazel test $TFCI_BAZEL_COMMON_ARGS --profile "$PROFILE_JSON_PATH"  --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_pycpp_test"
+fi
+
+if [ "$TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX" != "windows_x86_cpu" ]; then
+  tfrun bazel build $TFCI_BAZEL_COMMON_ARGS --config=cuda_wheel //tensorflow/tools/pip_package:wheel
 fi
 
 # Note: the profile can be viewed by visiting chrome://tracing in a Chrome browser.

ci/official/wheel.sh

@@ -45,5 +45,5 @@ if [[ "$TFCI_ARTIFACT_STAGING_GCS_ENABLE" == 1 ]]; then
 fi
 
 if [[ "$TFCI_WHL_BAZEL_TEST_ENABLE" == 1 ]]; then
-  tfrun bazel test $TFCI_BAZEL_COMMON_ARGS $TFCI_BUILD_PIP_PACKAGE_ARGS --repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_wheel_test"
+  tfrun bazel test $TFCI_BAZEL_COMMON_ARGS $TFCI_BUILD_PIP_PACKAGE_ARGS --repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_wheel_test"
 fi

tensorflow/BUILD

@@ -552,18 +552,30 @@ config_setting(
 
 config_setting(
     name = "linux_aarch64",
+    constraint_values = if_google(
+        ["//third_party/bazel_platforms/os:linux"],
+        [],
+    ),
     values = {"cpu": "aarch64"},
     visibility = ["//visibility:public"],
 )
 
 config_setting(
     name = "linux_armhf",
+    constraint_values = if_google(
+        ["//third_party/bazel_platforms/os:linux"],
+        [],
+    ),
     values = {"cpu": "armhf"},
     visibility = ["//visibility:public"],
 )
 
 config_setting(
     name = "linux_x86_64",
+    constraint_values = if_google(
+        ["//third_party/bazel_platforms/os:linux"],
+        [],
+    ),
     values = {"cpu": "k8"},
     visibility = ["//visibility:public"],
 )
@@ -577,6 +589,10 @@ config_setting(
 # This condition takes precedence over :linux_x86_64
 config_setting(
     name = "linux_x86_64_no_sse",
+    constraint_values = if_google(
+        ["//third_party/bazel_platforms/os:linux"],
+        [],
+    ),
     values = {
         "cpu": "k8",
         "copt": "-mno-sse4.2",
@@ -588,31 +604,51 @@ config_setting(
 # TODO(b/290533709): Remove this with PJRT build rule cleanup.
 config_setting(
     name = "linux_x86_64_with_weightwatcher",
+    constraint_values = if_google(
+        ["//third_party/bazel_platforms/os:linux"],
+        [],
+    ),
     define_values = {"tensorflow_weightwatcher": "true"},
     values = {"cpu": "k8"},
     visibility = ["//visibility:public"],
 )
 
 config_setting(
     name = "linux_ppc64le",
+    constraint_values = if_google(
+        ["//third_party/bazel_platforms/os:linux"],
+        [],
+    ),
     values = {"cpu": "ppc"},
     visibility = ["//visibility:public"],
 )
 
 config_setting(
     name = "linux_s390x",
+    constraint_values = if_google(
+        ["//third_party/bazel_platforms/os:linux"],
+        [],
+    ),
     values = {"cpu": "s390x"},
     visibility = ["//visibility:public"],
 )
 
 config_setting(
     name = "linux_mips64",
+    constraint_values = if_google(
+        ["//third_party/bazel_platforms/os:linux"],
+        [],
+    ),
     values = {"cpu": "mips64"},
     visibility = ["//visibility:public"],
 )
 
 config_setting(
     name = "linux_riscv64",
+    constraint_values = if_google(
+        ["//third_party/bazel_platforms/os:linux"],
+        [],
+    ),
     values = {"cpu": "riscv64"},
     visibility = ["//visibility:public"],
 )

tensorflow/c/eager/parallel_device/BUILD

@@ -65,6 +65,7 @@ cc_library(
         "//tensorflow/c/eager:c_api_experimental",
         "//tensorflow/c/eager:tfe_tensorhandle_internal",
         "//tensorflow/core/platform:status",
+        "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/types:optional",
         "@com_google_absl//absl/types:variant",
@@ -114,6 +115,7 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core/common_runtime/eager:context",
         "//tensorflow/core/platform:status",
+        "@com_google_absl//absl/status",
         "@com_google_googletest//:gtest_main",
     ],
 )

tensorflow/c/eager/parallel_device/parallel_device.cc

@@ -18,6 +18,7 @@ limitations under the License.
 #include <cstring>
 #include <memory>
 
+#include "absl/status/status.h"
 #include "absl/strings/str_cat.h"
 #include "absl/types/optional.h"
 #include "absl/types/variant.h"
@@ -28,7 +29,6 @@ limitations under the License.
 #include "tensorflow/c/tf_buffer.h"
 #include "tensorflow/c/tf_status.h"
 #include "tensorflow/c/tf_status_helper.h"
-#include "tensorflow/core/platform/status.h"
 
 namespace tensorflow {
 namespace parallel_device {

tensorflow/c/eager/parallel_device/parallel_device_lib.cc

@@ -37,7 +37,6 @@ limitations under the License.
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/errors.h"
 #include "tensorflow/core/platform/mutex.h"
-#include "tensorflow/core/platform/status.h"
 #include "tensorflow/core/util/device_name_utils.h"
 #include "tsl/platform/errors.h"
 #include "tsl/platform/thread_annotations.h"

tensorflow/c/eager/parallel_device/parallel_device_lib.h

@@ -22,6 +22,7 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "absl/status/status.h"
 #include "absl/types/optional.h"
 #include "absl/types/span.h"
 #include "absl/types/variant.h"

tensorflow/c/eager/parallel_device/parallel_device_lib_test.cc

@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/c/eager/parallel_device/parallel_device_lib.h"
 
 #include <gmock/gmock.h>
+#include "absl/status/status.h"
 #include "tensorflow/c/c_api_experimental.h"
 #include "tensorflow/c/eager/c_api.h"
 #include "tensorflow/c/eager/c_api_experimental.h"
@@ -31,7 +32,6 @@ limitations under the License.
 #include "tensorflow/core/framework/function.pb.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/types.pb.h"
-#include "tensorflow/core/platform/status.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {

tensorflow/c/env.cc

@@ -59,8 +59,7 @@ void TF_FileStat(const char* filename, TF_FileStatistics* stats,
                  TF_Status* status) {
   ::tensorflow::FileStatistics cc_stats;
   TF_SetStatus(status, TF_OK, "");
-  ::tensorflow::Status s =
-      ::tensorflow::Env::Default()->Stat(filename, &cc_stats);
+  absl::Status s = ::tensorflow::Env::Default()->Stat(filename, &cc_stats);
   ::tensorflow::Set_TF_Status_from_Status(status, s);
   if (s.ok()) {
     stats->length = cc_stats.length;
@@ -73,8 +72,7 @@ void TF_NewWritableFile(const char* filename, TF_WritableFileHandle** handle,
                         TF_Status* status) {
   std::unique_ptr<::tensorflow::WritableFile> f;
   TF_SetStatus(status, TF_OK, "");
-  ::tensorflow::Status s =
-      ::tensorflow::Env::Default()->NewWritableFile(filename, &f);
+  absl::Status s = ::tensorflow::Env::Default()->NewWritableFile(filename, &f);
   ::tensorflow::Set_TF_Status_from_Status(status, s);
 
   if (s.ok()) {