Skip to content

Commit

Permalink
Merge pull request #2727 from ROCm/develop-upstream-sync-241021
Browse files Browse the repository at this point in the history
Develop upstream sync 241021
  • Loading branch information
zoranjovanovic-ns authored Oct 25, 2024
2 parents 575fdfd + 58fc0de commit db5ae54
Show file tree
Hide file tree
Showing 2,401 changed files with 59,417 additions and 37,493 deletions.
5 changes: 5 additions & 0 deletions RELEASE.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ This release contains contributions from many people at Google, as well as:

Hermetic CUDA uses a specific downloadable version of CUDA instead of the user’s locally installed CUDA. Bazel will download CUDA, CUDNN and NCCL distributions, and then use CUDA libraries and tools as dependencies in various Bazel targets. This enables more reproducible builds for Google ML projects and supported CUDA versions.

* Remove the `EnumNamesXNNPackFlags` function in `tensorflow/lite/acceleration/configuration/configuration_generated.h`.

This change is a bug fix in the automatically generated code. This change is automatically generated by the new flatbuffer generator. The flatbuffers library is updated to 24.3.25 in https://github.com/tensorflow/tensorflow/commit/c17d64df85a83c1bd0fd7dcc0b1230812b0d3d48. The new flatbuffers library includes the following change https://github.com/google/flatbuffers/pull/7813 which fixed a underlying flatbuffer code generator bug.


### Known Caveats

### Major Features and Improvements
Expand Down
9 changes: 8 additions & 1 deletion ci/official/containers/ml_build/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ COPY builder.devtoolset/glibc2.17-inline.patch /glibc2.17-inline.patch
RUN /build_devtoolset.sh devtoolset-9 /dt9

# Setup Python
# TODO(quoct): Add python 3.13 when it's available on Oct 7th.
COPY setup.python.sh /setup.python.sh
COPY builder.requirements.txt /builder.requirements.txt
RUN /setup.python.sh python3.9 builder.requirements.txt
Expand All @@ -27,6 +26,14 @@ RUN /setup.python.sh python3.11 builder.requirements.txt
RUN /setup.python.sh python3.12 builder.requirements.txt
RUN /setup.python.sh python3.13 builder.requirements.txt

# Setup links for TensorFlow to compile.
# Referenced in devel.usertools/*.bazelrc.
# Set python3.12 as the default python version.
# TF does not support python3.13.
RUN ln -sf /usr/bin/python3.12 /usr/bin/python3
RUN ln -sf /usr/bin/python3.12 /usr/bin/python
RUN ln -sf /usr/lib/python3.12 /usr/lib/tf_python

# Make sure clang is on the path
RUN ln -s /usr/lib/llvm-18/bin/clang /usr/bin/clang

Expand Down
6 changes: 0 additions & 6 deletions ci/official/containers/ml_build/setup.python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,6 @@ for f in $(ls | grep python); do
done
popd

# Setup links for TensorFlow to compile.
# Referenced in devel.usertools/*.bazelrc
ln -sf /usr/bin/$VERSION /usr/bin/python3
ln -sf /usr/bin/$VERSION /usr/bin/python
ln -sf /usr/lib/$VERSION /usr/lib/tf_python

# Python 3.10 include headers fix:
# sysconfig.get_path('include') incorrectly points to /usr/local/include/python
# map /usr/include/python3.10 to /usr/local/include/python3.10
Expand Down
2 changes: 1 addition & 1 deletion ci/official/envs/linux_arm64
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_arm64_linux"
TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_arm64_linux"
TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_arm64
# Note: this is not set to "--cpu", because that changes the package name
# to tensorflow_cpu. These ARM builds are supposed to have the name "tensorflow"
Expand Down
2 changes: 1 addition & 1 deletion ci/official/envs/linux_arm64_cross_compile
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@
# limitations under the License.
# ==============================================================================
source ci/official/envs/linux_arm64
TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config cross_compile_linux_arm64"
TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config cross_compile_linux_arm64"
TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=cross_compile_linux_arm64
2 changes: 1 addition & 1 deletion ci/official/envs/linux_x86
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_cpu_linux"
TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_cpu_linux"
TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_cpu
TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow_cpu"
TFCI_DOCKER_ENABLE=1
Expand Down
29 changes: 29 additions & 0 deletions ci/official/envs/linux_x86_build
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Copyright 2024 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_cpu_linux"
TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_cpu
TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow_cpu"
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_IMAGE=tensorflow/build:2.18-python${TFCI_PYTHON_VERSION}
TFCI_DOCKER_PULL_ENABLE=1
TFCI_DOCKER_REBUILD_ARGS="--build-arg PYTHON_VERSION=python$TFCI_PYTHON_VERSION --target=devel tensorflow/tools/tf_sig_build_dockerfiles"
TFCI_INDEX_HTML_ENABLE=1
TFCI_LIB_SUFFIX="-cpu-linux-x86_64"
TFCI_OUTPUT_DIR=build_output
TFCI_WHL_AUDIT_ENABLE=1
TFCI_WHL_AUDIT_PLAT=manylinux2014_x86_64
TFCI_WHL_BAZEL_TEST_ENABLE=1
TFCI_WHL_SIZE_LIMIT=240M
TFCI_WHL_SIZE_LIMIT_ENABLE=1
2 changes: 1 addition & 1 deletion ci/official/envs/linux_x86_cuda
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.
# ==============================================================================
source ci/official/envs/linux_x86
TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_gpu_linux"
TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_gpu_linux"
TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_cuda
TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow"
TFCI_DOCKER_ARGS="--gpus all"
Expand Down
21 changes: 21 additions & 0 deletions ci/official/envs/linux_x86_cuda_build
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Copyright 2024 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
source ci/official/envs/linux_x86
TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_gpu_linux"
TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_cuda
TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow"
TFCI_DOCKER_ARGS="--gpus all"
TFCI_LIB_SUFFIX="-gpu-linux-x86_64"
TFCI_WHL_SIZE_LIMIT=610M
2 changes: 1 addition & 1 deletion ci/official/envs/linux_x86_tpu
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.
# ==============================================================================
source ci/official/envs/linux_x86
TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_cpu_linux --config=tpu"
TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_cpu_linux --config=tpu"
TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=linux_tpu
TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow_tpu"
TFCI_LIB_SUFFIX="-tpu-linux-x86_64"
Expand Down
2 changes: 1 addition & 1 deletion ci/official/envs/macos_arm64
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_macos_arm64"
TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config release_macos_arm64"
TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=macos_arm64
TFCI_BUILD_PIP_PACKAGE_ARGS="--repo_env=WHEEL_NAME=tensorflow"
TFCI_INDEX_HTML_ENABLE=1
Expand Down
2 changes: 1 addition & 1 deletion ci/official/envs/windows_x86
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@
TFCI_DOCKER_ENABLE=1
TFCI_DOCKER_PULL_ENABLE=1
TFCI_DOCKER_IMAGE="gcr.io/tensorflow-testing/tf-win2019-rbe@sha256:1082ef4299a72e44a84388f192ecefc81ec9091c146f507bc36070c089c0edcc"
TFCI_BAZEL_COMMON_ARGS="--repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION"
TFCI_BAZEL_COMMON_ARGS="--repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION"
TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX=windows_x86_cpu
TFCI_OUTPUT_DIR=build_output
8 changes: 6 additions & 2 deletions ci/official/pycpp.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,13 @@ if [[ "$TFCI_WHL_NUMPY_VERSION" == 1 ]]; then
fi

if [[ $TFCI_PYCPP_SWAP_TO_BUILD_ENABLE == 1 ]]; then
tfrun bazel build $TFCI_BAZEL_COMMON_ARGS --profile "$PROFILE_JSON_PATH" --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_pycpp_test"
tfrun bazel build $TFCI_BAZEL_COMMON_ARGS --profile "$PROFILE_JSON_PATH" --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_pycpp_test"
else
tfrun bazel test $TFCI_BAZEL_COMMON_ARGS --profile "$PROFILE_JSON_PATH" --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_pycpp_test"
tfrun bazel test $TFCI_BAZEL_COMMON_ARGS --profile "$PROFILE_JSON_PATH" --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_pycpp_test"
fi

if [ "$TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX" != "windows_x86_cpu" ]; then
tfrun bazel build $TFCI_BAZEL_COMMON_ARGS --config=cuda_wheel //tensorflow/tools/pip_package:wheel
fi

# Note: the profile can be viewed by visiting chrome://tracing in a Chrome browser.
Expand Down
2 changes: 1 addition & 1 deletion ci/official/wheel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,5 +45,5 @@ if [[ "$TFCI_ARTIFACT_STAGING_GCS_ENABLE" == 1 ]]; then
fi

if [[ "$TFCI_WHL_BAZEL_TEST_ENABLE" == 1 ]]; then
tfrun bazel test $TFCI_BAZEL_COMMON_ARGS $TFCI_BUILD_PIP_PACKAGE_ARGS --repo_env=HERMETIC_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_wheel_test"
tfrun bazel test $TFCI_BAZEL_COMMON_ARGS $TFCI_BUILD_PIP_PACKAGE_ARGS --repo_env=TF_PYTHON_VERSION=$TFCI_PYTHON_VERSION --config="${TFCI_BAZEL_TARGET_SELECTING_CONFIG_PREFIX}_wheel_test"
fi
36 changes: 36 additions & 0 deletions tensorflow/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -552,18 +552,30 @@ config_setting(

config_setting(
name = "linux_aarch64",
constraint_values = if_google(
["//third_party/bazel_platforms/os:linux"],
[],
),
values = {"cpu": "aarch64"},
visibility = ["//visibility:public"],
)

config_setting(
name = "linux_armhf",
constraint_values = if_google(
["//third_party/bazel_platforms/os:linux"],
[],
),
values = {"cpu": "armhf"},
visibility = ["//visibility:public"],
)

config_setting(
name = "linux_x86_64",
constraint_values = if_google(
["//third_party/bazel_platforms/os:linux"],
[],
),
values = {"cpu": "k8"},
visibility = ["//visibility:public"],
)
Expand All @@ -577,6 +589,10 @@ config_setting(
# This condition takes precedence over :linux_x86_64
config_setting(
name = "linux_x86_64_no_sse",
constraint_values = if_google(
["//third_party/bazel_platforms/os:linux"],
[],
),
values = {
"cpu": "k8",
"copt": "-mno-sse4.2",
Expand All @@ -588,31 +604,51 @@ config_setting(
# TODO(b/290533709): Remove this with PJRT build rule cleanup.
config_setting(
name = "linux_x86_64_with_weightwatcher",
constraint_values = if_google(
["//third_party/bazel_platforms/os:linux"],
[],
),
define_values = {"tensorflow_weightwatcher": "true"},
values = {"cpu": "k8"},
visibility = ["//visibility:public"],
)

config_setting(
name = "linux_ppc64le",
constraint_values = if_google(
["//third_party/bazel_platforms/os:linux"],
[],
),
values = {"cpu": "ppc"},
visibility = ["//visibility:public"],
)

config_setting(
name = "linux_s390x",
constraint_values = if_google(
["//third_party/bazel_platforms/os:linux"],
[],
),
values = {"cpu": "s390x"},
visibility = ["//visibility:public"],
)

config_setting(
name = "linux_mips64",
constraint_values = if_google(
["//third_party/bazel_platforms/os:linux"],
[],
),
values = {"cpu": "mips64"},
visibility = ["//visibility:public"],
)

config_setting(
name = "linux_riscv64",
constraint_values = if_google(
["//third_party/bazel_platforms/os:linux"],
[],
),
values = {"cpu": "riscv64"},
visibility = ["//visibility:public"],
)
Expand Down
2 changes: 2 additions & 0 deletions tensorflow/c/eager/parallel_device/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ cc_library(
"//tensorflow/c/eager:c_api_experimental",
"//tensorflow/c/eager:tfe_tensorhandle_internal",
"//tensorflow/core/platform:status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/types:optional",
"@com_google_absl//absl/types:variant",
Expand Down Expand Up @@ -114,6 +115,7 @@ tf_cc_test(
"//tensorflow/core:test_main",
"//tensorflow/core/common_runtime/eager:context",
"//tensorflow/core/platform:status",
"@com_google_absl//absl/status",
"@com_google_googletest//:gtest_main",
],
)
Expand Down
2 changes: 1 addition & 1 deletion tensorflow/c/eager/parallel_device/parallel_device.cc
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ limitations under the License.
#include <cstring>
#include <memory>

#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "absl/types/optional.h"
#include "absl/types/variant.h"
Expand All @@ -28,7 +29,6 @@ limitations under the License.
#include "tensorflow/c/tf_buffer.h"
#include "tensorflow/c/tf_status.h"
#include "tensorflow/c/tf_status_helper.h"
#include "tensorflow/core/platform/status.h"

namespace tensorflow {
namespace parallel_device {
Expand Down
1 change: 0 additions & 1 deletion tensorflow/c/eager/parallel_device/parallel_device_lib.cc
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ limitations under the License.
#include "tensorflow/core/platform/env.h"
#include "tensorflow/core/platform/errors.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/util/device_name_utils.h"
#include "tsl/platform/errors.h"
#include "tsl/platform/thread_annotations.h"
Expand Down
1 change: 1 addition & 0 deletions tensorflow/c/eager/parallel_device/parallel_device_lib.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ limitations under the License.
#include <utility>
#include <vector>

#include "absl/status/status.h"
#include "absl/types/optional.h"
#include "absl/types/span.h"
#include "absl/types/variant.h"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ limitations under the License.
#include "tensorflow/c/eager/parallel_device/parallel_device_lib.h"

#include <gmock/gmock.h>
#include "absl/status/status.h"
#include "tensorflow/c/c_api_experimental.h"
#include "tensorflow/c/eager/c_api.h"
#include "tensorflow/c/eager/c_api_experimental.h"
Expand All @@ -31,7 +32,6 @@ limitations under the License.
#include "tensorflow/core/framework/function.pb.h"
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/platform/status.h"
#include "tensorflow/core/platform/test.h"

namespace tensorflow {
Expand Down
6 changes: 2 additions & 4 deletions tensorflow/c/env.cc
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,7 @@ void TF_FileStat(const char* filename, TF_FileStatistics* stats,
TF_Status* status) {
::tensorflow::FileStatistics cc_stats;
TF_SetStatus(status, TF_OK, "");
::tensorflow::Status s =
::tensorflow::Env::Default()->Stat(filename, &cc_stats);
absl::Status s = ::tensorflow::Env::Default()->Stat(filename, &cc_stats);
::tensorflow::Set_TF_Status_from_Status(status, s);
if (s.ok()) {
stats->length = cc_stats.length;
Expand All @@ -73,8 +72,7 @@ void TF_NewWritableFile(const char* filename, TF_WritableFileHandle** handle,
TF_Status* status) {
std::unique_ptr<::tensorflow::WritableFile> f;
TF_SetStatus(status, TF_OK, "");
::tensorflow::Status s =
::tensorflow::Env::Default()->NewWritableFile(filename, &f);
absl::Status s = ::tensorflow::Env::Default()->NewWritableFile(filename, &f);
::tensorflow::Set_TF_Status_from_Status(status, s);

if (s.ok()) {
Expand Down
Loading

0 comments on commit db5ae54

Please sign in to comment.