diff --git a/tcmalloc/allocation_sampling.h b/tcmalloc/allocation_sampling.h index 818bbb602..ac25a46ee 100644 --- a/tcmalloc/allocation_sampling.h +++ b/tcmalloc/allocation_sampling.h @@ -50,11 +50,9 @@ std::unique_ptr DumpHeapProfile(Static& state); // avoid relying on the percpu code. extern "C" ABSL_CONST_INIT thread_local Sampler tcmalloc_sampler ABSL_ATTRIBUTE_INITIAL_EXEC; - -#endif - ABSL_CONST_INIT ABSL_ATTRIBUTE_WEAK thread_local Sampler tcmalloc_sampler ABSL_ATTRIBUTE_INITIAL_EXEC; +#endif inline Sampler& GetThreadSampler() { static_assert(sizeof(Sampler) == TCMALLOC_SAMPLER_SIZE, diff --git a/tcmalloc/internal/percpu.cc b/tcmalloc/internal/percpu.cc index fadc2ffa1..e4c94abfd 100644 --- a/tcmalloc/internal/percpu.cc +++ b/tcmalloc/internal/percpu.cc @@ -144,6 +144,9 @@ static void InitPerCpu() { // objects can overlap. volatile auto slabs_addr = reinterpret_cast(&tcmalloc_slabs); auto rseq_abi_addr = reinterpret_cast(&__rseq_abi); + volatile auto tcmalloc_rseq_layout_addr = + reinterpret_cast(&tcmalloc_rseq_layout); + TC_CHECK_EQ(rseq_abi_addr, tcmalloc_rseq_layout_addr); // Ensure __rseq_abi alignment required by ABI. TC_CHECK_EQ(rseq_abi_addr % 32, 0); // Ensure that all our TLS data is in a single cache line. diff --git a/tcmalloc/internal/percpu.h b/tcmalloc/internal/percpu.h index 8fa0b6cf5..ce80047b0 100644 --- a/tcmalloc/internal/percpu.h +++ b/tcmalloc/internal/percpu.h @@ -189,6 +189,15 @@ ABSL_CONST_INIT thread_local volatile kernel_rseq __rseq_abi }; ABSL_CONST_INIT thread_local volatile int tcmalloc_cached_vcpu ABSL_ATTRIBUTE_WEAK = kCpuIdUninitialized; +ABSL_CONST_INIT thread_local char tcmalloc_sampler ABSL_ATTRIBUTE_WEAK = 0; + +// Because of the weak definitions we also add this non-weak symbol. +// It's not used other than ensuring that our asm file with the real TLS layout +// is actually linked. Without it linker manages to not link the asm file +// in some configurations since all symbols it provides are already defined +// (as weak), even though the asm file is in the same static library as other +// linked in files. +extern "C" ABSL_CONST_INIT thread_local char tcmalloc_rseq_layout; inline int GetRealCpuUnsafe() { return __rseq_abi.cpu_id; } #else // !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ diff --git a/tcmalloc/internal/percpu_rseq_asm.S b/tcmalloc/internal/percpu_rseq_asm.S index 1219f8a89..40d708246 100644 --- a/tcmalloc/internal/percpu_rseq_asm.S +++ b/tcmalloc/internal/percpu_rseq_asm.S @@ -36,11 +36,13 @@ .type tcmalloc_cached_vcpu, @object .type tcmalloc_slabs, @object .type __rseq_abi, @object +.type tcmalloc_rseq_layout, @object .section .tdata, "awT", @progbits .globl tcmalloc_sampler .globl tcmalloc_cached_vcpu .globl tcmalloc_slabs .globl __rseq_abi +.globl tcmalloc_rseq_layout .p2align 6 .zero 64 + 32 - TCMALLOC_SAMPLER_SIZE - 8 tcmalloc_sampler: @@ -49,6 +51,7 @@ tcmalloc_cached_vcpu: .long 0xffffffff // cpu_id (kCpuIdUninitialized) tcmalloc_slabs: .long 0 +tcmalloc_rseq_layout: __rseq_abi: .long 0 // cpu_id_start .long 0xffffffff // cpu_id (kCpuIdUninitialized)