diff --git a/deps/jemalloc/.appveyor.yml b/deps/jemalloc/.appveyor.yml index 90b0368..9a7d00a 100644 --- a/deps/jemalloc/.appveyor.yml +++ b/deps/jemalloc/.appveyor.yml @@ -5,27 +5,27 @@ environment: - MSYSTEM: MINGW64 CPU: x86_64 MSVC: amd64 - CONFIG_FLAGS: --enable-debug - - MSYSTEM: MINGW64 - CPU: x86_64 - CONFIG_FLAGS: --enable-debug - MSYSTEM: MINGW32 CPU: i686 MSVC: x86 - CONFIG_FLAGS: --enable-debug + - MSYSTEM: MINGW64 + CPU: x86_64 - MSYSTEM: MINGW32 CPU: i686 - CONFIG_FLAGS: --enable-debug - MSYSTEM: MINGW64 CPU: x86_64 MSVC: amd64 - - MSYSTEM: MINGW64 - CPU: x86_64 + CONFIG_FLAGS: --enable-debug - MSYSTEM: MINGW32 CPU: i686 MSVC: x86 + CONFIG_FLAGS: --enable-debug + - MSYSTEM: MINGW64 + CPU: x86_64 + CONFIG_FLAGS: --enable-debug - MSYSTEM: MINGW32 CPU: i686 + CONFIG_FLAGS: --enable-debug install: - set PATH=c:\msys64\%MSYSTEM%\bin;c:\msys64\usr\bin;%PATH% diff --git a/deps/jemalloc/.cirrus.yml b/deps/jemalloc/.cirrus.yml deleted file mode 100644 index 019d2c3..0000000 --- a/deps/jemalloc/.cirrus.yml +++ /dev/null @@ -1,21 +0,0 @@ -env: - CIRRUS_CLONE_DEPTH: 1 - ARCH: amd64 - -task: - freebsd_instance: - matrix: - image: freebsd-12-0-release-amd64 - image: freebsd-11-2-release-amd64 - install_script: - - sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf - - pkg upgrade -y - - pkg install -y autoconf gmake - script: - - autoconf - #- ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" CXX="$CXX $COMPILER_FLAGS" } $CONFIGURE_FLAGS - - ./configure - - export JFLAG=`sysctl -n kern.smp.cpus` - - gmake -j${JFLAG} - - gmake -j${JFLAG} tests - - gmake check diff --git a/deps/jemalloc/.gitignore b/deps/jemalloc/.gitignore index 5ca0ad1..19199cc 100644 --- a/deps/jemalloc/.gitignore +++ b/deps/jemalloc/.gitignore @@ -30,6 +30,7 @@ /include/jemalloc/internal/public_namespace.h /include/jemalloc/internal/public_symbols.txt /include/jemalloc/internal/public_unnamespace.h +/include/jemalloc/internal/size_classes.h /include/jemalloc/jemalloc.h /include/jemalloc/jemalloc_defs.h /include/jemalloc/jemalloc_macros.h diff --git a/deps/jemalloc/.travis.yml b/deps/jemalloc/.travis.yml index 2da5da8..4cc116e 100644 --- a/deps/jemalloc/.travis.yml +++ b/deps/jemalloc/.travis.yml @@ -11,7 +11,7 @@ matrix: env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: &gcc_multilib + addons: apt: packages: - gcc-multilib @@ -21,10 +21,6 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -41,25 +37,20 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: osx env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: osx - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: osx - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: osx env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: *gcc_multilib + addons: + apt: + packages: + - gcc-multilib - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -70,39 +61,50 @@ matrix: env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: *gcc_multilib + addons: + apt: + packages: + - gcc-multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: *gcc_multilib + addons: + apt: + packages: + - gcc-multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: *gcc_multilib - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: *gcc_multilib - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: *gcc_multilib + addons: + apt: + packages: + - gcc-multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: *gcc_multilib + addons: + apt: + packages: + - gcc-multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: *gcc_multilib + addons: + apt: + packages: + - gcc-multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: *gcc_multilib + addons: + apt: + packages: + - gcc-multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: *gcc_multilib + addons: + apt: + packages: + - gcc-multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -113,10 +115,6 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -125,10 +123,6 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -137,24 +131,6 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --enable-opt-safety-checks" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-opt-safety-checks --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false,dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -167,25 +143,10 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary,background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu,background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - # Development build - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - # --enable-expermental-smallocx: - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - - # Valgrind - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind" - addons: - apt: - packages: - - valgrind before_script: - autoconf - - scripts/gen_travis.py > travis_script && diff .travis.yml travis_script - ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" CXX="$CXX $COMPILER_FLAGS" } $CONFIGURE_FLAGS - make -j3 - make -j3 tests diff --git a/deps/jemalloc/COPYING b/deps/jemalloc/COPYING index 3b7fd35..98458d9 100644 --- a/deps/jemalloc/COPYING +++ b/deps/jemalloc/COPYING @@ -1,10 +1,10 @@ Unless otherwise specified, files in the jemalloc source distribution are subject to the following license: -------------------------------------------------------------------------------- -Copyright (C) 2002-present Jason Evans . +Copyright (C) 2002-2018 Jason Evans . All rights reserved. Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-present Facebook, Inc. All rights reserved. +Copyright (C) 2009-2018 Facebook, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/deps/jemalloc/ChangeLog b/deps/jemalloc/ChangeLog index e55813b..29a00fb 100644 --- a/deps/jemalloc/ChangeLog +++ b/deps/jemalloc/ChangeLog @@ -4,143 +4,7 @@ brevity. Much more detail can be found in the git revision history: https://github.com/jemalloc/jemalloc -* 5.2.1 (August 5, 2019) - - This release is primarily about Windows. A critical virtual memory leak is - resolved on all Windows platforms. The regression was present in all releases - since 5.0.0. - - Bug fixes: - - Fix a severe virtual memory leak on Windows. This regression was first - released in 5.0.0. (@Ignition, @j0t, @frederik-h, @davidtgoldblatt, - @interwq) - - Fix size 0 handling in posix_memalign(). This regression was first released - in 5.2.0. (@interwq) - - Fix the prof_log unit test which may observe unexpected backtraces from - compiler optimizations. The test was first added in 5.2.0. (@marxin, - @gnzlbg, @interwq) - - Fix the declaration of the extent_avail tree. This regression was first - released in 5.1.0. (@zoulasc) - - Fix an incorrect reference in jeprof. This functionality was first released - in 3.0.0. (@prehistoric-penguin) - - Fix an assertion on the deallocation fast-path. This regression was first - released in 5.2.0. (@yinan1048576) - - Fix the TLS_MODEL attribute in headers. This regression was first released - in 5.0.0. (@zoulasc, @interwq) - - Optimizations and refactors: - - Implement opt.retain on Windows and enable by default on 64-bit. (@interwq, - @davidtgoldblatt) - - Optimize away a branch on the operator delete[] path. (@mgrice) - - Add format annotation to the format generator function. (@zoulasc) - - Refactor and improve the size class header generation. (@yinan1048576) - - Remove best fit. (@djwatson) - - Avoid blocking on background thread locks for stats. (@oranagra, @interwq) - -* 5.2.0 (April 2, 2019) - - This release includes a few notable improvements, which are summarized below: - 1) improved fast-path performance from the optimizations by @djwatson; 2) - reduced virtual memory fragmentation and metadata usage; and 3) bug fixes on - setting the number of background threads. In addition, peak / spike memory - usage is improved with certain allocation patterns. As usual, the release and - prior dev versions have gone through large-scale production testing. - - New features: - - Implement oversize_threshold, which uses a dedicated arena for allocations - crossing the specified threshold to reduce fragmentation. (@interwq) - - Add extents usage information to stats. (@tyleretzel) - - Log time information for sampled allocations. (@tyleretzel) - - Support 0 size in sdallocx. (@djwatson) - - Output rate for certain counters in malloc_stats. (@zinoale) - - Add configure option --enable-readlinkat, which allows the use of readlinkat - over readlink. (@davidtgoldblatt) - - Add configure options --{enable,disable}-{static,shared} to allow not - building unwanted libraries. (@Ericson2314) - - Add configure option --disable-libdl to enable fully static builds. - (@interwq) - - Add mallctl interfaces: - + opt.oversize_threshold (@interwq) - + stats.arenas..extent_avail (@tyleretzel) - + stats.arenas..extents..n{dirty,muzzy,retained} (@tyleretzel) - + stats.arenas..extents..{dirty,muzzy,retained}_bytes - (@tyleretzel) - - Portability improvements: - - Update MSVC builds. (@maksqwe, @rustyx) - - Workaround a compiler optimizer bug on s390x. (@rkmisra) - - Make use of pthread_set_name_np(3) on FreeBSD. (@trasz) - - Implement malloc_getcpu() to enable percpu_arena for windows. (@santagada) - - Link against -pthread instead of -lpthread. (@paravoid) - - Make background_thread not dependent on libdl. (@interwq) - - Add stringify to fix a linker directive issue on MSVC. (@daverigby) - - Detect and fall back when 8-bit atomics are unavailable. (@interwq) - - Fall back to the default pthread_create if dlsym(3) fails. (@interwq) - - Optimizations and refactors: - - Refactor the TSD module. (@davidtgoldblatt) - - Avoid taking extents_muzzy mutex when muzzy is disabled. (@interwq) - - Avoid taking large_mtx for auto arenas on the tcache flush path. (@interwq) - - Optimize ixalloc by avoiding a size lookup. (@interwq) - - Implement opt.oversize_threshold which uses a dedicated arena for requests - crossing the threshold, also eagerly purges the oversize extents. Default - the threshold to 8 MiB. (@interwq) - - Clean compilation with -Wextra. (@gnzlbg, @jasone) - - Refactor the size class module. (@davidtgoldblatt) - - Refactor the stats emitter. (@tyleretzel) - - Optimize pow2_ceil. (@rkmisra) - - Avoid runtime detection of lazy purging on FreeBSD. (@trasz) - - Optimize mmap(2) alignment handling on FreeBSD. (@trasz) - - Improve error handling for THP state initialization. (@jsteemann) - - Rework the malloc() fast path. (@djwatson) - - Rework the free() fast path. (@djwatson) - - Refactor and optimize the tcache fill / flush paths. (@djwatson) - - Optimize sync / lwsync on PowerPC. (@chmeeedalf) - - Bypass extent_dalloc() when retain is enabled. (@interwq) - - Optimize the locking on large deallocation. (@interwq) - - Reduce the number of pages committed from sanity checking in debug build. - (@trasz, @interwq) - - Deprecate OSSpinLock. (@interwq) - - Lower the default number of background threads to 4 (when the feature - is enabled). (@interwq) - - Optimize the trylock spin wait. (@djwatson) - - Use arena index for arena-matching checks. (@interwq) - - Avoid forced decay on thread termination when using background threads. - (@interwq) - - Disable muzzy decay by default. (@djwatson, @interwq) - - Only initialize libgcc unwinder when profiling is enabled. (@paravoid, - @interwq) - - Bug fixes (all only relevant to jemalloc 5.x): - - Fix background thread index issues with max_background_threads. (@djwatson, - @interwq) - - Fix stats output for opt.lg_extent_max_active_fit. (@interwq) - - Fix opt.prof_prefix initialization. (@davidtgoldblatt) - - Properly trigger decay on tcache destroy. (@interwq, @amosbird) - - Fix tcache.flush. (@interwq) - - Detect whether explicit extent zero out is necessary with huge pages or - custom extent hooks, which may change the purge semantics. (@interwq) - - Fix a side effect caused by extent_max_active_fit combined with decay-based - purging, where freed extents can accumulate and not be reused for an - extended period of time. (@interwq, @mpghf) - - Fix a missing unlock on extent register error handling. (@zoulasc) - - Testing: - - Simplify the Travis script output. (@gnzlbg) - - Update the test scripts for FreeBSD. (@devnexen) - - Add unit tests for the producer-consumer pattern. (@interwq) - - Add Cirrus-CI config for FreeBSD builds. (@jasone) - - Add size-matching sanity checks on tcache flush. (@davidtgoldblatt, - @interwq) - - Incompatible changes: - - Remove --with-lg-page-sizes. (@davidtgoldblatt) - - Documentation: - - Attempt to build docs by default, however skip doc building when xsltproc - is missing. (@interwq, @cmuellner) - -* 5.1.0 (May 4, 2018) +* 5.1.0 (May 4th, 2018) This release is primarily about fine-tuning, ranging from several new features to numerous notable performance and portability enhancements. The release and diff --git a/deps/jemalloc/INSTALL.md b/deps/jemalloc/INSTALL.md index b8f729b..ef328c6 100644 --- a/deps/jemalloc/INSTALL.md +++ b/deps/jemalloc/INSTALL.md @@ -221,6 +221,13 @@ any of the following arguments (not a definitive list) to 'configure': system page size may change between configuration and execution, e.g. when cross compiling. +* `--with-lg-page-sizes=` + + Specify the comma-separated base 2 logs of the page sizes to support. This + option may be useful when cross compiling in combination with + `--with-lg-page`, but its primary use case is for integration with FreeBSD's + libc, wherein jemalloc is embedded. + * `--with-lg-hugepage=` Specify the base 2 log of the system huge page size. This option is useful @@ -269,11 +276,6 @@ any of the following arguments (not a definitive list) to 'configure': in the same process, which will almost certainly result in confusing runtime crashes if pointers leak from one implementation to the other. -* `--disable-libdl` - - Disable the usage of libdl, namely dlsym(3) which is required by the lazy - lock option. This can allow building static binaries. - The following environment variables (not a definitive list) impact configure's behavior: diff --git a/deps/jemalloc/Makefile.in b/deps/jemalloc/Makefile.in index 7128b00..9b9347f 100644 --- a/deps/jemalloc/Makefile.in +++ b/deps/jemalloc/Makefile.in @@ -47,7 +47,6 @@ REV := @rev@ install_suffix := @install_suffix@ ABI := @abi@ XSLTPROC := @XSLTPROC@ -XSLROOT := @XSLROOT@ AUTOCONF := @AUTOCONF@ _RPATH = @RPATH@ RPATH = $(if $(1),$(call _RPATH,$(1))) @@ -56,12 +55,8 @@ cfghdrs_out := @cfghdrs_out@ cfgoutputs_in := $(addprefix $(srcroot),@cfgoutputs_in@) cfgoutputs_out := @cfgoutputs_out@ enable_autogen := @enable_autogen@ -enable_doc := @enable_doc@ -enable_shared := @enable_shared@ -enable_static := @enable_static@ enable_prof := @enable_prof@ enable_zone_allocator := @enable_zone_allocator@ -enable_experimental_smallocx := @enable_experimental_smallocx@ MALLOC_CONF := @JEMALLOC_CPREFIX@MALLOC_CONF link_whole_archive := @link_whole_archive@ DSO_LDFLAGS = @DSO_LDFLAGS@ @@ -107,7 +102,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/extent_dss.c \ $(srcroot)src/extent_mmap.c \ $(srcroot)src/hash.c \ - $(srcroot)src/hook.c \ + $(srcroot)src/hooks.c \ $(srcroot)src/large.c \ $(srcroot)src/log.c \ $(srcroot)src/malloc_io.c \ @@ -118,12 +113,9 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/prng.c \ $(srcroot)src/prof.c \ $(srcroot)src/rtree.c \ - $(srcroot)src/safety_check.c \ $(srcroot)src/stats.c \ - $(srcroot)src/sc.c \ $(srcroot)src/sz.c \ $(srcroot)src/tcache.c \ - $(srcroot)src/test_hooks.c \ $(srcroot)src/ticker.c \ $(srcroot)src/tsd.c \ $(srcroot)src/witness.c @@ -173,18 +165,14 @@ TESTS_UNIT := \ $(srcroot)test/unit/background_thread_enable.c \ $(srcroot)test/unit/base.c \ $(srcroot)test/unit/bitmap.c \ - $(srcroot)test/unit/bit_util.c \ - $(srcroot)test/unit/binshard.c \ $(srcroot)test/unit/ckh.c \ $(srcroot)test/unit/decay.c \ $(srcroot)test/unit/div.c \ $(srcroot)test/unit/emitter.c \ $(srcroot)test/unit/extent_quantize.c \ - $(srcroot)test/unit/extent_util.c \ $(srcroot)test/unit/fork.c \ $(srcroot)test/unit/hash.c \ - $(srcroot)test/unit/hook.c \ - $(srcroot)test/unit/huge.c \ + $(srcroot)test/unit/hooks.c \ $(srcroot)test/unit/junk.c \ $(srcroot)test/unit/junk_alloc.c \ $(srcroot)test/unit/junk_free.c \ @@ -202,7 +190,6 @@ TESTS_UNIT := \ $(srcroot)test/unit/prof_active.c \ $(srcroot)test/unit/prof_gdump.c \ $(srcroot)test/unit/prof_idump.c \ - $(srcroot)test/unit/prof_log.c \ $(srcroot)test/unit/prof_reset.c \ $(srcroot)test/unit/prof_tctx.c \ $(srcroot)test/unit/prof_thread_name.c \ @@ -211,17 +198,13 @@ TESTS_UNIT := \ $(srcroot)test/unit/rb.c \ $(srcroot)test/unit/retained.c \ $(srcroot)test/unit/rtree.c \ - $(srcroot)test/unit/safety_check.c \ - $(srcroot)test/unit/seq.c \ $(srcroot)test/unit/SFMT.c \ - $(srcroot)test/unit/sc.c \ $(srcroot)test/unit/size_classes.c \ $(srcroot)test/unit/slab.c \ $(srcroot)test/unit/smoothstep.c \ $(srcroot)test/unit/spin.c \ $(srcroot)test/unit/stats.c \ $(srcroot)test/unit/stats_print.c \ - $(srcroot)test/unit/test_hooks.c \ $(srcroot)test/unit/ticker.c \ $(srcroot)test/unit/nstime.c \ $(srcroot)test/unit/tsd.c \ @@ -234,21 +217,15 @@ endif TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ $(srcroot)test/integration/allocated.c \ $(srcroot)test/integration/extent.c \ - $(srcroot)test/integration/malloc.c \ $(srcroot)test/integration/mallocx.c \ $(srcroot)test/integration/MALLOCX_ARENA.c \ $(srcroot)test/integration/overflow.c \ $(srcroot)test/integration/posix_memalign.c \ $(srcroot)test/integration/rallocx.c \ $(srcroot)test/integration/sdallocx.c \ - $(srcroot)test/integration/slab_sizes.c \ $(srcroot)test/integration/thread_arena.c \ $(srcroot)test/integration/thread_tcache_enabled.c \ $(srcroot)test/integration/xallocx.c -ifeq (@enable_experimental_smallocx@, 1) -TESTS_INTEGRATION += \ - $(srcroot)test/integration/smallocx.c -endif ifeq (@enable_cxx@, 1) CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp @@ -256,9 +233,7 @@ else CPP_SRCS := TESTS_INTEGRATION_CPP := endif -TESTS_STRESS := $(srcroot)test/stress/microbench.c \ - $(srcroot)test/stress/hookbench.c - +TESTS_STRESS := $(srcroot)test/stress/microbench.c TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_INTEGRATION_CPP) $(TESTS_STRESS) @@ -299,24 +274,10 @@ all: build_lib dist: build_doc $(objroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl -ifneq ($(XSLROOT),) $(XSLTPROC) -o $@ $(objroot)doc/html.xsl $< -else -ifeq ($(wildcard $(DOCS_HTML)),) - @echo "

Missing xsltproc. Doc not built.

" > $@ -endif - @echo "Missing xsltproc. "$@" not (re)built." -endif $(objroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl -ifneq ($(XSLROOT),) $(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $< -else -ifeq ($(wildcard $(DOCS_MAN3)),) - @echo "Missing xsltproc. Doc not built." > $@ -endif - @echo "Missing xsltproc. "$@" not (re)built." -endif build_doc_html: $(DOCS_HTML) build_doc_man: $(DOCS_MAN3) @@ -439,7 +400,7 @@ $(objroot)test/unit/%$(EXE): $(objroot)test/unit/%.$(O) $(C_JET_OBJS) $(C_TESTLI $(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) @mkdir -p $(@D) - $(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -pthread -lstdc++,$(LIBS))) $(LM) $(EXTRA_LDFLAGS) + $(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -lpthread -lstdc++,$(LIBS))) $(LM) $(EXTRA_LDFLAGS) $(objroot)test/integration/cpp/%$(EXE): $(objroot)test/integration/cpp/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) @mkdir -p $(@D) @@ -451,12 +412,7 @@ $(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TE build_lib_shared: $(DSOS) build_lib_static: $(STATIC_LIBS) -ifeq ($(enable_shared), 1) -build_lib: build_lib_shared -endif -ifeq ($(enable_static), 1) -build_lib: build_lib_static -endif +build_lib: build_lib_shared build_lib_static install_bin: $(INSTALL) -d $(BINDIR) @@ -493,13 +449,7 @@ install_lib_pc: $(PC) $(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \ done -ifeq ($(enable_shared), 1) -install_lib: install_lib_shared -endif -ifeq ($(enable_static), 1) -install_lib: install_lib_static -endif -install_lib: install_lib_pc +install_lib: install_lib_shared install_lib_static install_lib_pc install_doc_html: $(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix) @@ -515,13 +465,9 @@ install_doc_man: $(INSTALL) -m 644 $$d $(MANDIR)/man3; \ done -install_doc: build_doc install_doc_html install_doc_man +install_doc: install_doc_html install_doc_man -install: install_bin install_include install_lib - -ifeq ($(enable_doc), 1) -install: install_doc -endif +install: install_bin install_include install_lib install_doc tests_unit: $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%$(EXE)) tests_integration: $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%$(EXE)) $(TESTS_INTEGRATION_CPP:$(srcroot)%.cpp=$(objroot)%$(EXE)) diff --git a/deps/jemalloc/bin/jeprof.in b/deps/jemalloc/bin/jeprof.in index 3ed408c..588c6b4 100644 --- a/deps/jemalloc/bin/jeprof.in +++ b/deps/jemalloc/bin/jeprof.in @@ -2909,7 +2909,6 @@ sub RemoveUninterestingFrames { '@JEMALLOC_PREFIX@xallocx', '@JEMALLOC_PREFIX@dallocx', '@JEMALLOC_PREFIX@sdallocx', - '@JEMALLOC_PREFIX@sdallocx_noflags', 'tc_calloc', 'tc_cfree', 'tc_malloc', @@ -5367,7 +5366,7 @@ sub GetProcedureBoundaries { my $demangle_flag = ""; my $cppfilt_flag = ""; my $to_devnull = ">$dev_null 2>&1"; - if (system(ShellEscape($nm, "--demangle", $image) . $to_devnull) == 0) { + if (system(ShellEscape($nm, "--demangle", "image") . $to_devnull) == 0) { # In this mode, we do "nm --demangle " $demangle_flag = "--demangle"; $cppfilt_flag = ""; diff --git a/deps/jemalloc/configure.ac b/deps/jemalloc/configure.ac index 261d81c..a6a08db 100644 --- a/deps/jemalloc/configure.ac +++ b/deps/jemalloc/configure.ac @@ -175,9 +175,6 @@ fi ], XSLROOT="${DEFAULT_XSLROOT}" ) -if test "x$XSLTPROC" = "xfalse" ; then - XSLROOT="" -fi AC_SUBST([XSLROOT]) dnl If CFLAGS isn't defined, set CFLAGS to something reasonable. Otherwise, @@ -245,7 +242,6 @@ if test "x$GCC" = "xyes" ; then fi fi JE_CFLAGS_ADD([-Wall]) - JE_CFLAGS_ADD([-Wextra]) JE_CFLAGS_ADD([-Wshorten-64-to-32]) JE_CFLAGS_ADD([-Wsign-compare]) JE_CFLAGS_ADD([-Wundef]) @@ -293,7 +289,6 @@ if test "x$enable_cxx" = "x1" ; then AX_CXX_COMPILE_STDCXX([14], [noext], [optional]) if test "x${HAVE_CXX14}" = "x1" ; then JE_CXXFLAGS_ADD([-Wall]) - JE_CXXFLAGS_ADD([-Wextra]) JE_CXXFLAGS_ADD([-g3]) SAVED_LIBS="${LIBS}" @@ -541,66 +536,6 @@ AC_PROG_NM AC_PROG_AWK -dnl ============================================================================ -dnl jemalloc version. -dnl - -AC_ARG_WITH([version], - [AS_HELP_STRING([--with-version=..--g], - [Version string])], - [ - echo "${with_version}" | grep ['^[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+-g[0-9a-f]\+$'] 2>&1 1>/dev/null - if test $? -eq 0 ; then - echo "$with_version" > "${objroot}VERSION" - else - echo "${with_version}" | grep ['^VERSION$'] 2>&1 1>/dev/null - if test $? -ne 0 ; then - AC_MSG_ERROR([${with_version} does not match ..--g or VERSION]) - fi - fi - ], [ - dnl Set VERSION if source directory is inside a git repository. - if test "x`test ! \"${srcroot}\" && cd \"${srcroot}\"; git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then - dnl Pattern globs aren't powerful enough to match both single- and - dnl double-digit version numbers, so iterate over patterns to support up - dnl to version 99.99.99 without any accidental matches. - for pattern in ['[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \ - '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \ - '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \ - '[0-9][0-9].[0-9][0-9].[0-9]' \ - '[0-9][0-9].[0-9][0-9].[0-9][0-9]']; do - (test ! "${srcroot}" && cd "${srcroot}"; git describe --long --abbrev=40 --match="${pattern}") > "${objroot}VERSION.tmp" 2>/dev/null - if test $? -eq 0 ; then - mv "${objroot}VERSION.tmp" "${objroot}VERSION" - break - fi - done - fi - rm -f "${objroot}VERSION.tmp" - ]) - -if test ! -e "${objroot}VERSION" ; then - if test ! -e "${srcroot}VERSION" ; then - AC_MSG_RESULT( - [Missing VERSION file, and unable to generate it; creating bogus VERSION]) - echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${objroot}VERSION" - else - cp ${srcroot}VERSION ${objroot}VERSION - fi -fi -jemalloc_version=`cat "${objroot}VERSION"` -jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'` -jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'` -jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'` -jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]4}'` -jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]5}'` -AC_SUBST([jemalloc_version]) -AC_SUBST([jemalloc_version_major]) -AC_SUBST([jemalloc_version_minor]) -AC_SUBST([jemalloc_version_bugfix]) -AC_SUBST([jemalloc_version_nrev]) -AC_SUBST([jemalloc_version_gid]) - dnl Platform-specific settings. abi and RPATH can probably be determined dnl programmatically, but doing so is error-prone, which makes it generally dnl not worth the trouble. @@ -738,9 +673,6 @@ case "${host}" in libprefix="" SOREV="${so}" PIC_CFLAGS="" - if test "${LG_SIZEOF_PTR}" = "3"; then - default_retain="1" - fi ;; *) AC_MSG_RESULT([Unsupported operating system: ${host}]) @@ -854,18 +786,6 @@ if test "x${je_cv_format_printf}" = "xyes" ; then AC_DEFINE([JEMALLOC_HAVE_ATTR_FORMAT_PRINTF], [ ]) fi -dnl Check for format_arg(...) attribute support. -JE_CFLAGS_SAVE() -JE_CFLAGS_ADD([-Werror]) -JE_CFLAGS_ADD([-herror_on_warning]) -JE_COMPILABLE([format(printf, ...) attribute], [#include ], - [const char * __attribute__((__format_arg__(1))) foo(const char *format);], - [je_cv_format_arg]) -JE_CFLAGS_RESTORE() -if test "x${je_cv_format_arg}" = "xyes" ; then - AC_DEFINE([JEMALLOC_HAVE_ATTR_FORMAT_ARG], [ ]) -fi - dnl Support optional additions to rpath. AC_ARG_WITH([rpath], [AS_HELP_STRING([--with-rpath=], [Colon-separated rpath (ELF systems only)])], @@ -896,49 +816,6 @@ AC_PROG_RANLIB AC_PATH_PROG([LD], [ld], [false], [$PATH]) AC_PATH_PROG([AUTOCONF], [autoconf], [false], [$PATH]) -dnl Enable documentation -AC_ARG_ENABLE([doc], - [AS_HELP_STRING([--enable-documentation], [Build documentation])], -if test "x$enable_doc" = "xno" ; then - enable_doc="0" -else - enable_doc="1" -fi -, -enable_doc="1" -) -AC_SUBST([enable_doc]) - -dnl Enable shared libs -AC_ARG_ENABLE([shared], - [AS_HELP_STRING([--enable-shared], [Build shared libaries])], -if test "x$enable_shared" = "xno" ; then - enable_shared="0" -else - enable_shared="1" -fi -, -enable_shared="1" -) -AC_SUBST([enable_shared]) - -dnl Enable static libs -AC_ARG_ENABLE([static], - [AS_HELP_STRING([--enable-static], [Build static libaries])], -if test "x$enable_static" = "xno" ; then - enable_static="0" -else - enable_static="1" -fi -, -enable_static="1" -) -AC_SUBST([enable_static]) - -if test "$enable_shared$enable_static" = "00" ; then - AC_MSG_ERROR([Please enable one of shared or static builds]) -fi - dnl Perform no name mangling by default. AC_ARG_WITH([mangling], [AS_HELP_STRING([--with-mangling=], [Mangle symbols in ])], @@ -971,7 +848,7 @@ AC_ARG_WITH([export], fi] ) -public_syms="aligned_alloc calloc dallocx free mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_message malloc_stats_print malloc_usable_size mallocx smallocx_${jemalloc_version_gid} nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" +public_syms="aligned_alloc calloc dallocx free mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_message malloc_stats_print malloc_usable_size mallocx nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" dnl Check for additional platform-specific public API functions. AC_CHECK_FUNC([memalign], [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN], [ ]) @@ -1089,6 +966,7 @@ cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_symbols.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_namespace.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_namespace.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_unnamespace.sh" +cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/size_classes.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_rename.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_mangle.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc.sh" @@ -1101,6 +979,7 @@ cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/private_symbols_jet.awk" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_symbols.txt" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_namespace.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_unnamespace.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/size_classes.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_protos_jet.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_rename.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_mangle.h" @@ -1112,10 +991,6 @@ cfghdrs_tup="include/jemalloc/jemalloc_defs.h:include/jemalloc/jemalloc_defs.h.i cfghdrs_tup="${cfghdrs_tup} include/jemalloc/internal/jemalloc_internal_defs.h:include/jemalloc/internal/jemalloc_internal_defs.h.in" cfghdrs_tup="${cfghdrs_tup} test/include/test/jemalloc_test_defs.h:test/include/test/jemalloc_test_defs.h.in" -dnl ============================================================================ -dnl jemalloc build options. -dnl - dnl Do not compile with debugging by default. AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug], @@ -1168,22 +1043,6 @@ if test "x$enable_stats" = "x1" ; then fi AC_SUBST([enable_stats]) -dnl Do not enable smallocx by default. -AC_ARG_ENABLE([experimental_smallocx], - [AS_HELP_STRING([--enable-experimental-smallocx], [Enable experimental smallocx API])], -[if test "x$enable_experimental_smallocx" = "xno" ; then -enable_experimental_smallocx="0" -else -enable_experimental_smallocx="1" -fi -], -[enable_experimental_smallocx="0"] -) -if test "x$enable_experimental_smallocx" = "x1" ; then - AC_DEFINE([JEMALLOC_EXPERIMENTAL_SMALLOCX_API]) -fi -AC_SUBST([enable_experimental_smallocx]) - dnl Do not enable profiling by default. AC_ARG_ENABLE([prof], [AS_HELP_STRING([--enable-prof], [Enable allocation profiling])], @@ -1418,38 +1277,6 @@ if test "x$enable_log" = "x1" ; then fi AC_SUBST([enable_log]) -dnl Do not use readlinkat by default -AC_ARG_ENABLE([readlinkat], - [AS_HELP_STRING([--enable-readlinkat], [Use readlinkat over readlink])], -[if test "x$enable_readlinkat" = "xno" ; then - enable_readlinkat="0" -else - enable_readlinkat="1" -fi -], -[enable_readlinkat="0"] -) -if test "x$enable_readlinkat" = "x1" ; then - AC_DEFINE([JEMALLOC_READLINKAT], [ ]) -fi -AC_SUBST([enable_readlinkat]) - -dnl Avoid extra safety checks by default -AC_ARG_ENABLE([opt-safety-checks], - [AS_HELP_STRING([--enable-opt-safety-checks], - [Perform certain low-overhead checks, even in opt mode])], -[if test "x$enable_opt_safety_checks" = "xno" ; then - enable_opt_safety_checks="0" -else - enable_opt_safety_checks="1" -fi -], -[enable_opt_safety_checks="0"] -) -if test "x$enable_opt_safety_checks" = "x1" ; then - AC_DEFINE([JEMALLOC_OPT_SAFETY_CHECKS], [ ]) -fi -AC_SUBST([enable_opt_safety_checks]) JE_COMPILABLE([a program using __builtin_unreachable], [ void foo (void) { @@ -1506,21 +1333,6 @@ else fi fi -JE_COMPILABLE([a program using __builtin_popcountl], [ -#include -#include -#include -], [ - { - int rv = __builtin_popcountl(0x08); - printf("%d\n", rv); - } -], [je_cv_gcc_builtin_popcountl]) -if test "x${je_cv_gcc_builtin_popcountl}" = "xyes" ; then - AC_DEFINE([JEMALLOC_INTERNAL_POPCOUNT], [__builtin_popcount]) - AC_DEFINE([JEMALLOC_INTERNAL_POPCOUNTL], [__builtin_popcountl]) -fi - AC_ARG_WITH([lg_quantum], [AS_HELP_STRING([--with-lg-quantum=], [Base 2 log of minimum allocation alignment])], @@ -1618,20 +1430,70 @@ if test "x${LG_PAGE}" != "xundefined" -a \ fi AC_DEFINE_UNQUOTED([LG_HUGEPAGE], [${je_cv_lg_hugepage}]) +AC_ARG_WITH([lg_page_sizes], + [AS_HELP_STRING([--with-lg-page-sizes=], + [Base 2 logs of system page sizes to support])], + [LG_PAGE_SIZES="$with_lg_page_sizes"], [LG_PAGE_SIZES="$LG_PAGE"]) + dnl ============================================================================ -dnl Enable libdl by default. -AC_ARG_ENABLE([libdl], - [AS_HELP_STRING([--disable-libdl], - [Do not use libdl])], -[if test "x$enable_libdl" = "xno" ; then - enable_libdl="0" -else - enable_libdl="1" +dnl jemalloc configuration. +dnl + +AC_ARG_WITH([version], + [AS_HELP_STRING([--with-version=..--g], + [Version string])], + [ + echo "${with_version}" | grep ['^[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+-g[0-9a-f]\+$'] 2>&1 1>/dev/null + if test $? -eq 0 ; then + echo "$with_version" > "${objroot}VERSION" + else + echo "${with_version}" | grep ['^VERSION$'] 2>&1 1>/dev/null + if test $? -ne 0 ; then + AC_MSG_ERROR([${with_version} does not match ..--g or VERSION]) + fi + fi + ], [ + dnl Set VERSION if source directory is inside a git repository. + if test "x`test ! \"${srcroot}\" && cd \"${srcroot}\"; git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then + dnl Pattern globs aren't powerful enough to match both single- and + dnl double-digit version numbers, so iterate over patterns to support up + dnl to version 99.99.99 without any accidental matches. + for pattern in ['[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \ + '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \ + '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \ + '[0-9][0-9].[0-9][0-9].[0-9]' \ + '[0-9][0-9].[0-9][0-9].[0-9][0-9]']; do + (test ! "${srcroot}" && cd "${srcroot}"; git describe --long --abbrev=40 --match="${pattern}") > "${objroot}VERSION.tmp" 2>/dev/null + if test $? -eq 0 ; then + mv "${objroot}VERSION.tmp" "${objroot}VERSION" + break + fi + done + fi + rm -f "${objroot}VERSION.tmp" + ]) + +if test ! -e "${objroot}VERSION" ; then + if test ! -e "${srcroot}VERSION" ; then + AC_MSG_RESULT( + [Missing VERSION file, and unable to generate it; creating bogus VERSION]) + echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${objroot}VERSION" + else + cp ${srcroot}VERSION ${objroot}VERSION + fi fi -], -[enable_libdl="1"] -) -AC_SUBST([libdl]) +jemalloc_version=`cat "${objroot}VERSION"` +jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'` +jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'` +jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'` +jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]4}'` +jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]5}'` +AC_SUBST([jemalloc_version]) +AC_SUBST([jemalloc_version_major]) +AC_SUBST([jemalloc_version_minor]) +AC_SUBST([jemalloc_version_bugfix]) +AC_SUBST([jemalloc_version_nrev]) +AC_SUBST([jemalloc_version_gid]) dnl ============================================================================ dnl Configure pthreads. @@ -1641,26 +1503,20 @@ if test "x$abi" != "xpecoff" ; then AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])]) dnl Some systems may embed pthreads functionality in libc; check for libpthread dnl first, but try libc too before failing. - AC_CHECK_LIB([pthread], [pthread_create], [JE_APPEND_VS(LIBS, -pthread)], + AC_CHECK_LIB([pthread], [pthread_create], [JE_APPEND_VS(LIBS, -lpthread)], [AC_SEARCH_LIBS([pthread_create], , , AC_MSG_ERROR([libpthread is missing]))]) wrap_syms="${wrap_syms} pthread_create" have_pthread="1" - -dnl Check if we have dlsym support. - if test "x$enable_libdl" = "x1" ; then - have_dlsym="1" - AC_CHECK_HEADERS([dlfcn.h], - AC_CHECK_FUNC([dlsym], [], - [AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"], [have_dlsym="0"])]), - [have_dlsym="0"]) - if test "x$have_dlsym" = "x1" ; then - AC_DEFINE([JEMALLOC_HAVE_DLSYM], [ ]) - fi - else - have_dlsym="0" + dnl Check if we have dlsym support. + have_dlsym="1" + AC_CHECK_HEADERS([dlfcn.h], + AC_CHECK_FUNC([dlsym], [], + [AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"], [have_dlsym="0"])]), + [have_dlsym="0"]) + if test "x$have_dlsym" = "x1" ; then + AC_DEFINE([JEMALLOC_HAVE_DLSYM], [ ]) fi - JE_COMPILABLE([pthread_atfork(3)], [ #include ], [ @@ -1924,19 +1780,6 @@ JE_COMPILABLE([GCC __atomic atomics], [ ], [je_cv_gcc_atomic_atomics]) if test "x${je_cv_gcc_atomic_atomics}" = "xyes" ; then AC_DEFINE([JEMALLOC_GCC_ATOMIC_ATOMICS]) - - dnl check for 8-bit atomic support - JE_COMPILABLE([GCC 8-bit __atomic atomics], [ - ], [ - unsigned char x = 0; - int val = 1; - int y = __atomic_fetch_add(&x, val, __ATOMIC_RELAXED); - int after_add = (int)x; - return after_add == 1; - ], [je_cv_gcc_u8_atomic_atomics]) - if test "x${je_cv_gcc_u8_atomic_atomics}" = "xyes" ; then - AC_DEFINE([JEMALLOC_GCC_U8_ATOMIC_ATOMICS]) - fi fi dnl ============================================================================ @@ -1951,24 +1794,12 @@ JE_COMPILABLE([GCC __sync atomics], [ ], [je_cv_gcc_sync_atomics]) if test "x${je_cv_gcc_sync_atomics}" = "xyes" ; then AC_DEFINE([JEMALLOC_GCC_SYNC_ATOMICS]) - - dnl check for 8-bit atomic support - JE_COMPILABLE([GCC 8-bit __sync atomics], [ - ], [ - unsigned char x = 0; - int before_add = __sync_fetch_and_add(&x, 1); - int after_add = (int)x; - return (before_add == 0) && (after_add == 1); - ], [je_cv_gcc_u8_sync_atomics]) - if test "x${je_cv_gcc_u8_sync_atomics}" = "xyes" ; then - AC_DEFINE([JEMALLOC_GCC_U8_SYNC_ATOMICS]) - fi fi dnl ============================================================================ dnl Check for atomic(3) operations as provided on Darwin. dnl We need this not for the atomic operations (which are provided above), but -dnl rather for the OS_unfair_lock type it exposes. +dnl rather for the OSSpinLock type it exposes. JE_COMPILABLE([Darwin OSAtomic*()], [ #include @@ -2039,7 +1870,7 @@ if test "x${je_cv_madvise}" = "xyes" ; then if test "x${je_cv_madv_dontdump}" = "xyes" ; then AC_DEFINE([JEMALLOC_MADVISE_DONTDUMP], [ ]) fi - + dnl Check for madvise(..., MADV_[NO]HUGEPAGE). JE_COMPILABLE([madvise(..., MADV_[[NO]]HUGEPAGE)], [ #include @@ -2058,6 +1889,40 @@ case "${host_cpu}" in esac fi +dnl ============================================================================ +dnl Check whether __sync_{add,sub}_and_fetch() are available despite +dnl __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n macros being undefined. + +AC_DEFUN([JE_SYNC_COMPARE_AND_SWAP_CHECK],[ + AC_CACHE_CHECK([whether to force $1-bit __sync_{add,sub}_and_fetch()], + [je_cv_sync_compare_and_swap_$2], + [AC_LINK_IFELSE([AC_LANG_PROGRAM([ + #include + ], + [ + #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2 + { + uint$1_t x$1 = 0; + __sync_add_and_fetch(&x$1, 42); + __sync_sub_and_fetch(&x$1, 1); + } + #else + #error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2 is defined, no need to force + #endif + ])], + [je_cv_sync_compare_and_swap_$2=yes], + [je_cv_sync_compare_and_swap_$2=no])]) + + if test "x${je_cv_sync_compare_and_swap_$2}" = "xyes" ; then + AC_DEFINE([JE_FORCE_SYNC_COMPARE_AND_SWAP_$2], [ ]) + fi +]) + +if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then + JE_SYNC_COMPARE_AND_SWAP_CHECK(32, 4) + JE_SYNC_COMPARE_AND_SWAP_CHECK(64, 8) +fi + dnl ============================================================================ dnl Check for __builtin_clz() and __builtin_clzl(). @@ -2100,6 +1965,21 @@ if test "x${je_cv_os_unfair_lock}" = "xyes" ; then AC_DEFINE([JEMALLOC_OS_UNFAIR_LOCK], [ ]) fi +dnl ============================================================================ +dnl Check for spinlock(3) operations as provided on Darwin. + +JE_COMPILABLE([Darwin OSSpin*()], [ +#include +#include +], [ + OSSpinLock lock = 0; + OSSpinLockLock(&lock); + OSSpinLockUnlock(&lock); +], [je_cv_osspin]) +if test "x${je_cv_osspin}" = "xyes" ; then + AC_DEFINE([JEMALLOC_OSSPIN], [ ]) +fi + dnl ============================================================================ dnl Darwin-related configuration. @@ -2152,7 +2032,9 @@ fi dnl ============================================================================ dnl Enable background threads if possible. -if test "x${have_pthread}" = "x1" -a "x${je_cv_os_unfair_lock}" != "xyes" ; then +if test "x${have_pthread}" = "x1" -a "x${have_dlsym}" = "x1" \ + -a "x${je_cv_os_unfair_lock}" != "xyes" \ + -a "x${je_cv_osspin}" != "xyes" ; then AC_DEFINE([JEMALLOC_BACKGROUND_THREAD]) fi @@ -2293,6 +2175,16 @@ AC_CONFIG_COMMANDS([include/jemalloc/internal/public_unnamespace.h], [ srcdir="${srcdir}" objroot="${objroot}" ]) +AC_CONFIG_COMMANDS([include/jemalloc/internal/size_classes.h], [ + mkdir -p "${objroot}include/jemalloc/internal" + "${SHELL}" "${srcdir}/include/jemalloc/internal/size_classes.sh" "${LG_QUANTA}" 3 "${LG_PAGE_SIZES}" 2 > "${objroot}include/jemalloc/internal/size_classes.h" +], [ + SHELL="${SHELL}" + srcdir="${srcdir}" + objroot="${objroot}" + LG_QUANTA="${LG_QUANTA}" + LG_PAGE_SIZES="${LG_PAGE_SIZES}" +]) AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_protos_jet.h], [ mkdir -p "${objroot}include/jemalloc" cat "${srcdir}/include/jemalloc/jemalloc_protos.h.in" | sed -e 's/@je_@/jet_/g' > "${objroot}include/jemalloc/jemalloc_protos_jet.h" @@ -2385,13 +2277,9 @@ AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE]) AC_MSG_RESULT([ : ${JEMALLOC_PRIVATE_NAMESPACE}]) AC_MSG_RESULT([install_suffix : ${install_suffix}]) AC_MSG_RESULT([malloc_conf : ${config_malloc_conf}]) -AC_MSG_RESULT([documentation : ${enable_doc}]) -AC_MSG_RESULT([shared libs : ${enable_shared}]) -AC_MSG_RESULT([static libs : ${enable_static}]) AC_MSG_RESULT([autogen : ${enable_autogen}]) AC_MSG_RESULT([debug : ${enable_debug}]) AC_MSG_RESULT([stats : ${enable_stats}]) -AC_MSG_RESULT([experimetal_smallocx : ${enable_experimental_smallocx}]) AC_MSG_RESULT([prof : ${enable_prof}]) AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}]) AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}]) diff --git a/deps/jemalloc/doc/jemalloc.xml.in b/deps/jemalloc/doc/jemalloc.xml.in index 7fecda7..1e12fd3 100644 --- a/deps/jemalloc/doc/jemalloc.xml.in +++ b/deps/jemalloc/doc/jemalloc.xml.in @@ -424,7 +424,7 @@ for (i = 0; i < nbins; i++) { called repeatedly. General information that never changes during execution can be omitted by specifying g as a character within the opts string. Note that - malloc_stats_print() uses the + malloc_message() uses the mallctl*() functions internally, so inconsistent statistics can be reported if multiple threads use these functions simultaneously. If is specified during @@ -433,11 +433,10 @@ for (i = 0; i < nbins; i++) { arena statistics, respectively; b and l can be specified to omit per size class statistics for bins and large objects, respectively; x can be specified to omit all mutex - statistics; e can be used to omit extent statistics. - Unrecognized characters are silently ignored. Note that thread caching - may prevent some statistics from being completely up to date, since extra - locking would be required to merge counters that track thread cache - operations. + statistics. Unrecognized characters are silently ignored. Note that + thread caching may prevent some statistics from being completely up to + date, since extra locking would be required to merge counters that track + thread cache operations. The malloc_usable_size() function returns the usable size of the allocation pointed to by @@ -904,23 +903,6 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", - - - opt.confirm_conf - (bool) - r- - - Confirm-runtime-options-when-program-starts - enabled/disabled. If true, the string specified via - , the string pointed to by the - global variable malloc_conf, the name - of the file referenced by the symbolic link named - /etc/malloc.conf, and the value of - the environment variable MALLOC_CONF, will be printed in - order. Then, each option being set will be individually printed. This - option is disabled by default. - - opt.abort_conf @@ -961,19 +943,16 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", munmap 2 or equivalent (see stats.retained for related details). - It also makes jemalloc use - mmap2 - or equivalent in a more greedy way, mapping larger - chunks in one go. This option is disabled by default unless discarding - virtual memory is known to trigger platform-specific performance - problems, namely 1) for [64-bit] Linux, which has a quirk in its virtual - memory allocation algorithm that causes semi-permanent VM map holes - under normal jemalloc operation; and 2) for [64-bit] Windows, which - disallows split / merged regions with - MEM_RELEASE. Although the - same issues may present on 32-bit platforms as well, retaining virtual - memory for 32-bit Linux and Windows is disabled by default due to the - practical possibility of address space exhaustion. + This option is disabled by default unless discarding virtual memory is + known to trigger + platform-specific performance problems, e.g. for [64-bit] Linux, which + has a quirk in its virtual memory allocation algorithm that causes + semi-permanent VM map holes under normal jemalloc operation. Although + munmap + 2 causes issues on 32-bit Linux as + well, retaining virtual memory for 32-bit Linux is disabled by default + due to the practical possibility of address space exhaustion. + @@ -1009,24 +988,6 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", number of CPUs, or one if there is a single CPU. - - - opt.oversize_threshold - (size_t) - r- - - The threshold in bytes of which requests are considered - oversize. Allocation requests with greater sizes are fulfilled from a - dedicated arena (automatically managed, however not within - narenas), in order to reduce fragmentation by not - mixing huge allocations with small ones. In addition, the decay API - guarantees on the extents greater than the specified threshold may be - overridden. Note that requests with arena index specified via - MALLOCX_ARENA, or threads associated with explicit - arenas will not be considered. The default threshold is 8MiB. Values - not within large size classes disables this feature. - - opt.percpu_arena @@ -1048,7 +1009,7 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", opt.background_thread - (bool) + (const bool) r- Internal background worker threads enabled/disabled. @@ -1063,7 +1024,7 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", opt.max_background_threads - (size_t) + (const size_t) r- Maximum number of background threads that will be created @@ -1094,11 +1055,7 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", linkend="arena.i.dirty_decay_ms">arena.<i>.dirty_decay_ms for related dynamic control options. See opt.muzzy_decay_ms - for a description of muzzy pages.for a description of muzzy pages. Note - that when the oversize_threshold - feature is enabled, the arenas reserved for oversize requests may have - its own default decay settings. + for a description of muzzy pages. @@ -1806,11 +1763,10 @@ malloc_conf = "xmalloc:true";]]> to control allocation for arenas explicitly created via arenas.create such that all extents originate from an application-supplied extent allocator - (by specifying the custom extent hook functions during arena creation). - However, the API guarantees for the automatically created arenas may be - relaxed -- hooks set there may be called in a "best effort" fashion; in - addition there may be extents created prior to the application having an - opportunity to take over extent allocation. + (by specifying the custom extent hook functions during arena creation), + but the automatically created arenas will have already created extents + prior to the application having an opportunity to take over extent + allocation. - - - stats.arenas.<i>.extent_avail - (size_t) - r- - [] - - Number of allocated (but unused) extent structs in this - arena. - - stats.arenas.<i>.base @@ -2815,28 +2760,6 @@ struct extent_hooks_s { all bin size classes. - - - stats.arenas.<i>.small.nfills - (uint64_t) - r- - [] - - Cumulative number of tcache fills by all small size - classes. - - - - - stats.arenas.<i>.small.nflushes - (uint64_t) - r- - [] - - Cumulative number of tcache flushes by all small size - classes. - - stats.arenas.<i>.large.allocated @@ -2887,28 +2810,6 @@ struct extent_hooks_s { all large size classes. - - - stats.arenas.<i>.large.nfills - (uint64_t) - r- - [] - - Cumulative number of tcache fills by all large size - classes. - - - - - stats.arenas.<i>.large.nflushes - (uint64_t) - r- - [] - - Cumulative number of tcache flushes by all large size - classes. - - stats.arenas.<i>.bins.<j>.nmalloc @@ -3008,17 +2909,6 @@ struct extent_hooks_s { Current number of slabs. - - - - stats.arenas.<i>.bins.<j>.nonfull_slabs - (size_t) - r- - [] - - Current number of nonfull slabs. - - stats.arenas.<i>.bins.<j>.mutex.{counter} @@ -3032,30 +2922,6 @@ struct extent_hooks_s { counters. - - - stats.arenas.<i>.extents.<j>.n{extent_type} - (size_t) - r- - [] - - Number of extents of the given type in this arena in - the bucket corresponding to page size index <j>. The extent type - is one of dirty, muzzy, or retained. - - - - - stats.arenas.<i>.extents.<j>.{extent_type}_bytes - (size_t) - r- - [] - - Sum of the bytes managed by extents of the given type - in this arena in the bucket corresponding to page size index <j>. - The extent type is one of dirty, muzzy, or retained. - - stats.arenas.<i>.lextents.<j>.nmalloc diff --git a/deps/jemalloc/include/jemalloc/internal/arena_externs.h b/deps/jemalloc/include/jemalloc/internal/arena_externs.h index a4523ae..4b3732b 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/arena_externs.h @@ -3,8 +3,8 @@ #include "jemalloc/internal/bin.h" #include "jemalloc/internal/extent_dss.h" -#include "jemalloc/internal/hook.h" #include "jemalloc/internal/pages.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" extern ssize_t opt_dirty_decay_ms; @@ -16,17 +16,13 @@ extern const char *percpu_arena_mode_names[]; extern const uint64_t h_steps[SMOOTHSTEP_NSTEPS]; extern malloc_mutex_t arenas_lock; -extern size_t opt_oversize_threshold; -extern size_t oversize_threshold; - void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy); void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, - bin_stats_t *bstats, arena_stats_large_t *lstats, - arena_stats_extents_t *estats); + bin_stats_t *bstats, arena_stats_large_t *lstats); void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent); #ifdef JEMALLOC_JET @@ -60,17 +56,16 @@ void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero); void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero, tcache_t *tcache); -void arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize); +void arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize); void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path); -void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - szind_t binind, extent_t *extent, void *ptr); +void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, + extent_t *extent, void *ptr); void arena_dalloc_small(tsdn_t *tsdn, void *ptr); bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero, size_t *newsize); + size_t extra, bool zero); void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t alignment, bool zero, tcache_t *tcache, - hook_ralloc_args_t *hook_args); + size_t size, size_t alignment, bool zero, tcache_t *tcache); dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); ssize_t arena_dirty_decay_ms_default_get(void); @@ -84,12 +79,7 @@ void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal); size_t arena_extent_sn_next(arena_t *arena); arena_t *arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); -bool arena_init_huge(void); -bool arena_is_huge(unsigned arena_ind); -arena_t *arena_choose_huge(tsd_t *tsd); -bin_t *arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, - unsigned *binshard); -void arena_boot(sc_data_t *sc_data); +void arena_boot(void); void arena_prefork0(tsdn_t *tsdn, arena_t *arena); void arena_prefork1(tsdn_t *tsdn, arena_t *arena); void arena_prefork2(tsdn_t *tsdn, arena_t *arena); diff --git a/deps/jemalloc/include/jemalloc/internal/arena_inlines_b.h b/deps/jemalloc/include/jemalloc/internal/arena_inlines_b.h index dd92657..2b7e77e 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena_inlines_b.h +++ b/deps/jemalloc/include/jemalloc/internal/arena_inlines_b.h @@ -4,36 +4,10 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" -JEMALLOC_ALWAYS_INLINE bool -arena_has_default_hooks(arena_t *arena) { - return (extent_hooks_get(arena) == &extent_hooks_default); -} - -JEMALLOC_ALWAYS_INLINE arena_t * -arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) { - if (arena != NULL) { - return arena; - } - - /* - * For huge allocations, use the dedicated huge arena if both are true: - * 1) is using auto arena selection (i.e. arena == NULL), and 2) the - * thread is not assigned to a manual arena. - */ - if (unlikely(size >= oversize_threshold)) { - arena_t *tsd_arena = tsd_arena_get(tsd); - if (tsd_arena == NULL || arena_is_auto(tsd_arena)) { - return arena_choose_huge(tsd); - } - } - - return arena_choose(tsd, NULL); -} - JEMALLOC_ALWAYS_INLINE prof_tctx_t * arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { cassert(config_prof); @@ -54,7 +28,7 @@ arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { } JEMALLOC_ALWAYS_INLINE void -arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, +arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, UNUSED size_t usize, alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); @@ -73,7 +47,7 @@ arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, } static inline void -arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { +arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, UNUSED prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); @@ -83,32 +57,6 @@ arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { large_prof_tctx_reset(tsdn, extent); } -JEMALLOC_ALWAYS_INLINE nstime_t -arena_prof_alloc_time_get(tsdn_t *tsdn, const void *ptr, - alloc_ctx_t *alloc_ctx) { - cassert(config_prof); - assert(ptr != NULL); - - extent_t *extent = iealloc(tsdn, ptr); - /* - * Unlike arena_prof_prof_tctx_{get, set}, we only call this once we're - * sure we have a sampled allocation. - */ - assert(!extent_slab_get(extent)); - return large_prof_alloc_time_get(extent); -} - -JEMALLOC_ALWAYS_INLINE void -arena_prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx, - nstime_t t) { - cassert(config_prof); - assert(ptr != NULL); - - extent_t *extent = iealloc(tsdn, ptr); - assert(!extent_slab_get(extent)); - large_prof_alloc_time_set(extent, t); -} - JEMALLOC_ALWAYS_INLINE void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) { tsd_t *tsd; @@ -135,33 +83,14 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) { arena_decay_ticks(tsdn, arena, 1); } -/* Purge a single extent to retained / unmapped directly. */ -JEMALLOC_ALWAYS_INLINE void -arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t **r_extent_hooks, - extent_t *extent) { - size_t extent_size = extent_size_get(extent); - extent_dalloc_wrapper(tsdn, arena, - r_extent_hooks, extent); - if (config_stats) { - /* Update stats accordingly. */ - arena_stats_lock(tsdn, &arena->stats); - arena_stats_add_u64(tsdn, &arena->stats, - &arena->decay_dirty.stats->nmadvise, 1); - arena_stats_add_u64(tsdn, &arena->stats, - &arena->decay_dirty.stats->purged, extent_size >> LG_PAGE); - arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped, - extent_size); - arena_stats_unlock(tsdn, &arena->stats); - } -} - JEMALLOC_ALWAYS_INLINE void * arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); + assert(size != 0); if (likely(tcache != NULL)) { - if (likely(size <= SC_SMALL_MAXCLASS)) { + if (likely(size <= SMALL_MAXCLASS)) { return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache, size, ind, zero, slow_path); } @@ -190,7 +119,7 @@ arena_salloc(tsdn_t *tsdn, const void *ptr) { szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); - assert(szind != SC_NSIZES); + assert(szind != NSIZES); return sz_index2size(szind); } @@ -223,21 +152,11 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) { /* Only slab members should be looked up via interior pointers. */ assert(extent_addr_get(extent) == ptr || extent_slab_get(extent)); - assert(szind != SC_NSIZES); + assert(szind != NSIZES); return sz_index2size(szind); } -static inline void -arena_dalloc_large_no_tcache(tsdn_t *tsdn, void *ptr, szind_t szind) { - if (config_prof && unlikely(szind < SC_NBINS)) { - arena_dalloc_promoted(tsdn, ptr, NULL, true); - } else { - extent_t *extent = iealloc(tsdn, ptr); - large_dalloc(tsdn, extent); - } -} - static inline void arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { assert(ptr != NULL); @@ -254,28 +173,13 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); assert(szind == extent_szind_get(extent)); - assert(szind < SC_NSIZES); + assert(szind < NSIZES); assert(slab == extent_slab_get(extent)); } if (likely(slab)) { /* Small allocation. */ arena_dalloc_small(tsdn, ptr); - } else { - arena_dalloc_large_no_tcache(tsdn, ptr, szind); - } -} - -JEMALLOC_ALWAYS_INLINE void -arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind, - bool slow_path) { - if (szind < nhbins) { - if (config_prof && unlikely(szind < SC_NBINS)) { - arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); - } else { - tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, szind, - slow_path); - } } else { extent_t *extent = iealloc(tsdn, ptr); large_dalloc(tsdn, extent); @@ -299,7 +203,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, if (alloc_ctx != NULL) { szind = alloc_ctx->szind; slab = alloc_ctx->slab; - assert(szind != SC_NSIZES); + assert(szind != NSIZES); } else { rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, @@ -311,7 +215,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); assert(szind == extent_szind_get(extent)); - assert(szind < SC_NSIZES); + assert(szind < NSIZES); assert(slab == extent_slab_get(extent)); } @@ -320,14 +224,25 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind, slow_path); } else { - arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path); + if (szind < nhbins) { + if (config_prof && unlikely(szind < NBINS)) { + arena_dalloc_promoted(tsdn, ptr, tcache, + slow_path); + } else { + tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, + szind, slow_path); + } + } else { + extent_t *extent = iealloc(tsdn, ptr); + large_dalloc(tsdn, extent); + } } } static inline void arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { assert(ptr != NULL); - assert(size <= SC_LARGE_MAXCLASS); + assert(size <= LARGE_MAXCLASS); szind_t szind; bool slab; @@ -337,7 +252,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { * object, so base szind and slab on the given size. */ szind = sz_size2index(size); - slab = (szind < SC_NBINS); + slab = (szind < NBINS); } if ((config_prof && opt_prof) || config_debug) { @@ -349,7 +264,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { (uintptr_t)ptr, true, &szind, &slab); assert(szind == sz_size2index(size)); - assert((config_prof && opt_prof) || slab == (szind < SC_NBINS)); + assert((config_prof && opt_prof) || slab == (szind < NBINS)); if (config_debug) { extent_t *extent = rtree_extent_read(tsdn, @@ -363,7 +278,8 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { /* Small allocation. */ arena_dalloc_small(tsdn, ptr); } else { - arena_dalloc_large_no_tcache(tsdn, ptr, szind); + extent_t *extent = iealloc(tsdn, ptr); + large_dalloc(tsdn, extent); } } @@ -372,7 +288,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, alloc_ctx_t *alloc_ctx, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); assert(ptr != NULL); - assert(size <= SC_LARGE_MAXCLASS); + assert(size <= LARGE_MAXCLASS); if (unlikely(tcache == NULL)) { arena_sdalloc_no_tcache(tsdn, ptr, size); @@ -381,7 +297,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, szind_t szind; bool slab; - alloc_ctx_t local_ctx; + UNUSED alloc_ctx_t local_ctx; if (config_prof && opt_prof) { if (alloc_ctx == NULL) { /* Uncommon case and should be a static check. */ @@ -402,7 +318,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, * object, so base szind and slab on the given size. */ szind = sz_size2index(size); - slab = (szind < SC_NBINS); + slab = (szind < NBINS); } if (config_debug) { @@ -420,7 +336,18 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind, slow_path); } else { - arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path); + if (szind < nhbins) { + if (config_prof && unlikely(szind < NBINS)) { + arena_dalloc_promoted(tsdn, ptr, tcache, + slow_path); + } else { + tcache_dalloc_large(tsdn_tsd(tsdn), + tcache, ptr, szind, slow_path); + } + } else { + extent_t *extent = iealloc(tsdn, ptr); + large_dalloc(tsdn, extent); + } } } diff --git a/deps/jemalloc/include/jemalloc/internal/arena_stats.h b/deps/jemalloc/include/jemalloc/internal/arena_stats.h index 23949ed..5f3dca8 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena_stats.h +++ b/deps/jemalloc/include/jemalloc/internal/arena_stats.h @@ -4,9 +4,7 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex_prof.h" -#include "jemalloc/internal/sc.h" - -JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS +#include "jemalloc/internal/size_classes.h" /* * In those architectures that support 64-bit atomics, we use atomic updates for @@ -35,13 +33,6 @@ struct arena_stats_large_s { * periodically merges into this counter. */ arena_stats_u64_t nrequests; /* Partially derived. */ - /* - * Number of tcache fills / flushes for large (similarly, periodically - * merged). Note that there is no large tcache batch-fill currently - * (i.e. only fill 1 at a time); however flush may be batched. - */ - arena_stats_u64_t nfills; /* Partially derived. */ - arena_stats_u64_t nflushes; /* Partially derived. */ /* Current number of allocations of this size class. */ size_t curlextents; /* Derived. */ @@ -57,22 +48,6 @@ struct arena_stats_decay_s { arena_stats_u64_t purged; }; -typedef struct arena_stats_extents_s arena_stats_extents_t; -struct arena_stats_extents_s { - /* - * Stats for a given index in the range [0, SC_NPSIZES] in an extents_t. - * We track both bytes and # of extents: two extents in the same bucket - * may have different sizes if adjacent size classes differ by more than - * a page, so bytes cannot always be derived from # of extents. - */ - atomic_zu_t ndirty; - atomic_zu_t dirty_bytes; - atomic_zu_t nmuzzy; - atomic_zu_t muzzy_bytes; - atomic_zu_t nretained; - atomic_zu_t retained_bytes; -}; - /* * Arena stats. Note that fields marked "derived" are not directly maintained * within the arena code; rather their values are derived during stats merge @@ -94,9 +69,6 @@ struct arena_stats_s { */ atomic_zu_t retained; /* Derived. */ - /* Number of extent_t structs allocated by base, but not being used. */ - atomic_zu_t extent_avail; - arena_stats_decay_t decay_dirty; arena_stats_decay_t decay_muzzy; @@ -108,27 +80,22 @@ struct arena_stats_s { atomic_zu_t allocated_large; /* Derived. */ arena_stats_u64_t nmalloc_large; /* Derived. */ arena_stats_u64_t ndalloc_large; /* Derived. */ - arena_stats_u64_t nfills_large; /* Derived. */ - arena_stats_u64_t nflushes_large; /* Derived. */ arena_stats_u64_t nrequests_large; /* Derived. */ - /* VM space had to be leaked (undocumented). Normally 0. */ - atomic_zu_t abandoned_vm; - /* Number of bytes cached in tcache associated with this arena. */ atomic_zu_t tcache_bytes; /* Derived. */ mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]; /* One element for each large size class. */ - arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; + arena_stats_large_t lstats[NSIZES - NBINS]; /* Arena uptime. */ nstime_t uptime; }; static inline bool -arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) { +arena_stats_init(UNUSED tsdn_t *tsdn, arena_stats_t *arena_stats) { if (config_debug) { for (size_t i = 0; i < sizeof(arena_stats_t); i++) { assert(((char *)arena_stats)[i] == 0); @@ -180,11 +147,11 @@ arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, #endif } -static inline void +UNUSED static inline void arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, arena_stats_u64_t *p, uint64_t x) { #ifdef JEMALLOC_ATOMIC_U64 - uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED); + UNUSED uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED); assert(r - x <= r); #else malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); @@ -209,8 +176,7 @@ arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) { } static inline size_t -arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, - atomic_zu_t *p) { +arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p) { #ifdef JEMALLOC_ATOMIC_U64 return atomic_load_zu(p, ATOMIC_RELAXED); #else @@ -220,8 +186,8 @@ arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, } static inline void -arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, - atomic_zu_t *p, size_t x) { +arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, + size_t x) { #ifdef JEMALLOC_ATOMIC_U64 atomic_fetch_add_zu(p, x, ATOMIC_RELAXED); #else @@ -232,10 +198,10 @@ arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, } static inline void -arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, - atomic_zu_t *p, size_t x) { +arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, + size_t x) { #ifdef JEMALLOC_ATOMIC_U64 - size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED); + UNUSED size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED); assert(r - x <= r); #else malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); @@ -252,12 +218,11 @@ arena_stats_accum_zu(atomic_zu_t *dst, size_t src) { } static inline void -arena_stats_large_flush_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats, +arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats, szind_t szind, uint64_t nrequests) { arena_stats_lock(tsdn, arena_stats); - arena_stats_large_t *lstats = &arena_stats->lstats[szind - SC_NBINS]; - arena_stats_add_u64(tsdn, arena_stats, &lstats->nrequests, nrequests); - arena_stats_add_u64(tsdn, arena_stats, &lstats->nflushes, 1); + arena_stats_add_u64(tsdn, arena_stats, &arena_stats->lstats[szind - + NBINS].nrequests, nrequests); arena_stats_unlock(tsdn, arena_stats); } @@ -268,4 +233,5 @@ arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) { arena_stats_unlock(tsdn, arena_stats); } + #endif /* JEMALLOC_INTERNAL_ARENA_STATS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/arena_structs_b.h b/deps/jemalloc/include/jemalloc/internal/arena_structs_b.h index eeab57f..38bc959 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena_structs_b.h +++ b/deps/jemalloc/include/jemalloc/internal/arena_structs_b.h @@ -10,7 +10,7 @@ #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/nstime.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/smoothstep.h" #include "jemalloc/internal/ticker.h" @@ -90,9 +90,6 @@ struct arena_s { */ atomic_u_t nthreads[2]; - /* Next bin shard for binding new threads. Synchronization: atomic. */ - atomic_u_t binshard_next; - /* * When percpu_arena is enabled, to amortize the cost of reading / * updating the current CPU id, track the most recent thread accessing @@ -116,6 +113,7 @@ struct arena_s { /* Synchronization: internal. */ prof_accum_t prof_accum; + uint64_t prof_accumbytes; /* * PRNG state for cache index randomization of large allocation base @@ -198,7 +196,6 @@ struct arena_s { * Synchronization: extent_avail_mtx. */ extent_tree_t extent_avail; - atomic_zu_t extent_avail_cnt; malloc_mutex_t extent_avail_mtx; /* @@ -206,7 +203,7 @@ struct arena_s { * * Synchronization: internal. */ - bins_t bins[SC_NBINS]; + bin_t bins[NBINS]; /* * Base allocator, from which arena metadata are allocated. diff --git a/deps/jemalloc/include/jemalloc/internal/arena_types.h b/deps/jemalloc/include/jemalloc/internal/arena_types.h index 624937e..70001b5 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena_types.h +++ b/deps/jemalloc/include/jemalloc/internal/arena_types.h @@ -1,15 +1,13 @@ #ifndef JEMALLOC_INTERNAL_ARENA_TYPES_H #define JEMALLOC_INTERNAL_ARENA_TYPES_H -#include "jemalloc/internal/sc.h" - /* Maximum number of regions in one slab. */ -#define LG_SLAB_MAXREGS (LG_PAGE - SC_LG_TINY_MIN) +#define LG_SLAB_MAXREGS (LG_PAGE - LG_TINY_MIN) #define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS) /* Default decay times in milliseconds. */ #define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000) -#define MUZZY_DECAY_MS_DEFAULT (0) +#define MUZZY_DECAY_MS_DEFAULT ZD(10 * 1000) /* Number of event ticks between time checks. */ #define DECAY_NTICKS_PER_UPDATE 1000 @@ -42,10 +40,4 @@ typedef enum { #define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base) #define PERCPU_ARENA_DEFAULT percpu_arena_disabled -/* - * When allocation_size >= oversize_threshold, use the dedicated huge arena - * (unless have explicitly spicified arena index). 0 disables the feature. - */ -#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20) - #endif /* JEMALLOC_INTERNAL_ARENA_TYPES_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/atomic.h b/deps/jemalloc/include/jemalloc/internal/atomic.h index a76f54c..adadb1a 100644 --- a/deps/jemalloc/include/jemalloc/internal/atomic.h +++ b/deps/jemalloc/include/jemalloc/internal/atomic.h @@ -1,19 +1,12 @@ #ifndef JEMALLOC_INTERNAL_ATOMIC_H #define JEMALLOC_INTERNAL_ATOMIC_H -#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE +#define ATOMIC_INLINE static inline -#define JEMALLOC_U8_ATOMICS #if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) # include "jemalloc/internal/atomic_gcc_atomic.h" -# if !defined(JEMALLOC_GCC_U8_ATOMIC_ATOMICS) -# undef JEMALLOC_U8_ATOMICS -# endif #elif defined(JEMALLOC_GCC_SYNC_ATOMICS) # include "jemalloc/internal/atomic_gcc_sync.h" -# if !defined(JEMALLOC_GCC_U8_SYNC_ATOMICS) -# undef JEMALLOC_U8_ATOMICS -# endif #elif defined(_MSC_VER) # include "jemalloc/internal/atomic_msvc.h" #elif defined(JEMALLOC_C11_ATOMICS) @@ -73,8 +66,6 @@ JEMALLOC_GENERATE_INT_ATOMICS(size_t, zu, LG_SIZEOF_PTR) JEMALLOC_GENERATE_INT_ATOMICS(ssize_t, zd, LG_SIZEOF_PTR) -JEMALLOC_GENERATE_INT_ATOMICS(uint8_t, u8, 0) - JEMALLOC_GENERATE_INT_ATOMICS(uint32_t, u32, 2) #ifdef JEMALLOC_ATOMIC_U64 diff --git a/deps/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h b/deps/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h index 471515e..6b73a14 100644 --- a/deps/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h +++ b/deps/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h @@ -67,8 +67,7 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ \ ATOMIC_INLINE bool \ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ - UNUSED type *expected, type desired, \ - atomic_memory_order_t success_mo, \ + type *expected, type desired, atomic_memory_order_t success_mo, \ atomic_memory_order_t failure_mo) { \ return __atomic_compare_exchange(&a->repr, expected, &desired, \ true, atomic_enum_to_builtin(success_mo), \ @@ -77,8 +76,7 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ \ ATOMIC_INLINE bool \ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ - UNUSED type *expected, type desired, \ - atomic_memory_order_t success_mo, \ + type *expected, type desired, atomic_memory_order_t success_mo, \ atomic_memory_order_t failure_mo) { \ return __atomic_compare_exchange(&a->repr, expected, &desired, \ false, \ diff --git a/deps/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h b/deps/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h index e02b7cb..30846e4 100644 --- a/deps/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h +++ b/deps/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h @@ -27,10 +27,8 @@ atomic_fence(atomic_memory_order_t mo) { asm volatile("" ::: "memory"); # if defined(__i386__) || defined(__x86_64__) /* This is implicit on x86. */ -# elif defined(__ppc64__) - asm volatile("lwsync"); # elif defined(__ppc__) - asm volatile("sync"); + asm volatile("lwsync"); # elif defined(__sparc__) && defined(__arch64__) if (mo == atomic_memory_order_acquire) { asm volatile("membar #LoadLoad | #LoadStore"); @@ -115,8 +113,8 @@ atomic_store_##short_type(atomic_##short_type##_t *a, \ } \ \ ATOMIC_INLINE type \ -atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ +atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ + atomic_memory_order_t mo) { \ /* \ * Because of FreeBSD, we care about gcc 4.2, which doesn't have\ * an atomic exchange builtin. We fake it with a CAS loop. \ @@ -131,9 +129,8 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ \ ATOMIC_INLINE bool \ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, \ - atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ + type *expected, type desired, atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ type prev = __sync_val_compare_and_swap(&a->repr, *expected, \ desired); \ if (prev == *expected) { \ @@ -145,9 +142,8 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ } \ ATOMIC_INLINE bool \ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, \ - atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ + type *expected, type desired, atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ type prev = __sync_val_compare_and_swap(&a->repr, *expected, \ desired); \ if (prev == *expected) { \ diff --git a/deps/jemalloc/include/jemalloc/internal/background_thread_externs.h b/deps/jemalloc/include/jemalloc/internal/background_thread_externs.h index 0f997e1..3209aa4 100644 --- a/deps/jemalloc/include/jemalloc/internal/background_thread_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/background_thread_externs.h @@ -8,6 +8,7 @@ extern atomic_b_t background_thread_enabled_state; extern size_t n_background_threads; extern size_t max_background_threads; extern background_thread_info_t *background_thread_info; +extern bool can_enable_background_thread; bool background_thread_create(tsd_t *tsd, unsigned arena_ind); bool background_threads_enable(tsd_t *tsd); diff --git a/deps/jemalloc/include/jemalloc/internal/background_thread_inlines.h b/deps/jemalloc/include/jemalloc/internal/background_thread_inlines.h index f85e86f..ef50231 100644 --- a/deps/jemalloc/include/jemalloc/internal/background_thread_inlines.h +++ b/deps/jemalloc/include/jemalloc/internal/background_thread_inlines.h @@ -15,12 +15,7 @@ background_thread_enabled_set(tsdn_t *tsdn, bool state) { JEMALLOC_ALWAYS_INLINE background_thread_info_t * arena_background_thread_info_get(arena_t *arena) { unsigned arena_ind = arena_ind_get(arena); - return &background_thread_info[arena_ind % max_background_threads]; -} - -JEMALLOC_ALWAYS_INLINE background_thread_info_t * -background_thread_info_get(size_t ind) { - return &background_thread_info[ind % max_background_threads]; + return &background_thread_info[arena_ind % ncpus]; } JEMALLOC_ALWAYS_INLINE uint64_t diff --git a/deps/jemalloc/include/jemalloc/internal/background_thread_structs.h b/deps/jemalloc/include/jemalloc/internal/background_thread_structs.h index c02aa43..c1107df 100644 --- a/deps/jemalloc/include/jemalloc/internal/background_thread_structs.h +++ b/deps/jemalloc/include/jemalloc/internal/background_thread_structs.h @@ -9,7 +9,6 @@ #define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX #define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT -#define DEFAULT_NUM_BACKGROUND_THREAD 4 typedef enum { background_thread_stopped, diff --git a/deps/jemalloc/include/jemalloc/internal/base_structs.h b/deps/jemalloc/include/jemalloc/internal/base_structs.h index 07f214e..2102247 100644 --- a/deps/jemalloc/include/jemalloc/internal/base_structs.h +++ b/deps/jemalloc/include/jemalloc/internal/base_structs.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" /* Embedded at the beginning of every block of base-managed virtual memory. */ struct base_block_s { @@ -46,7 +46,7 @@ struct base_s { base_block_t *blocks; /* Heap of extents that track unused trailing space within blocks. */ - extent_heap_t avail[SC_NSIZES]; + extent_heap_t avail[NSIZES]; /* Stats, only maintained if config_stats. */ size_t allocated; diff --git a/deps/jemalloc/include/jemalloc/internal/bin.h b/deps/jemalloc/include/jemalloc/internal/bin.h index 8547e89..9b416ad 100644 --- a/deps/jemalloc/include/jemalloc/internal/bin.h +++ b/deps/jemalloc/include/jemalloc/internal/bin.h @@ -1,12 +1,10 @@ #ifndef JEMALLOC_INTERNAL_BIN_H #define JEMALLOC_INTERNAL_BIN_H -#include "jemalloc/internal/bin_stats.h" -#include "jemalloc/internal/bin_types.h" #include "jemalloc/internal/extent_types.h" #include "jemalloc/internal/extent_structs.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/bin_stats.h" /* * A bin contains a set of extents that are currently being used for slab @@ -43,9 +41,6 @@ struct bin_info_s { /* Total number of regions in a slab for this bin's size class. */ uint32_t nregs; - /* Number of sharded bins in each arena for this size class. */ - uint32_t n_shards; - /* * Metadata used to manipulate bitmaps for slabs associated with this * bin. @@ -53,7 +48,8 @@ struct bin_info_s { bitmap_info_t bitmap_info; }; -extern bin_info_t bin_infos[SC_NBINS]; +extern const bin_info_t bin_infos[NBINS]; + typedef struct bin_s bin_t; struct bin_s { @@ -82,18 +78,6 @@ struct bin_s { bin_stats_t stats; }; -/* A set of sharded bins of the same size class. */ -typedef struct bins_s bins_t; -struct bins_s { - /* Sharded bins. Dynamically sized. */ - bin_t *bin_shards; -}; - -void bin_shard_sizes_boot(unsigned bin_shards[SC_NBINS]); -bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size, - size_t end_size, size_t nshards); -void bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]); - /* Initializes a bin to empty. Returns true on error. */ bool bin_init(bin_t *bin); @@ -106,7 +90,7 @@ void bin_postfork_child(tsdn_t *tsdn, bin_t *bin); static inline void bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) { malloc_mutex_lock(tsdn, &bin->lock); - malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock); + malloc_mutex_prof_read(tsdn, &dst_bin_stats->mutex_data, &bin->lock); dst_bin_stats->nmalloc += bin->stats.nmalloc; dst_bin_stats->ndalloc += bin->stats.ndalloc; dst_bin_stats->nrequests += bin->stats.nrequests; @@ -116,7 +100,6 @@ bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) { dst_bin_stats->nslabs += bin->stats.nslabs; dst_bin_stats->reslabs += bin->stats.reslabs; dst_bin_stats->curslabs += bin->stats.curslabs; - dst_bin_stats->nonfull_slabs += bin->stats.nonfull_slabs; malloc_mutex_unlock(tsdn, &bin->lock); } diff --git a/deps/jemalloc/include/jemalloc/internal/bin_stats.h b/deps/jemalloc/include/jemalloc/internal/bin_stats.h index d04519c..86e673e 100644 --- a/deps/jemalloc/include/jemalloc/internal/bin_stats.h +++ b/deps/jemalloc/include/jemalloc/internal/bin_stats.h @@ -45,9 +45,6 @@ struct bin_stats_s { /* Current number of slabs in this bin. */ size_t curslabs; - /* Current size of nonfull slabs heap in this bin. */ - size_t nonfull_slabs; - mutex_prof_data_t mutex_data; }; diff --git a/deps/jemalloc/include/jemalloc/internal/bin_types.h b/deps/jemalloc/include/jemalloc/internal/bin_types.h deleted file mode 100644 index 3533606..0000000 --- a/deps/jemalloc/include/jemalloc/internal/bin_types.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_BIN_TYPES_H -#define JEMALLOC_INTERNAL_BIN_TYPES_H - -#include "jemalloc/internal/sc.h" - -#define BIN_SHARDS_MAX (1 << EXTENT_BITS_BINSHARD_WIDTH) -#define N_BIN_SHARDS_DEFAULT 1 - -/* Used in TSD static initializer only. Real init in arena_bind(). */ -#define TSD_BINSHARDS_ZERO_INITIALIZER {{UINT8_MAX}} - -typedef struct tsd_binshards_s tsd_binshards_t; -struct tsd_binshards_s { - uint8_t binshard[SC_NBINS]; -}; - -#endif /* JEMALLOC_INTERNAL_BIN_TYPES_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/bit_util.h b/deps/jemalloc/include/jemalloc/internal/bit_util.h index c045eb8..8d078a8 100644 --- a/deps/jemalloc/include/jemalloc/internal/bit_util.h +++ b/deps/jemalloc/include/jemalloc/internal/bit_util.h @@ -27,25 +27,6 @@ ffs_u(unsigned bitmap) { return JEMALLOC_INTERNAL_FFS(bitmap); } -#ifdef JEMALLOC_INTERNAL_POPCOUNTL -BIT_UTIL_INLINE unsigned -popcount_lu(unsigned long bitmap) { - return JEMALLOC_INTERNAL_POPCOUNTL(bitmap); -} -#endif - -/* - * Clears first unset bit in bitmap, and returns - * place of bit. bitmap *must not* be 0. - */ - -BIT_UTIL_INLINE size_t -cfs_lu(unsigned long* bitmap) { - size_t bit = ffs_lu(*bitmap) - 1; - *bitmap ^= ZU(1) << bit; - return bit; -} - BIT_UTIL_INLINE unsigned ffs_zu(size_t bitmap) { #if LG_SIZEOF_PTR == LG_SIZEOF_INT @@ -82,22 +63,6 @@ ffs_u32(uint32_t bitmap) { BIT_UTIL_INLINE uint64_t pow2_ceil_u64(uint64_t x) { -#if (defined(__amd64__) || defined(__x86_64__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ)) - if(unlikely(x <= 1)) { - return x; - } - size_t msb_on_index; -#if (defined(__amd64__) || defined(__x86_64__)) - asm ("bsrq %1, %0" - : "=r"(msb_on_index) // Outputs. - : "r"(x-1) // Inputs. - ); -#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) - msb_on_index = (63 ^ __builtin_clzll(x - 1)); -#endif - assert(msb_on_index < 63); - return 1ULL << (msb_on_index + 1); -#else x--; x |= x >> 1; x |= x >> 2; @@ -107,27 +72,10 @@ pow2_ceil_u64(uint64_t x) { x |= x >> 32; x++; return x; -#endif } BIT_UTIL_INLINE uint32_t pow2_ceil_u32(uint32_t x) { -#if ((defined(__i386__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ)) && (!defined(__s390__))) - if(unlikely(x <= 1)) { - return x; - } - size_t msb_on_index; -#if (defined(__i386__)) - asm ("bsr %1, %0" - : "=r"(msb_on_index) // Outputs. - : "r"(x-1) // Inputs. - ); -#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) - msb_on_index = (31 ^ __builtin_clz(x - 1)); -#endif - assert(msb_on_index < 31); - return 1U << (msb_on_index + 1); -#else x--; x |= x >> 1; x |= x >> 2; @@ -136,7 +84,6 @@ pow2_ceil_u32(uint32_t x) { x |= x >> 16; x++; return x; -#endif } /* Compute the smallest power of 2 that is >= x. */ @@ -213,27 +160,6 @@ lg_floor(size_t x) { } #endif -BIT_UTIL_INLINE unsigned -lg_ceil(size_t x) { - return lg_floor(x) + ((x & (x - 1)) == 0 ? 0 : 1); -} - #undef BIT_UTIL_INLINE -/* A compile-time version of lg_floor and lg_ceil. */ -#define LG_FLOOR_1(x) 0 -#define LG_FLOOR_2(x) (x < (1ULL << 1) ? LG_FLOOR_1(x) : 1 + LG_FLOOR_1(x >> 1)) -#define LG_FLOOR_4(x) (x < (1ULL << 2) ? LG_FLOOR_2(x) : 2 + LG_FLOOR_2(x >> 2)) -#define LG_FLOOR_8(x) (x < (1ULL << 4) ? LG_FLOOR_4(x) : 4 + LG_FLOOR_4(x >> 4)) -#define LG_FLOOR_16(x) (x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8)) -#define LG_FLOOR_32(x) (x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16)) -#define LG_FLOOR_64(x) (x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32)) -#if LG_SIZEOF_PTR == 2 -# define LG_FLOOR(x) LG_FLOOR_32((x)) -#else -# define LG_FLOOR(x) LG_FLOOR_64((x)) -#endif - -#define LG_CEIL(x) (LG_FLOOR(x) + (((x) & ((x) - 1)) == 0 ? 0 : 1)) - #endif /* JEMALLOC_INTERNAL_BIT_UTIL_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/bitmap.h b/deps/jemalloc/include/jemalloc/internal/bitmap.h index c3f9cb4..ac99029 100644 --- a/deps/jemalloc/include/jemalloc/internal/bitmap.h +++ b/deps/jemalloc/include/jemalloc/internal/bitmap.h @@ -3,18 +3,18 @@ #include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/bit_util.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" typedef unsigned long bitmap_t; #define LG_SIZEOF_BITMAP LG_SIZEOF_LONG /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ -#if LG_SLAB_MAXREGS > LG_CEIL(SC_NSIZES) +#if LG_SLAB_MAXREGS > LG_CEIL_NSIZES /* Maximum bitmap bit count is determined by maximum regions per slab. */ # define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS #else /* Maximum bitmap bit count is determined by number of extent size classes. */ -# define LG_BITMAP_MAXBITS LG_CEIL(SC_NSIZES) +# define LG_BITMAP_MAXBITS LG_CEIL_NSIZES #endif #define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS) diff --git a/deps/jemalloc/include/jemalloc/internal/cache_bin.h b/deps/jemalloc/include/jemalloc/internal/cache_bin.h index d14556a..12f3ef2 100644 --- a/deps/jemalloc/include/jemalloc/internal/cache_bin.h +++ b/deps/jemalloc/include/jemalloc/internal/cache_bin.h @@ -88,21 +88,11 @@ JEMALLOC_ALWAYS_INLINE void * cache_bin_alloc_easy(cache_bin_t *bin, bool *success) { void *ret; - bin->ncached--; - - /* - * Check for both bin->ncached == 0 and ncached < low_water - * in a single branch. - */ - if (unlikely(bin->ncached <= bin->low_water)) { - bin->low_water = bin->ncached; - if (bin->ncached == -1) { - bin->ncached = 0; - *success = false; - return NULL; - } + if (unlikely(bin->ncached == 0)) { + bin->low_water = -1; + *success = false; + return NULL; } - /* * success (instead of ret) should be checked upon the return of this * function. We avoid checking (ret == NULL) because there is never a @@ -111,21 +101,14 @@ cache_bin_alloc_easy(cache_bin_t *bin, bool *success) { * cacheline). */ *success = true; - ret = *(bin->avail - (bin->ncached + 1)); + ret = *(bin->avail - bin->ncached); + bin->ncached--; + + if (unlikely(bin->ncached < bin->low_water)) { + bin->low_water = bin->ncached; + } return ret; } -JEMALLOC_ALWAYS_INLINE bool -cache_bin_dalloc_easy(cache_bin_t *bin, cache_bin_info_t *bin_info, void *ptr) { - if (unlikely(bin->ncached == bin_info->ncached_max)) { - return false; - } - assert(bin->ncached < bin_info->ncached_max); - bin->ncached++; - *(bin->avail - bin->ncached) = ptr; - - return true; -} - #endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/ctl.h b/deps/jemalloc/include/jemalloc/internal/ctl.h index 1d1aacc..d927d94 100644 --- a/deps/jemalloc/include/jemalloc/internal/ctl.h +++ b/deps/jemalloc/include/jemalloc/internal/ctl.h @@ -5,7 +5,7 @@ #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex_prof.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" /* Maximum ctl tree depth. */ @@ -39,12 +39,9 @@ typedef struct ctl_arena_stats_s { uint64_t nmalloc_small; uint64_t ndalloc_small; uint64_t nrequests_small; - uint64_t nfills_small; - uint64_t nflushes_small; - bin_stats_t bstats[SC_NBINS]; - arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; - arena_stats_extents_t estats[SC_NPSIZES]; + bin_stats_t bstats[NBINS]; + arena_stats_large_t lstats[NSIZES - NBINS]; } ctl_arena_stats_t; typedef struct ctl_stats_s { diff --git a/deps/jemalloc/include/jemalloc/internal/emitter.h b/deps/jemalloc/include/jemalloc/internal/emitter.h index 542bc79..3a2b2f7 100644 --- a/deps/jemalloc/include/jemalloc/internal/emitter.h +++ b/deps/jemalloc/include/jemalloc/internal/emitter.h @@ -45,9 +45,7 @@ struct emitter_col_s { int int_val; unsigned unsigned_val; uint32_t uint32_val; - uint32_t uint32_t_val; uint64_t uint64_val; - uint64_t uint64_t_val; size_t size_val; ssize_t ssize_val; const char *str_val; @@ -62,6 +60,17 @@ struct emitter_row_s { ql_head(emitter_col_t) cols; }; +static inline void +emitter_row_init(emitter_row_t *row) { + ql_new(&row->cols); +} + +static inline void +emitter_col_init(emitter_col_t *col, emitter_row_t *row) { + ql_elm_new(col, link); + ql_tail_insert(&row->cols, col, link); +} + typedef struct emitter_s emitter_t; struct emitter_s { emitter_output_t output; @@ -71,10 +80,18 @@ struct emitter_s { int nesting_depth; /* True if we've already emitted a value at the given depth. */ bool item_at_depth; - /* True if we emitted a key and will emit corresponding value next. */ - bool emitted_key; }; +static inline void +emitter_init(emitter_t *emitter, emitter_output_t emitter_output, + void (*write_cb)(void *, const char *), void *cbopaque) { + emitter->output = emitter_output; + emitter->write_cb = write_cb; + emitter->cbopaque = cbopaque; + emitter->item_at_depth = false; + emitter->nesting_depth = 0; +} + /* Internal convenience function. Write to the emitter the given string. */ JEMALLOC_FORMAT_PRINTF(2, 3) static inline void @@ -86,11 +103,22 @@ emitter_printf(emitter_t *emitter, const char *format, ...) { va_end(ap); } -static inline const char * JEMALLOC_FORMAT_ARG(3) +/* Write to the emitter the given string, but only in table mode. */ +JEMALLOC_FORMAT_PRINTF(2, 3) +static inline void +emitter_table_printf(emitter_t *emitter, const char *format, ...) { + if (emitter->output == emitter_output_table) { + va_list ap; + va_start(ap, format); + malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap); + va_end(ap); + } +} + +static inline void emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier, emitter_justify_t justify, int width) { size_t written; - fmt_specifier++; if (justify == emitter_justify_none) { written = malloc_snprintf(out_fmt, out_size, "%%%s", fmt_specifier); @@ -103,7 +131,6 @@ emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier, } /* Only happens in case of bad format string, which *we* choose. */ assert(written < out_size); - return out_fmt; } /* @@ -129,27 +156,26 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width, char buf[BUF_SIZE]; #define EMIT_SIMPLE(type, format) \ - emitter_printf(emitter, \ - emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width), \ - *(const type *)value); + emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width); \ + emitter_printf(emitter, fmt, *(const type *)value); \ switch (value_type) { case emitter_type_bool: - emitter_printf(emitter, - emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), - *(const bool *)value ? "true" : "false"); + emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width); + emitter_printf(emitter, fmt, *(const bool *)value ? + "true" : "false"); break; case emitter_type_int: - EMIT_SIMPLE(int, "%d") + EMIT_SIMPLE(int, "d") break; case emitter_type_unsigned: - EMIT_SIMPLE(unsigned, "%u") + EMIT_SIMPLE(unsigned, "u") break; case emitter_type_ssize: - EMIT_SIMPLE(ssize_t, "%zd") + EMIT_SIMPLE(ssize_t, "zd") break; case emitter_type_size: - EMIT_SIMPLE(size_t, "%zu") + EMIT_SIMPLE(size_t, "zu") break; case emitter_type_string: str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"", @@ -159,17 +185,17 @@ emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width, * anywhere near the fmt size. */ assert(str_written < BUF_SIZE); - emitter_printf(emitter, - emitter_gen_fmt(fmt, FMT_SIZE, "%s", justify, width), buf); + emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width); + emitter_printf(emitter, fmt, buf); break; case emitter_type_uint32: - EMIT_SIMPLE(uint32_t, "%" FMTu32) + EMIT_SIMPLE(uint32_t, FMTu32) break; case emitter_type_uint64: - EMIT_SIMPLE(uint64_t, "%" FMTu64) + EMIT_SIMPLE(uint64_t, FMTu64) break; case emitter_type_title: - EMIT_SIMPLE(char *const, "%s"); + EMIT_SIMPLE(char *const, "s"); break; default: unreachable(); @@ -209,143 +235,47 @@ emitter_indent(emitter_t *emitter) { static inline void emitter_json_key_prefix(emitter_t *emitter) { - if (emitter->emitted_key) { - emitter->emitted_key = false; - return; - } emitter_printf(emitter, "%s\n", emitter->item_at_depth ? "," : ""); emitter_indent(emitter); } -/******************************************************************************/ -/* Public functions for emitter_t. */ - static inline void -emitter_init(emitter_t *emitter, emitter_output_t emitter_output, - void (*write_cb)(void *, const char *), void *cbopaque) { - emitter->output = emitter_output; - emitter->write_cb = write_cb; - emitter->cbopaque = cbopaque; - emitter->item_at_depth = false; - emitter->emitted_key = false; - emitter->nesting_depth = 0; -} - -/******************************************************************************/ -/* JSON public API. */ - -/* - * Emits a key (e.g. as appears in an object). The next json entity emitted will - * be the corresponding value. - */ -static inline void -emitter_json_key(emitter_t *emitter, const char *json_key) { +emitter_begin(emitter_t *emitter) { if (emitter->output == emitter_output_json) { - emitter_json_key_prefix(emitter); - emitter_printf(emitter, "\"%s\": ", json_key); - emitter->emitted_key = true; - } -} - -static inline void -emitter_json_value(emitter_t *emitter, emitter_type_t value_type, - const void *value) { - if (emitter->output == emitter_output_json) { - emitter_json_key_prefix(emitter); - emitter_print_value(emitter, emitter_justify_none, -1, - value_type, value); - emitter->item_at_depth = true; - } -} - -/* Shorthand for calling emitter_json_key and then emitter_json_value. */ -static inline void -emitter_json_kv(emitter_t *emitter, const char *json_key, - emitter_type_t value_type, const void *value) { - emitter_json_key(emitter, json_key); - emitter_json_value(emitter, value_type, value); -} - -static inline void -emitter_json_array_begin(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - emitter_json_key_prefix(emitter); - emitter_printf(emitter, "["); - emitter_nest_inc(emitter); - } -} - -/* Shorthand for calling emitter_json_key and then emitter_json_array_begin. */ -static inline void -emitter_json_array_kv_begin(emitter_t *emitter, const char *json_key) { - emitter_json_key(emitter, json_key); - emitter_json_array_begin(emitter); -} - -static inline void -emitter_json_array_end(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth > 0); - emitter_nest_dec(emitter); - emitter_printf(emitter, "\n"); - emitter_indent(emitter); - emitter_printf(emitter, "]"); - } -} - -static inline void -emitter_json_object_begin(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - emitter_json_key_prefix(emitter); + assert(emitter->nesting_depth == 0); emitter_printf(emitter, "{"); emitter_nest_inc(emitter); + } else { + // tabular init + emitter_printf(emitter, "%s", ""); } } -/* Shorthand for calling emitter_json_key and then emitter_json_object_begin. */ static inline void -emitter_json_object_kv_begin(emitter_t *emitter, const char *json_key) { - emitter_json_key(emitter, json_key); - emitter_json_object_begin(emitter); -} - -static inline void -emitter_json_object_end(emitter_t *emitter) { +emitter_end(emitter_t *emitter) { if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth > 0); + assert(emitter->nesting_depth == 1); emitter_nest_dec(emitter); - emitter_printf(emitter, "\n"); - emitter_indent(emitter); - emitter_printf(emitter, "}"); + emitter_printf(emitter, "\n}\n"); } } - -/******************************************************************************/ -/* Table public API. */ - +/* + * Note emits a different kv pair as well, but only in table mode. Omits the + * note if table_note_key is NULL. + */ static inline void -emitter_table_dict_begin(emitter_t *emitter, const char *table_key) { - if (emitter->output == emitter_output_table) { - emitter_indent(emitter); - emitter_printf(emitter, "%s\n", table_key); - emitter_nest_inc(emitter); - } -} - -static inline void -emitter_table_dict_end(emitter_t *emitter) { - if (emitter->output == emitter_output_table) { - emitter_nest_dec(emitter); - } -} - -static inline void -emitter_table_kv_note(emitter_t *emitter, const char *table_key, +emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key, emitter_type_t value_type, const void *value, const char *table_note_key, emitter_type_t table_note_value_type, const void *table_note_value) { - if (emitter->output == emitter_output_table) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth > 0); + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "\"%s\": ", json_key); + emitter_print_value(emitter, emitter_justify_none, -1, + value_type, value); + } else { emitter_indent(emitter); emitter_printf(emitter, "%s: ", table_key); emitter_print_value(emitter, emitter_justify_none, -1, @@ -362,22 +292,130 @@ emitter_table_kv_note(emitter_t *emitter, const char *table_key, } static inline void -emitter_table_kv(emitter_t *emitter, const char *table_key, +emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key, emitter_type_t value_type, const void *value) { - emitter_table_kv_note(emitter, table_key, value_type, value, NULL, + emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL, emitter_type_bool, NULL); } - -/* Write to the emitter the given string, but only in table mode. */ -JEMALLOC_FORMAT_PRINTF(2, 3) static inline void -emitter_table_printf(emitter_t *emitter, const char *format, ...) { +emitter_json_kv(emitter_t *emitter, const char *json_key, + emitter_type_t value_type, const void *value) { + if (emitter->output == emitter_output_json) { + emitter_kv(emitter, json_key, NULL, value_type, value); + } +} + +static inline void +emitter_table_kv(emitter_t *emitter, const char *table_key, + emitter_type_t value_type, const void *value) { if (emitter->output == emitter_output_table) { - va_list ap; - va_start(ap, format); - malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap); - va_end(ap); + emitter_kv(emitter, NULL, table_key, value_type, value); + } +} + +static inline void +emitter_dict_begin(emitter_t *emitter, const char *json_key, + const char *table_header) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "\"%s\": {", json_key); + emitter_nest_inc(emitter); + } else { + emitter_indent(emitter); + emitter_printf(emitter, "%s\n", table_header); + emitter_nest_inc(emitter); + } +} + +static inline void +emitter_dict_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth > 0); + emitter_nest_dec(emitter); + emitter_printf(emitter, "\n"); + emitter_indent(emitter); + emitter_printf(emitter, "}"); + } else { + emitter_nest_dec(emitter); + } +} + +static inline void +emitter_json_dict_begin(emitter_t *emitter, const char *json_key) { + if (emitter->output == emitter_output_json) { + emitter_dict_begin(emitter, json_key, NULL); + } +} + +static inline void +emitter_json_dict_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + emitter_dict_end(emitter); + } +} + +static inline void +emitter_table_dict_begin(emitter_t *emitter, const char *table_key) { + if (emitter->output == emitter_output_table) { + emitter_dict_begin(emitter, NULL, table_key); + } +} + +static inline void +emitter_table_dict_end(emitter_t *emitter) { + if (emitter->output == emitter_output_table) { + emitter_dict_end(emitter); + } +} + +static inline void +emitter_json_arr_begin(emitter_t *emitter, const char *json_key) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "\"%s\": [", json_key); + emitter_nest_inc(emitter); + } +} + +static inline void +emitter_json_arr_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth > 0); + emitter_nest_dec(emitter); + emitter_printf(emitter, "\n"); + emitter_indent(emitter); + emitter_printf(emitter, "]"); + } +} + +static inline void +emitter_json_arr_obj_begin(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "{"); + emitter_nest_inc(emitter); + } +} + +static inline void +emitter_json_arr_obj_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth > 0); + emitter_nest_dec(emitter); + emitter_printf(emitter, "\n"); + emitter_indent(emitter); + emitter_printf(emitter, "}"); + } +} + +static inline void +emitter_json_arr_value(emitter_t *emitter, emitter_type_t value_type, + const void *value) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_print_value(emitter, emitter_justify_none, -1, + value_type, value); } } @@ -394,93 +432,4 @@ emitter_table_row(emitter_t *emitter, emitter_row_t *row) { emitter_table_printf(emitter, "\n"); } -static inline void -emitter_row_init(emitter_row_t *row) { - ql_new(&row->cols); -} - -static inline void -emitter_col_init(emitter_col_t *col, emitter_row_t *row) { - ql_elm_new(col, link); - ql_tail_insert(&row->cols, col, link); -} - - -/******************************************************************************/ -/* - * Generalized public API. Emits using either JSON or table, according to - * settings in the emitter_t. */ - -/* - * Note emits a different kv pair as well, but only in table mode. Omits the - * note if table_note_key is NULL. - */ -static inline void -emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key, - emitter_type_t value_type, const void *value, - const char *table_note_key, emitter_type_t table_note_value_type, - const void *table_note_value) { - if (emitter->output == emitter_output_json) { - emitter_json_key(emitter, json_key); - emitter_json_value(emitter, value_type, value); - } else { - emitter_table_kv_note(emitter, table_key, value_type, value, - table_note_key, table_note_value_type, table_note_value); - } - emitter->item_at_depth = true; -} - -static inline void -emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key, - emitter_type_t value_type, const void *value) { - emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL, - emitter_type_bool, NULL); -} - -static inline void -emitter_dict_begin(emitter_t *emitter, const char *json_key, - const char *table_header) { - if (emitter->output == emitter_output_json) { - emitter_json_key(emitter, json_key); - emitter_json_object_begin(emitter); - } else { - emitter_table_dict_begin(emitter, table_header); - } -} - -static inline void -emitter_dict_end(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - emitter_json_object_end(emitter); - } else { - emitter_table_dict_end(emitter); - } -} - -static inline void -emitter_begin(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth == 0); - emitter_printf(emitter, "{"); - emitter_nest_inc(emitter); - } else { - /* - * This guarantees that we always call write_cb at least once. - * This is useful if some invariant is established by each call - * to write_cb, but doesn't hold initially: e.g., some buffer - * holds a null-terminated string. - */ - emitter_printf(emitter, "%s", ""); - } -} - -static inline void -emitter_end(emitter_t *emitter) { - if (emitter->output == emitter_output_json) { - assert(emitter->nesting_depth == 1); - emitter_nest_dec(emitter); - emitter_printf(emitter, "\n}\n"); - } -} - #endif /* JEMALLOC_INTERNAL_EMITTER_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/extent_externs.h b/deps/jemalloc/include/jemalloc/internal/extent_externs.h index 8aba576..b8a4d02 100644 --- a/deps/jemalloc/include/jemalloc/internal/extent_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/extent_externs.h @@ -24,17 +24,13 @@ size_t extent_size_quantize_floor(size_t size); size_t extent_size_quantize_ceil(size_t size); #endif -ph_proto(, extent_avail_, extent_tree_t, extent_t) +rb_proto(, extent_avail_, extent_tree_t, extent_t) ph_proto(, extent_heap_, extent_heap_t, extent_t) bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, bool delay_coalesce); extent_state_t extents_state_get(const extents_t *extents); size_t extents_npages_get(extents_t *extents); -/* Get the number of extents in the given page size index. */ -size_t extents_nextents_get(extents_t *extents, pszind_t ind); -/* Get the sum total bytes of the extents in the given page size index. */ -size_t extents_nbytes_get(extents_t *extents, pszind_t ind); extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, @@ -74,10 +70,4 @@ bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, bool extent_boot(void); -void extent_util_stats_get(tsdn_t *tsdn, const void *ptr, - size_t *nfree, size_t *nregs, size_t *size); -void extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr, - size_t *nfree, size_t *nregs, size_t *size, - size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr); - #endif /* JEMALLOC_INTERNAL_EXTENT_EXTERNS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/extent_inlines.h b/deps/jemalloc/include/jemalloc/internal/extent_inlines.h index 77fa4c4..77181df 100644 --- a/deps/jemalloc/include/jemalloc/internal/extent_inlines.h +++ b/deps/jemalloc/include/jemalloc/internal/extent_inlines.h @@ -6,7 +6,6 @@ #include "jemalloc/internal/pages.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" static inline void @@ -35,19 +34,18 @@ extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { (uintptr_t)extent2); } -static inline unsigned -extent_arena_ind_get(const extent_t *extent) { - unsigned arena_ind = (unsigned)((extent->e_bits & - EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT); - assert(arena_ind < MALLOCX_ARENA_LIMIT); - - return arena_ind; -} - static inline arena_t * extent_arena_get(const extent_t *extent) { - unsigned arena_ind = extent_arena_ind_get(extent); - + unsigned arena_ind = (unsigned)((extent->e_bits & + EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT); + /* + * The following check is omitted because we should never actually read + * a NULL arena pointer. + */ + if (false && arena_ind >= MALLOCX_ARENA_LIMIT) { + return NULL; + } + assert(arena_ind < MALLOCX_ARENA_LIMIT); return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE); } @@ -55,14 +53,14 @@ static inline szind_t extent_szind_get_maybe_invalid(const extent_t *extent) { szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >> EXTENT_BITS_SZIND_SHIFT); - assert(szind <= SC_NSIZES); + assert(szind <= NSIZES); return szind; } static inline szind_t extent_szind_get(const extent_t *extent) { szind_t szind = extent_szind_get_maybe_invalid(extent); - assert(szind < SC_NSIZES); /* Never call when "invalid". */ + assert(szind < NSIZES); /* Never call when "invalid". */ return szind; } @@ -71,14 +69,6 @@ extent_usize_get(const extent_t *extent) { return sz_index2size(extent_szind_get(extent)); } -static inline unsigned -extent_binshard_get(const extent_t *extent) { - unsigned binshard = (unsigned)((extent->e_bits & - EXTENT_BITS_BINSHARD_MASK) >> EXTENT_BITS_BINSHARD_SHIFT); - assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); - return binshard; -} - static inline size_t extent_sn_get(const extent_t *extent) { return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >> @@ -186,11 +176,6 @@ extent_prof_tctx_get(const extent_t *extent) { ATOMIC_ACQUIRE); } -static inline nstime_t -extent_prof_alloc_time_get(const extent_t *extent) { - return extent->e_alloc_time; -} - static inline void extent_arena_set(extent_t *extent, arena_t *arena) { unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U << @@ -199,21 +184,13 @@ extent_arena_set(extent_t *extent, arena_t *arena) { ((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT); } -static inline void -extent_binshard_set(extent_t *extent, unsigned binshard) { - /* The assertion assumes szind is set already. */ - assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); - extent->e_bits = (extent->e_bits & ~EXTENT_BITS_BINSHARD_MASK) | - ((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT); -} - static inline void extent_addr_set(extent_t *extent, void *addr) { extent->e_addr = addr; } static inline void -extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) { +extent_addr_randomize(UNUSED tsdn_t *tsdn, extent_t *extent, size_t alignment) { assert(extent_base_get(extent) == extent_addr_get(extent)); if (alignment < PAGE) { @@ -257,7 +234,7 @@ extent_bsize_set(extent_t *extent, size_t bsize) { static inline void extent_szind_set(extent_t *extent, szind_t szind) { - assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */ + assert(szind <= NSIZES); /* NSIZES means "invalid". */ extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) | ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT); } @@ -269,16 +246,6 @@ extent_nfree_set(extent_t *extent, unsigned nfree) { ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); } -static inline void -extent_nfree_binshard_set(extent_t *extent, unsigned nfree, unsigned binshard) { - /* The assertion assumes szind is set already. */ - assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); - extent->e_bits = (extent->e_bits & - (~EXTENT_BITS_NFREE_MASK & ~EXTENT_BITS_BINSHARD_MASK)) | - ((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT) | - ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); -} - static inline void extent_nfree_inc(extent_t *extent) { assert(extent_slab_get(extent)); @@ -291,12 +258,6 @@ extent_nfree_dec(extent_t *extent) { extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); } -static inline void -extent_nfree_sub(extent_t *extent, uint64_t n) { - assert(extent_slab_get(extent)); - extent->e_bits -= (n << EXTENT_BITS_NFREE_SHIFT); -} - static inline void extent_sn_set(extent_t *extent, size_t sn) { extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) | @@ -338,35 +299,10 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE); } -static inline void -extent_prof_alloc_time_set(extent_t *extent, nstime_t t) { - nstime_copy(&extent->e_alloc_time, &t); -} - -static inline bool -extent_is_head_get(extent_t *extent) { - if (maps_coalesce) { - not_reached(); - } - - return (bool)((extent->e_bits & EXTENT_BITS_IS_HEAD_MASK) >> - EXTENT_BITS_IS_HEAD_SHIFT); -} - -static inline void -extent_is_head_set(extent_t *extent, bool is_head) { - if (maps_coalesce) { - not_reached(); - } - - extent->e_bits = (extent->e_bits & ~EXTENT_BITS_IS_HEAD_MASK) | - ((uint64_t)is_head << EXTENT_BITS_IS_HEAD_SHIFT); -} - static inline void extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, - bool committed, bool dumpable, extent_head_state_t is_head) { + bool committed, bool dumpable) { assert(addr == PAGE_ADDR2BASE(addr) || !slab); extent_arena_set(extent, arena); @@ -380,10 +316,6 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, extent_committed_set(extent, committed); extent_dumpable_set(extent, dumpable); ql_elm_new(extent, ql_link); - if (!maps_coalesce) { - extent_is_head_set(extent, (is_head == EXTENT_IS_HEAD) ? true : - false); - } if (config_prof) { extent_prof_tctx_set(extent, NULL); } @@ -395,7 +327,7 @@ extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { extent_addr_set(extent, addr); extent_bsize_set(extent, bsize); extent_slab_set(extent, false); - extent_szind_set(extent, SC_NSIZES); + extent_szind_set(extent, NSIZES); extent_sn_set(extent, sn); extent_state_set(extent, extent_state_active); extent_zeroed_set(extent, true); diff --git a/deps/jemalloc/include/jemalloc/internal/extent_structs.h b/deps/jemalloc/include/jemalloc/internal/extent_structs.h index 767cd89..4873b9e 100644 --- a/deps/jemalloc/include/jemalloc/internal/extent_structs.h +++ b/deps/jemalloc/include/jemalloc/internal/extent_structs.h @@ -2,12 +2,11 @@ #define JEMALLOC_INTERNAL_EXTENT_STRUCTS_H #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/ql.h" #include "jemalloc/internal/ph.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" typedef enum { extent_state_active = 0, @@ -29,10 +28,9 @@ struct extent_s { * t: state * i: szind * f: nfree - * s: bin_shard * n: sn * - * nnnnnnnn ... nnnnnnss ssssffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa + * nnnnnnnn ... nnnnffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa * * arena_ind: Arena from which this extent came, or all 1 bits if * unassociated. @@ -77,8 +75,6 @@ struct extent_s { * * nfree: Number of free regions in slab. * - * bin_shard: the shard of the bin from which this extent came. - * * sn: Serial number (potentially non-unique). * * Serial numbers may wrap around if !opt_retain, but as long as @@ -116,7 +112,7 @@ struct extent_s { #define EXTENT_BITS_STATE_SHIFT (EXTENT_BITS_ZEROED_WIDTH + EXTENT_BITS_ZEROED_SHIFT) #define EXTENT_BITS_STATE_MASK MASK(EXTENT_BITS_STATE_WIDTH, EXTENT_BITS_STATE_SHIFT) -#define EXTENT_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES) +#define EXTENT_BITS_SZIND_WIDTH LG_CEIL_NSIZES #define EXTENT_BITS_SZIND_SHIFT (EXTENT_BITS_STATE_WIDTH + EXTENT_BITS_STATE_SHIFT) #define EXTENT_BITS_SZIND_MASK MASK(EXTENT_BITS_SZIND_WIDTH, EXTENT_BITS_SZIND_SHIFT) @@ -124,15 +120,7 @@ struct extent_s { #define EXTENT_BITS_NFREE_SHIFT (EXTENT_BITS_SZIND_WIDTH + EXTENT_BITS_SZIND_SHIFT) #define EXTENT_BITS_NFREE_MASK MASK(EXTENT_BITS_NFREE_WIDTH, EXTENT_BITS_NFREE_SHIFT) -#define EXTENT_BITS_BINSHARD_WIDTH 6 -#define EXTENT_BITS_BINSHARD_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT) -#define EXTENT_BITS_BINSHARD_MASK MASK(EXTENT_BITS_BINSHARD_WIDTH, EXTENT_BITS_BINSHARD_SHIFT) - -#define EXTENT_BITS_IS_HEAD_WIDTH 1 -#define EXTENT_BITS_IS_HEAD_SHIFT (EXTENT_BITS_BINSHARD_WIDTH + EXTENT_BITS_BINSHARD_SHIFT) -#define EXTENT_BITS_IS_HEAD_MASK MASK(EXTENT_BITS_IS_HEAD_WIDTH, EXTENT_BITS_IS_HEAD_SHIFT) - -#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_IS_HEAD_WIDTH + EXTENT_BITS_IS_HEAD_SHIFT) +#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT) #define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT) /* Pointer to the extent that this structure is responsible for. */ @@ -172,13 +160,11 @@ struct extent_s { /* Small region slab metadata. */ arena_slab_data_t e_slab_data; - /* Profiling data, used for large objects. */ - struct { - /* Time when this was allocated. */ - nstime_t e_alloc_time; - /* Points to a prof_tctx_t. */ - atomic_p_t e_prof_tctx; - }; + /* + * Profile counters, used for large objects. Points to a + * prof_tctx_t. + */ + atomic_p_t e_prof_tctx; }; }; typedef ql_head(extent_t) extent_list_t; @@ -194,16 +180,14 @@ struct extents_s { * * Synchronization: mtx. */ - extent_heap_t heaps[SC_NPSIZES + 1]; - atomic_zu_t nextents[SC_NPSIZES + 1]; - atomic_zu_t nbytes[SC_NPSIZES + 1]; + extent_heap_t heaps[NPSIZES+1]; /* * Bitmap for which set bits correspond to non-empty heaps. * * Synchronization: mtx. */ - bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)]; + bitmap_t bitmap[BITMAP_GROUPS(NPSIZES+1)]; /* * LRU of all extents in heaps. @@ -232,25 +216,4 @@ struct extents_s { bool delay_coalesce; }; -/* - * The following two structs are for experimental purposes. See - * experimental_utilization_query_ctl and - * experimental_utilization_batch_query_ctl in src/ctl.c. - */ - -struct extent_util_stats_s { - size_t nfree; - size_t nregs; - size_t size; -}; - -struct extent_util_stats_verbose_s { - void *slabcur_addr; - size_t nfree; - size_t nregs; - size_t size; - size_t bin_nfree; - size_t bin_nregs; -}; - #endif /* JEMALLOC_INTERNAL_EXTENT_STRUCTS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/extent_types.h b/deps/jemalloc/include/jemalloc/internal/extent_types.h index 96925cf..c0561d9 100644 --- a/deps/jemalloc/include/jemalloc/internal/extent_types.h +++ b/deps/jemalloc/include/jemalloc/internal/extent_types.h @@ -4,20 +4,14 @@ typedef struct extent_s extent_t; typedef struct extents_s extents_t; -typedef struct extent_util_stats_s extent_util_stats_t; -typedef struct extent_util_stats_verbose_s extent_util_stats_verbose_t; - #define EXTENT_HOOKS_INITIALIZER NULL +#define EXTENT_GROW_MAX_PIND (NPSIZES - 1) + /* * When reuse (and split) an active extent, (1U << opt_lg_extent_max_active_fit) * is the max ratio between the size of the active extent and the new extent. */ #define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6 -typedef enum { - EXTENT_NOT_HEAD, - EXTENT_IS_HEAD /* Only relevant for Windows && opt.retain. */ -} extent_head_state_t; - #endif /* JEMALLOC_INTERNAL_EXTENT_TYPES_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/hash.h b/deps/jemalloc/include/jemalloc/internal/hash.h index 0270034..dcfc992 100644 --- a/deps/jemalloc/include/jemalloc/internal/hash.h +++ b/deps/jemalloc/include/jemalloc/internal/hash.h @@ -104,8 +104,8 @@ hash_x86_32(const void *key, int len, uint32_t seed) { uint32_t k1 = 0; switch (len & 3) { - case 3: k1 ^= tail[2] << 16; JEMALLOC_FALLTHROUGH - case 2: k1 ^= tail[1] << 8; JEMALLOC_FALLTHROUGH + case 3: k1 ^= tail[2] << 16; + case 2: k1 ^= tail[1] << 8; case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; } @@ -119,7 +119,7 @@ hash_x86_32(const void *key, int len, uint32_t seed) { return h1; } -static inline void +UNUSED static inline void hash_x86_128(const void *key, const int len, uint32_t seed, uint64_t r_out[2]) { const uint8_t * data = (const uint8_t *) key; @@ -177,29 +177,28 @@ hash_x86_128(const void *key, const int len, uint32_t seed, uint32_t k4 = 0; switch (len & 15) { - case 15: k4 ^= tail[14] << 16; JEMALLOC_FALLTHROUGH - case 14: k4 ^= tail[13] << 8; JEMALLOC_FALLTHROUGH + case 15: k4 ^= tail[14] << 16; + case 14: k4 ^= tail[13] << 8; case 13: k4 ^= tail[12] << 0; k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; - JEMALLOC_FALLTHROUGH - case 12: k3 ^= tail[11] << 24; JEMALLOC_FALLTHROUGH - case 11: k3 ^= tail[10] << 16; JEMALLOC_FALLTHROUGH - case 10: k3 ^= tail[ 9] << 8; JEMALLOC_FALLTHROUGH + + case 12: k3 ^= tail[11] << 24; + case 11: k3 ^= tail[10] << 16; + case 10: k3 ^= tail[ 9] << 8; case 9: k3 ^= tail[ 8] << 0; k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; - JEMALLOC_FALLTHROUGH - case 8: k2 ^= tail[ 7] << 24; JEMALLOC_FALLTHROUGH - case 7: k2 ^= tail[ 6] << 16; JEMALLOC_FALLTHROUGH - case 6: k2 ^= tail[ 5] << 8; JEMALLOC_FALLTHROUGH + + case 8: k2 ^= tail[ 7] << 24; + case 7: k2 ^= tail[ 6] << 16; + case 6: k2 ^= tail[ 5] << 8; case 5: k2 ^= tail[ 4] << 0; k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; - JEMALLOC_FALLTHROUGH - case 4: k1 ^= tail[ 3] << 24; JEMALLOC_FALLTHROUGH - case 3: k1 ^= tail[ 2] << 16; JEMALLOC_FALLTHROUGH - case 2: k1 ^= tail[ 1] << 8; JEMALLOC_FALLTHROUGH + + case 4: k1 ^= tail[ 3] << 24; + case 3: k1 ^= tail[ 2] << 16; + case 2: k1 ^= tail[ 1] << 8; case 1: k1 ^= tail[ 0] << 0; k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; - JEMALLOC_FALLTHROUGH } } @@ -221,7 +220,7 @@ hash_x86_128(const void *key, const int len, uint32_t seed, r_out[1] = (((uint64_t) h4) << 32) | h3; } -static inline void +UNUSED static inline void hash_x64_128(const void *key, const int len, const uint32_t seed, uint64_t r_out[2]) { const uint8_t *data = (const uint8_t *) key; @@ -261,22 +260,22 @@ hash_x64_128(const void *key, const int len, const uint32_t seed, uint64_t k2 = 0; switch (len & 15) { - case 15: k2 ^= ((uint64_t)(tail[14])) << 48; JEMALLOC_FALLTHROUGH - case 14: k2 ^= ((uint64_t)(tail[13])) << 40; JEMALLOC_FALLTHROUGH - case 13: k2 ^= ((uint64_t)(tail[12])) << 32; JEMALLOC_FALLTHROUGH - case 12: k2 ^= ((uint64_t)(tail[11])) << 24; JEMALLOC_FALLTHROUGH - case 11: k2 ^= ((uint64_t)(tail[10])) << 16; JEMALLOC_FALLTHROUGH - case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; JEMALLOC_FALLTHROUGH + case 15: k2 ^= ((uint64_t)(tail[14])) << 48; /* falls through */ + case 14: k2 ^= ((uint64_t)(tail[13])) << 40; /* falls through */ + case 13: k2 ^= ((uint64_t)(tail[12])) << 32; /* falls through */ + case 12: k2 ^= ((uint64_t)(tail[11])) << 24; /* falls through */ + case 11: k2 ^= ((uint64_t)(tail[10])) << 16; /* falls through */ + case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; /* falls through */ case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0; k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; - JEMALLOC_FALLTHROUGH - case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; JEMALLOC_FALLTHROUGH - case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; JEMALLOC_FALLTHROUGH - case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; JEMALLOC_FALLTHROUGH - case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; JEMALLOC_FALLTHROUGH - case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; JEMALLOC_FALLTHROUGH - case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; JEMALLOC_FALLTHROUGH - case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; JEMALLOC_FALLTHROUGH + /* falls through */ + case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; /* falls through */ + case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; /* falls through */ + case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; /* falls through */ + case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; /* falls through */ + case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; /* falls through */ + case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; /* falls through */ + case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; /* falls through */ case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0; k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; } diff --git a/deps/jemalloc/include/jemalloc/internal/hook.h b/deps/jemalloc/include/jemalloc/internal/hook.h deleted file mode 100644 index ee246b1..0000000 --- a/deps/jemalloc/include/jemalloc/internal/hook.h +++ /dev/null @@ -1,163 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_HOOK_H -#define JEMALLOC_INTERNAL_HOOK_H - -#include "jemalloc/internal/tsd.h" - -/* - * This API is *extremely* experimental, and may get ripped out, changed in API- - * and ABI-incompatible ways, be insufficiently or incorrectly documented, etc. - * - * It allows hooking the stateful parts of the API to see changes as they - * happen. - * - * Allocation hooks are called after the allocation is done, free hooks are - * called before the free is done, and expand hooks are called after the - * allocation is expanded. - * - * For realloc and rallocx, if the expansion happens in place, the expansion - * hook is called. If it is moved, then the alloc hook is called on the new - * location, and then the free hook is called on the old location (i.e. both - * hooks are invoked in between the alloc and the dalloc). - * - * If we return NULL from OOM, then usize might not be trustworthy. Calling - * realloc(NULL, size) only calls the alloc hook, and calling realloc(ptr, 0) - * only calls the free hook. (Calling realloc(NULL, 0) is treated as malloc(0), - * and only calls the alloc hook). - * - * Reentrancy: - * Reentrancy is guarded against from within the hook implementation. If you - * call allocator functions from within a hook, the hooks will not be invoked - * again. - * Threading: - * The installation of a hook synchronizes with all its uses. If you can - * prove the installation of a hook happens-before a jemalloc entry point, - * then the hook will get invoked (unless there's a racing removal). - * - * Hook insertion appears to be atomic at a per-thread level (i.e. if a thread - * allocates and has the alloc hook invoked, then a subsequent free on the - * same thread will also have the free hook invoked). - * - * The *removal* of a hook does *not* block until all threads are done with - * the hook. Hook authors have to be resilient to this, and need some - * out-of-band mechanism for cleaning up any dynamically allocated memory - * associated with their hook. - * Ordering: - * Order of hook execution is unspecified, and may be different than insertion - * order. - */ - -#define HOOK_MAX 4 - -enum hook_alloc_e { - hook_alloc_malloc, - hook_alloc_posix_memalign, - hook_alloc_aligned_alloc, - hook_alloc_calloc, - hook_alloc_memalign, - hook_alloc_valloc, - hook_alloc_mallocx, - - /* The reallocating functions have both alloc and dalloc variants */ - hook_alloc_realloc, - hook_alloc_rallocx, -}; -/* - * We put the enum typedef after the enum, since this file may get included by - * jemalloc_cpp.cpp, and C++ disallows enum forward declarations. - */ -typedef enum hook_alloc_e hook_alloc_t; - -enum hook_dalloc_e { - hook_dalloc_free, - hook_dalloc_dallocx, - hook_dalloc_sdallocx, - - /* - * The dalloc halves of reallocation (not called if in-place expansion - * happens). - */ - hook_dalloc_realloc, - hook_dalloc_rallocx, -}; -typedef enum hook_dalloc_e hook_dalloc_t; - - -enum hook_expand_e { - hook_expand_realloc, - hook_expand_rallocx, - hook_expand_xallocx, -}; -typedef enum hook_expand_e hook_expand_t; - -typedef void (*hook_alloc)( - void *extra, hook_alloc_t type, void *result, uintptr_t result_raw, - uintptr_t args_raw[3]); - -typedef void (*hook_dalloc)( - void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]); - -typedef void (*hook_expand)( - void *extra, hook_expand_t type, void *address, size_t old_usize, - size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); - -typedef struct hooks_s hooks_t; -struct hooks_s { - hook_alloc alloc_hook; - hook_dalloc dalloc_hook; - hook_expand expand_hook; - void *extra; -}; - -/* - * Begin implementation details; everything above this point might one day live - * in a public API. Everything below this point never will. - */ - -/* - * The realloc pathways haven't gotten any refactoring love in a while, and it's - * fairly difficult to pass information from the entry point to the hooks. We - * put the informaiton the hooks will need into a struct to encapsulate - * everything. - * - * Much of these pathways are force-inlined, so that the compiler can avoid - * materializing this struct until we hit an extern arena function. For fairly - * goofy reasons, *many* of the realloc paths hit an extern arena function. - * These paths are cold enough that it doesn't matter; eventually, we should - * rewrite the realloc code to make the expand-in-place and the - * free-then-realloc paths more orthogonal, at which point we don't need to - * spread the hook logic all over the place. - */ -typedef struct hook_ralloc_args_s hook_ralloc_args_t; -struct hook_ralloc_args_s { - /* I.e. as opposed to rallocx. */ - bool is_realloc; - /* - * The expand hook takes 4 arguments, even if only 3 are actually used; - * we add an extra one in case the user decides to memcpy without - * looking too closely at the hooked function. - */ - uintptr_t args[4]; -}; - -/* - * Returns an opaque handle to be used when removing the hook. NULL means that - * we couldn't install the hook. - */ -bool hook_boot(); - -void *hook_install(tsdn_t *tsdn, hooks_t *hooks); -/* Uninstalls the hook with the handle previously returned from hook_install. */ -void hook_remove(tsdn_t *tsdn, void *opaque); - -/* Hooks */ - -void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, - uintptr_t args_raw[3]); - -void hook_invoke_dalloc(hook_dalloc_t type, void *address, - uintptr_t args_raw[3]); - -void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, - size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); - -#endif /* JEMALLOC_INTERNAL_HOOK_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/hooks.h b/deps/jemalloc/include/jemalloc/internal/hooks.h new file mode 100644 index 0000000..cd49afc --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/hooks.h @@ -0,0 +1,19 @@ +#ifndef JEMALLOC_INTERNAL_HOOKS_H +#define JEMALLOC_INTERNAL_HOOKS_H + +extern JEMALLOC_EXPORT void (*hooks_arena_new_hook)(); +extern JEMALLOC_EXPORT void (*hooks_libc_hook)(); + +#define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) + +#define open JEMALLOC_HOOK(open, hooks_libc_hook) +#define read JEMALLOC_HOOK(read, hooks_libc_hook) +#define write JEMALLOC_HOOK(write, hooks_libc_hook) +#define readlink JEMALLOC_HOOK(readlink, hooks_libc_hook) +#define close JEMALLOC_HOOK(close, hooks_libc_hook) +#define creat JEMALLOC_HOOK(creat, hooks_libc_hook) +#define secure_getenv JEMALLOC_HOOK(secure_getenv, hooks_libc_hook) +/* Note that this is undef'd and re-define'd in src/prof.c. */ +#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook) + +#endif /* JEMALLOC_INTERNAL_HOOKS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h index 7d6053e..be70df5 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h @@ -31,9 +31,6 @@ # include # endif # include -# ifdef __FreeBSD__ -# include -# endif # include # ifdef JEMALLOC_OS_UNFAIR_LOCK # include diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in index c442a21..8dad9a1 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -48,13 +48,25 @@ /* Defined if GCC __atomic atomics are available. */ #undef JEMALLOC_GCC_ATOMIC_ATOMICS -/* and the 8-bit variant support. */ -#undef JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ #undef JEMALLOC_GCC_SYNC_ATOMICS -/* and the 8-bit variant support. */ -#undef JEMALLOC_GCC_U8_SYNC_ATOMICS + +/* + * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and + * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the + * functions are defined in libgcc instead of being inlines). + */ +#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 + +/* + * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and + * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the + * functions are defined in libgcc instead of being inlines). + */ +#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -66,6 +78,12 @@ */ #undef JEMALLOC_OS_UNFAIR_LOCK +/* + * Defined if OSSpin*() functions are available, as provided by Darwin, and + * documented in the spinlock(3) manual page. + */ +#undef JEMALLOC_OSSPIN + /* Defined if syscall(2) is usable. */ #undef JEMALLOC_USE_SYSCALL @@ -135,9 +153,6 @@ /* JEMALLOC_STATS enables statistics calculation. */ #undef JEMALLOC_STATS -/* JEMALLOC_EXPERIMENTAL_SMALLOCX_API enables experimental smallocx API. */ -#undef JEMALLOC_EXPERIMENTAL_SMALLOCX_API - /* JEMALLOC_PROF enables allocation profiling. */ #undef JEMALLOC_PROF @@ -218,12 +233,6 @@ #undef JEMALLOC_INTERNAL_FFSL #undef JEMALLOC_INTERNAL_FFS -/* - * popcount*() functions to use for bitmapping. - */ -#undef JEMALLOC_INTERNAL_POPCOUNTL -#undef JEMALLOC_INTERNAL_POPCOUNT - /* * If defined, explicitly attempt to more uniformly distribute large allocation * pointer alignments across all cache indices. @@ -236,12 +245,6 @@ */ #undef JEMALLOC_LOG -/* - * If defined, use readlinkat() (instead of readlink()) to follow - * /etc/malloc_conf. - */ -#undef JEMALLOC_READLINKAT - /* * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. */ @@ -360,7 +363,4 @@ */ #undef JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE -/* Performs additional safety checks when defined. */ -#undef JEMALLOC_OPT_SAFETY_CHECKS - #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h index d291170..e10fb27 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h @@ -2,6 +2,7 @@ #define JEMALLOC_INTERNAL_EXTERNS_H #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/tsd_types.h" /* TSD checks this to set thread local slow state accordingly. */ @@ -10,7 +11,6 @@ extern bool malloc_slow; /* Run-time options. */ extern bool opt_abort; extern bool opt_abort_conf; -extern bool opt_confirm_conf; extern const char *opt_junk; extern bool opt_junk_alloc; extern bool opt_junk_free; @@ -25,9 +25,6 @@ extern unsigned ncpus; /* Number of arenas used for automatic multiplexing of threads and arenas. */ extern unsigned narenas_auto; -/* Base index for manual arenas. */ -extern unsigned manual_arena_base; - /* * Arenas that are used to service external requests. Not all elements of the * arenas array are necessarily used; arenas are created lazily as needed. @@ -52,6 +49,5 @@ void jemalloc_prefork(void); void jemalloc_postfork_parent(void); void jemalloc_postfork_child(void); bool malloc_initialized(void); -void je_sdallocx_noflags(void *ptr, size_t size); #endif /* JEMALLOC_INTERNAL_EXTERNS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h index ddde9b4..c6a1f7e 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h @@ -4,15 +4,13 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/ticker.h" JEMALLOC_ALWAYS_INLINE malloc_cpuid_t malloc_getcpu(void) { assert(have_percpu_arena); -#if defined(_WIN32) - return GetCurrentProcessorNumber(); -#elif defined(JEMALLOC_HAVE_SCHED_GETCPU) +#if defined(JEMALLOC_HAVE_SCHED_GETCPU) return (malloc_cpuid_t)sched_getcpu(); #else not_reached(); @@ -110,14 +108,14 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) { JEMALLOC_ALWAYS_INLINE cache_bin_t * tcache_small_bin_get(tcache_t *tcache, szind_t binind) { - assert(binind < SC_NBINS); + assert(binind < NBINS); return &tcache->bins_small[binind]; } JEMALLOC_ALWAYS_INLINE cache_bin_t * tcache_large_bin_get(tcache_t *tcache, szind_t binind) { - assert(binind >= SC_NBINS &&binind < nhbins); - return &tcache->bins_large[binind - SC_NBINS]; + assert(binind >= NBINS &&binind < nhbins); + return &tcache->bins_large[binind - NBINS]; } JEMALLOC_ALWAYS_INLINE bool @@ -158,7 +156,7 @@ pre_reentrancy(tsd_t *tsd, arena_t *arena) { if (fast) { /* Prepare slow path for reentrancy. */ tsd_slow_update(tsd); - assert(tsd_state_get(tsd) == tsd_state_nominal_slow); + assert(tsd->state == tsd_state_nominal_slow); } } diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h index 70d6e57..2e76e5d 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h @@ -71,8 +71,7 @@ arena_ichoose(tsd_t *tsd, arena_t *arena) { static inline bool arena_is_auto(arena_t *arena) { assert(narenas_auto > 0); - - return (arena_ind_get(arena) < manual_arena_base); + return (arena_ind_get(arena) < narenas_auto); } JEMALLOC_ALWAYS_INLINE extent_t * diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h index cdb10eb..2685802 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -1,7 +1,6 @@ #ifndef JEMALLOC_INTERNAL_INLINES_C_H #define JEMALLOC_INTERNAL_INLINES_C_H -#include "jemalloc/internal/hook.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/witness.h" @@ -43,6 +42,7 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool is_internal, arena_t *arena, bool slow_path) { void *ret; + assert(size != 0); assert(!is_internal || tcache == NULL); assert(!is_internal || arena == NULL || arena_is_auto(arena)); if (!tsdn_null(tsdn) && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) == 0) { @@ -133,20 +133,31 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, JEMALLOC_ALWAYS_INLINE void * iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, - hook_ralloc_args_t *hook_args) { + size_t extra, size_t alignment, bool zero, tcache_t *tcache, + arena_t *arena) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); void *p; size_t usize, copysize; - usize = sz_sa2u(size, alignment); - if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { + usize = sz_sa2u(size + extra, alignment); + if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { return NULL; } p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); if (p == NULL) { - return NULL; + if (extra == 0) { + return NULL; + } + /* Try again, without extra this time. */ + usize = sz_sa2u(size, alignment); + if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + return NULL; + } + p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); + if (p == NULL) { + return NULL; + } } /* * Copy at most size bytes (not size+extra), since the caller has no @@ -154,26 +165,13 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(p, ptr, copysize); - hook_invoke_alloc(hook_args->is_realloc - ? hook_alloc_realloc : hook_alloc_rallocx, p, (uintptr_t)p, - hook_args->args); - hook_invoke_dalloc(hook_args->is_realloc - ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); return p; } -/* - * is_realloc threads through the knowledge of whether or not this call comes - * from je_realloc (as opposed to je_rallocx); this ensures that we pass the - * correct entry point into any hooks. - * Note that these functions are all force-inlined, so no actual bool gets - * passed-around anywhere. - */ JEMALLOC_ALWAYS_INLINE void * iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero, tcache_t *tcache, arena_t *arena, hook_ralloc_args_t *hook_args) -{ + bool zero, tcache_t *tcache, arena_t *arena) { assert(ptr != NULL); assert(size != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -185,24 +183,24 @@ iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, * Existing object alignment is inadequate; allocate new space * and copy. */ - return iralloct_realign(tsdn, ptr, oldsize, size, alignment, - zero, tcache, arena, hook_args); + return iralloct_realign(tsdn, ptr, oldsize, size, 0, alignment, + zero, tcache, arena); } return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero, - tcache, hook_args); + tcache); } JEMALLOC_ALWAYS_INLINE void * iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero, hook_ralloc_args_t *hook_args) { + bool zero) { return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero, - tcache_get(tsd), NULL, hook_args); + tcache_get(tsd), NULL); } JEMALLOC_ALWAYS_INLINE bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero, size_t *newsize) { + size_t alignment, bool zero) { assert(ptr != NULL); assert(size != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -211,12 +209,49 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { /* Existing object alignment is inadequate. */ - *newsize = oldsize; return true; } - return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero, - newsize); + return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero); +} + +JEMALLOC_ALWAYS_INLINE int +iget_defrag_hint(tsdn_t *tsdn, void* ptr) { + int defrag = 0; + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + szind_t szind; + bool is_slab; + rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &szind, &is_slab); + if (likely(is_slab)) { + /* Small allocation. */ + extent_t *slab = iealloc(tsdn, ptr); + arena_t *arena = extent_arena_get(slab); + szind_t binind = extent_szind_get(slab); + bin_t *bin = &arena->bins[binind]; + malloc_mutex_lock(tsdn, &bin->lock); + /* don't bother moving allocations from the slab currently used for new allocations */ + if (slab != bin->slabcur) { + int free_in_slab = extent_nfree_get(slab); + if (free_in_slab) { + const bin_info_t *bin_info = &bin_infos[binind]; + int curslabs = bin->stats.curslabs; + size_t curregs = bin->stats.curregs; + if (bin->slabcur) { + /* remove slabcur from the overall utilization */ + curregs -= bin_info->nregs - extent_nfree_get(bin->slabcur); + curslabs -= 1; + } + /* Compare the utilization ratio of the slab in question to the total average, + * to avoid precision lost and division, we do that by extrapolating the usage + * of the slab as if all slabs have the same usage. If this slab is less used + * than the average, we'll prefer to evict the data to hopefully more used ones */ + defrag = (bin_info->nregs - free_in_slab) * curslabs <= curregs; + } + } + malloc_mutex_unlock(tsdn, &bin->lock); + } + return defrag; } #endif /* JEMALLOC_INTERNAL_INLINES_C_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h index d8ea06f..ed75d37 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h @@ -30,7 +30,7 @@ # define restrict #endif -/* Various function pointers are static and immutable except during testing. */ +/* Various function pointers are statick and immutable except during testing. */ #ifdef JEMALLOC_JET # define JET_MUTABLE #else @@ -40,75 +40,4 @@ #define JEMALLOC_VA_ARGS_HEAD(head, ...) head #define JEMALLOC_VA_ARGS_TAIL(head, ...) __VA_ARGS__ -#if (defined(__GNUC__) || defined(__GNUG__)) && !defined(__clang__) \ - && defined(JEMALLOC_HAVE_ATTR) && (__GNUC__ >= 7) -#define JEMALLOC_FALLTHROUGH JEMALLOC_ATTR(fallthrough); -#else -#define JEMALLOC_FALLTHROUGH /* falls through */ -#endif - -/* Diagnostic suppression macros */ -#if defined(_MSC_VER) && !defined(__clang__) -# define JEMALLOC_DIAGNOSTIC_PUSH __pragma(warning(push)) -# define JEMALLOC_DIAGNOSTIC_POP __pragma(warning(pop)) -# define JEMALLOC_DIAGNOSTIC_IGNORE(W) __pragma(warning(disable:W)) -# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS -# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS -# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN -# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS -/* #pragma GCC diagnostic first appeared in gcc 4.6. */ -#elif (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && \ - (__GNUC_MINOR__ > 5)))) || defined(__clang__) -/* - * The JEMALLOC_PRAGMA__ macro is an implementation detail of the GCC and Clang - * diagnostic suppression macros and should not be used anywhere else. - */ -# define JEMALLOC_PRAGMA__(X) _Pragma(#X) -# define JEMALLOC_DIAGNOSTIC_PUSH JEMALLOC_PRAGMA__(GCC diagnostic push) -# define JEMALLOC_DIAGNOSTIC_POP JEMALLOC_PRAGMA__(GCC diagnostic pop) -# define JEMALLOC_DIAGNOSTIC_IGNORE(W) \ - JEMALLOC_PRAGMA__(GCC diagnostic ignored W) - -/* - * The -Wmissing-field-initializers warning is buggy in GCC versions < 5.1 and - * all clang versions up to version 7 (currently trunk, unreleased). This macro - * suppresses the warning for the affected compiler versions only. - */ -# if ((defined(__GNUC__) && !defined(__clang__)) && (__GNUC__ < 5)) || \ - defined(__clang__) -# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS \ - JEMALLOC_DIAGNOSTIC_IGNORE("-Wmissing-field-initializers") -# else -# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS -# endif - -# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS \ - JEMALLOC_DIAGNOSTIC_IGNORE("-Wtype-limits") -# define JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER \ - JEMALLOC_DIAGNOSTIC_IGNORE("-Wunused-parameter") -# if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) -# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN \ - JEMALLOC_DIAGNOSTIC_IGNORE("-Walloc-size-larger-than=") -# else -# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN -# endif -# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS \ - JEMALLOC_DIAGNOSTIC_PUSH \ - JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER -#else -# define JEMALLOC_DIAGNOSTIC_PUSH -# define JEMALLOC_DIAGNOSTIC_POP -# define JEMALLOC_DIAGNOSTIC_IGNORE(W) -# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS -# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS -# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN -# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS -#endif - -/* - * Disables spurious diagnostics for all headers. Since these headers are not - * included by users directly, it does not affect their diagnostic settings. - */ -JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS - #endif /* JEMALLOC_INTERNAL_MACROS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h index e296c5a..1b750b1 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h @@ -1,8 +1,6 @@ #ifndef JEMALLOC_INTERNAL_TYPES_H #define JEMALLOC_INTERNAL_TYPES_H -#include "jemalloc/internal/quantum.h" - /* Page size index type. */ typedef unsigned pszind_t; @@ -52,6 +50,79 @@ typedef int malloc_cpuid_t; /* Smallest size class to support. */ #define TINY_MIN (1U << LG_TINY_MIN) +/* + * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +#ifndef LG_QUANTUM +# if (defined(__i386__) || defined(_M_IX86)) +# define LG_QUANTUM 4 +# endif +# ifdef __ia64__ +# define LG_QUANTUM 4 +# endif +# ifdef __alpha__ +# define LG_QUANTUM 4 +# endif +# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) +# define LG_QUANTUM 4 +# endif +# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) +# define LG_QUANTUM 4 +# endif +# ifdef __arm__ +# define LG_QUANTUM 3 +# endif +# ifdef __aarch64__ +# define LG_QUANTUM 4 +# endif +# ifdef __hppa__ +# define LG_QUANTUM 4 +# endif +# ifdef __m68k__ +# define LG_QUANTUM 3 +# endif +# ifdef __mips__ +# define LG_QUANTUM 3 +# endif +# ifdef __nios2__ +# define LG_QUANTUM 3 +# endif +# ifdef __or1k__ +# define LG_QUANTUM 3 +# endif +# ifdef __powerpc__ +# define LG_QUANTUM 4 +# endif +# if defined(__riscv) || defined(__riscv__) +# define LG_QUANTUM 4 +# endif +# ifdef __s390__ +# define LG_QUANTUM 4 +# endif +# if (defined (__SH3E__) || defined(__SH4_SINGLE__) || defined(__SH4__) || \ + defined(__SH4_SINGLE_ONLY__)) +# define LG_QUANTUM 4 +# endif +# ifdef __tile__ +# define LG_QUANTUM 4 +# endif +# ifdef __le32__ +# define LG_QUANTUM 4 +# endif +# ifndef LG_QUANTUM +# error "Unknown minimum alignment for architecture; specify via " + "--with-lg-quantum" +# endif +#endif + +#define QUANTUM ((size_t)(1U << LG_QUANTUM)) +#define QUANTUM_MASK (QUANTUM - 1) + +/* Return the smallest quantum multiple that is >= a. */ +#define QUANTUM_CEILING(a) \ + (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) + #define LONG ((size_t)(1U << LG_SIZEOF_LONG)) #define LONG_MASK (LONG - 1) diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in b/deps/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in index 3418cbf..e621fbc 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in @@ -21,7 +21,7 @@ # include "../jemalloc@install_suffix@.h" #endif -#if defined(JEMALLOC_OSATOMIC) +#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) #include #endif @@ -45,7 +45,7 @@ # include "jemalloc/internal/private_namespace_jet.h" # endif #endif -#include "jemalloc/internal/test_hooks.h" +#include "jemalloc/internal/hooks.h" #ifdef JEMALLOC_DEFINE_MADVISE_FREE # define JEMALLOC_MADV_FREE 8 @@ -161,26 +161,7 @@ static const bool config_log = false #endif ; -/* - * Are extra safety checks enabled; things like checking the size of sized - * deallocations, double-frees, etc. - */ -static const bool config_opt_safety_checks = -#ifdef JEMALLOC_OPT_SAFETY_CHECKS - true -#elif defined(JEMALLOC_DEBUG) - /* - * This lets us only guard safety checks by one flag instead of two; fast - * checks can guard solely by config_opt_safety_checks and run in debug mode - * too. - */ - true -#else - false -#endif - ; - -#if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU) +#ifdef JEMALLOC_HAVE_SCHED_GETCPU /* Currently percpu_arena depends on sched_getcpu. */ #define JEMALLOC_PERCPU_ARENA #endif diff --git a/deps/jemalloc/include/jemalloc/internal/large_externs.h b/deps/jemalloc/include/jemalloc/internal/large_externs.h index a05019e..3f36282 100644 --- a/deps/jemalloc/include/jemalloc/internal/large_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/large_externs.h @@ -1,16 +1,13 @@ #ifndef JEMALLOC_INTERNAL_LARGE_EXTERNS_H #define JEMALLOC_INTERNAL_LARGE_EXTERNS_H -#include "jemalloc/internal/hook.h" - void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero); void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero); bool large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, size_t usize_max, bool zero); -void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, - size_t alignment, bool zero, tcache_t *tcache, - hook_ralloc_args_t *hook_args); +void *large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, + size_t alignment, bool zero, tcache_t *tcache); typedef void (large_dalloc_junk_t)(void *, size_t); extern large_dalloc_junk_t *JET_MUTABLE large_dalloc_junk; @@ -26,7 +23,4 @@ prof_tctx_t *large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent); void large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx); void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent); -nstime_t large_prof_alloc_time_get(const extent_t *extent); -void large_prof_alloc_time_set(extent_t *extent, nstime_t time); - #endif /* JEMALLOC_INTERNAL_LARGE_EXTERNS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/malloc_io.h b/deps/jemalloc/include/jemalloc/internal/malloc_io.h index 1d1a414..bfe556b 100644 --- a/deps/jemalloc/include/jemalloc/internal/malloc_io.h +++ b/deps/jemalloc/include/jemalloc/internal/malloc_io.h @@ -54,7 +54,7 @@ size_t malloc_vsnprintf(char *str, size_t size, const char *format, size_t malloc_snprintf(char *str, size_t size, const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); /* - * The caller can set write_cb to null to choose to print with the + * The caller can set write_cb and cbopaque to null to choose to print with the * je_malloc_message hook. */ void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, diff --git a/deps/jemalloc/include/jemalloc/internal/mutex.h b/deps/jemalloc/include/jemalloc/internal/mutex.h index 7c24f07..6520c25 100644 --- a/deps/jemalloc/include/jemalloc/internal/mutex.h +++ b/deps/jemalloc/include/jemalloc/internal/mutex.h @@ -37,17 +37,14 @@ struct malloc_mutex_s { # endif #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock lock; +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLock lock; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) pthread_mutex_t lock; malloc_mutex_t *postponed_next; #else pthread_mutex_t lock; #endif - /* - * Hint flag to avoid exclusive cache line contention - * during spin waiting - */ - atomic_b_t locked; }; /* * We only touch witness when configured w/ debug. However we @@ -87,6 +84,10 @@ struct malloc_mutex_s { # define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) # define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock) # define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock)) +#elif (defined(JEMALLOC_OSSPIN)) +# define MALLOC_MUTEX_LOCK(m) OSSpinLockLock(&(m)->lock) +# define MALLOC_MUTEX_UNLOCK(m) OSSpinLockUnlock(&(m)->lock) +# define MALLOC_MUTEX_TRYLOCK(m) (!OSSpinLockTry(&(m)->lock)) #else # define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock) # define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) @@ -100,37 +101,22 @@ struct malloc_mutex_s { #ifdef _WIN32 # define MALLOC_MUTEX_INITIALIZER #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) -# if defined(JEMALLOC_DEBUG) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} -# else -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +#elif (defined(JEMALLOC_OSSPIN)) +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, 0}}, \ WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} -# endif #elif (defined(JEMALLOC_MUTEX_INIT_CB)) -# if (defined(JEMALLOC_DEBUG)) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} -# else -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} -# endif - +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} #else # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT -# if defined(JEMALLOC_DEBUG) # define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} -# else -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} -# endif + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} #endif #ifdef JEMALLOC_LAZY_LOCK @@ -153,7 +139,6 @@ void malloc_mutex_lock_slow(malloc_mutex_t *mutex); static inline void malloc_mutex_lock_final(malloc_mutex_t *mutex) { MALLOC_MUTEX_LOCK(mutex); - atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); } static inline bool @@ -179,7 +164,6 @@ malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { - atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); return true; } mutex_owner_stats_update(tsdn, mutex); @@ -219,7 +203,6 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { malloc_mutex_lock_slow(mutex); - atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); } mutex_owner_stats_update(tsdn, mutex); } @@ -228,7 +211,6 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { static inline void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { - atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED); witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { MALLOC_MUTEX_UNLOCK(mutex); @@ -263,26 +245,4 @@ malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); } -static inline void -malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data, - malloc_mutex_t *mutex) { - mutex_prof_data_t *source = &mutex->prof_data; - /* Can only read holding the mutex. */ - malloc_mutex_assert_owner(tsdn, mutex); - - nstime_add(&data->tot_wait_time, &source->tot_wait_time); - if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) { - nstime_copy(&data->max_wait_time, &source->max_wait_time); - } - data->n_wait_times += source->n_wait_times; - data->n_spin_acquired += source->n_spin_acquired; - if (data->max_n_thds < source->max_n_thds) { - data->max_n_thds = source->max_n_thds; - } - /* n_wait_thds is not reported. */ - atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); - data->n_owner_switches += source->n_owner_switches; - data->n_lock_ops += source->n_lock_ops; -} - #endif /* JEMALLOC_INTERNAL_MUTEX_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/mutex_prof.h b/deps/jemalloc/include/jemalloc/internal/mutex_prof.h index 2cb8fb0..ce183d3 100644 --- a/deps/jemalloc/include/jemalloc/internal/mutex_prof.h +++ b/deps/jemalloc/include/jemalloc/internal/mutex_prof.h @@ -35,31 +35,22 @@ typedef enum { mutex_prof_num_arena_mutexes } mutex_prof_arena_ind_t; -/* - * The forth parameter is a boolean value that is true for derived rate counters - * and false for real ones. - */ #define MUTEX_PROF_UINT64_COUNTERS \ - OP(num_ops, uint64_t, "n_lock_ops", false, num_ops) \ - OP(num_ops_ps, uint64_t, "(#/sec)", true, num_ops) \ - OP(num_wait, uint64_t, "n_waiting", false, num_wait) \ - OP(num_wait_ps, uint64_t, "(#/sec)", true, num_wait) \ - OP(num_spin_acq, uint64_t, "n_spin_acq", false, num_spin_acq) \ - OP(num_spin_acq_ps, uint64_t, "(#/sec)", true, num_spin_acq) \ - OP(num_owner_switch, uint64_t, "n_owner_switch", false, num_owner_switch) \ - OP(num_owner_switch_ps, uint64_t, "(#/sec)", true, num_owner_switch) \ - OP(total_wait_time, uint64_t, "total_wait_ns", false, total_wait_time) \ - OP(total_wait_time_ps, uint64_t, "(#/sec)", true, total_wait_time) \ - OP(max_wait_time, uint64_t, "max_wait_ns", false, max_wait_time) + OP(num_ops, uint64_t, "n_lock_ops") \ + OP(num_wait, uint64_t, "n_waiting") \ + OP(num_spin_acq, uint64_t, "n_spin_acq") \ + OP(num_owner_switch, uint64_t, "n_owner_switch") \ + OP(total_wait_time, uint64_t, "total_wait_ns") \ + OP(max_wait_time, uint64_t, "max_wait_ns") #define MUTEX_PROF_UINT32_COUNTERS \ - OP(max_num_thds, uint32_t, "max_n_thds", false, max_num_thds) + OP(max_num_thds, uint32_t, "max_n_thds") #define MUTEX_PROF_COUNTERS \ MUTEX_PROF_UINT64_COUNTERS \ MUTEX_PROF_UINT32_COUNTERS -#define OP(counter, type, human, derived, base_counter) mutex_counter_##counter, +#define OP(counter, type, human) mutex_counter_##counter, #define COUNTER_ENUM(counter_list, t) \ typedef enum { \ diff --git a/deps/jemalloc/include/jemalloc/internal/prof_externs.h b/deps/jemalloc/include/jemalloc/internal/prof_externs.h index 094f3e1..0434869 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/prof_externs.h @@ -14,7 +14,6 @@ extern bool opt_prof_gdump; /* High-water memory dumping. */ extern bool opt_prof_final; /* Final profile dumping. */ extern bool opt_prof_leak; /* Dump leak summary at exit. */ extern bool opt_prof_accum; /* Report cumulative bytes. */ -extern bool opt_prof_log; /* Turn logging on at boot. */ extern char opt_prof_prefix[ /* Minimize memory bloat for non-prof builds. */ #ifdef JEMALLOC_PROF @@ -46,8 +45,7 @@ extern size_t lg_prof_sample; void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated); void prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx); -void prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize, - prof_tctx_t *tctx); +void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx); void bt_init(prof_bt_t *bt, void **vec); void prof_backtrace(prof_bt_t *bt); prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt); @@ -91,15 +89,4 @@ void prof_postfork_parent(tsdn_t *tsdn); void prof_postfork_child(tsdn_t *tsdn); void prof_sample_threshold_update(prof_tdata_t *tdata); -bool prof_log_start(tsdn_t *tsdn, const char *filename); -bool prof_log_stop(tsdn_t *tsdn); -#ifdef JEMALLOC_JET -size_t prof_log_bt_count(void); -size_t prof_log_alloc_count(void); -size_t prof_log_thr_count(void); -bool prof_log_is_logging(void); -bool prof_log_rep_check(void); -void prof_log_dummy_set(bool new_value); -#endif - #endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/prof_inlines_a.h b/deps/jemalloc/include/jemalloc/internal/prof_inlines_a.h index 471d985..a6efb48 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof_inlines_a.h +++ b/deps/jemalloc/include/jemalloc/internal/prof_inlines_a.h @@ -4,8 +4,7 @@ #include "jemalloc/internal/mutex.h" static inline bool -prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, - uint64_t accumbytes) { +prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { cassert(config_prof); bool overflow; @@ -43,8 +42,7 @@ prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, } static inline void -prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, - size_t usize) { +prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { cassert(config_prof); /* @@ -57,15 +55,15 @@ prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, #ifdef JEMALLOC_ATOMIC_U64 a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); do { - a1 = (a0 >= SC_LARGE_MINCLASS - usize) - ? a0 - (SC_LARGE_MINCLASS - usize) : 0; + a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - + usize) : 0; } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); #else malloc_mutex_lock(tsdn, &prof_accum->mtx); a0 = prof_accum->accumbytes; - a1 = (a0 >= SC_LARGE_MINCLASS - usize) - ? a0 - (SC_LARGE_MINCLASS - usize) : 0; + a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - usize) : + 0; prof_accum->accumbytes = a1; malloc_mutex_unlock(tsdn, &prof_accum->mtx); #endif diff --git a/deps/jemalloc/include/jemalloc/internal/prof_inlines_b.h b/deps/jemalloc/include/jemalloc/internal/prof_inlines_b.h index 8ba8a1e..6ff465a 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof_inlines_b.h +++ b/deps/jemalloc/include/jemalloc/internal/prof_inlines_b.h @@ -1,7 +1,6 @@ #ifndef JEMALLOC_INTERNAL_PROF_INLINES_B_H #define JEMALLOC_INTERNAL_PROF_INLINES_B_H -#include "jemalloc/internal/safety_check.h" #include "jemalloc/internal/sz.h" JEMALLOC_ALWAYS_INLINE bool @@ -62,54 +61,13 @@ prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { arena_prof_tctx_reset(tsdn, ptr, tctx); } -JEMALLOC_ALWAYS_INLINE nstime_t -prof_alloc_time_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { - cassert(config_prof); - assert(ptr != NULL); - - return arena_prof_alloc_time_get(tsdn, ptr, alloc_ctx); -} - -JEMALLOC_ALWAYS_INLINE void -prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx, - nstime_t t) { - cassert(config_prof); - assert(ptr != NULL); - - arena_prof_alloc_time_set(tsdn, ptr, alloc_ctx, t); -} - -JEMALLOC_ALWAYS_INLINE bool -prof_sample_check(tsd_t *tsd, size_t usize, bool update) { - ssize_t check = update ? 0 : usize; - - int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd); - if (update) { - bytes_until_sample -= usize; - if (tsd_nominal(tsd)) { - tsd_bytes_until_sample_set(tsd, bytes_until_sample); - } - } - if (likely(bytes_until_sample >= check)) { - return true; - } - - return false; -} - JEMALLOC_ALWAYS_INLINE bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, - prof_tdata_t **tdata_out) { + prof_tdata_t **tdata_out) { prof_tdata_t *tdata; cassert(config_prof); - /* Fastpath: no need to load tdata */ - if (likely(prof_sample_check(tsd, usize, update))) { - return true; - } - - bool booted = tsd_prof_tdata_get(tsd); tdata = prof_tdata_get(tsd, true); if (unlikely((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)) { tdata = NULL; @@ -123,23 +81,21 @@ prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, return true; } - /* - * If this was the first creation of tdata, then - * prof_tdata_get() reset bytes_until_sample, so decrement and - * check it again - */ - if (!booted && prof_sample_check(tsd, usize, update)) { + if (likely(tdata->bytes_until_sample >= usize)) { + if (update) { + tdata->bytes_until_sample -= usize; + } return true; + } else { + if (tsd_reentrancy_level_get(tsd) > 0) { + return true; + } + /* Compute new sample threshold. */ + if (update) { + prof_sample_threshold_update(tdata); + } + return !tdata->active; } - - if (tsd_reentrancy_level_get(tsd) > 0) { - return true; - } - /* Compute new sample threshold. */ - if (update) { - prof_sample_threshold_update(tdata); - } - return !tdata->active; } JEMALLOC_ALWAYS_INLINE prof_tctx_t * @@ -231,7 +187,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, * counters. */ if (unlikely(old_sampled)) { - prof_free_sampled_object(tsd, ptr, old_usize, old_tctx); + prof_free_sampled_object(tsd, old_usize, old_tctx); } } @@ -243,7 +199,7 @@ prof_free(tsd_t *tsd, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx) { assert(usize == isalloc(tsd_tsdn(tsd), ptr)); if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) { - prof_free_sampled_object(tsd, ptr, usize, tctx); + prof_free_sampled_object(tsd, usize, tctx); } } diff --git a/deps/jemalloc/include/jemalloc/internal/prof_structs.h b/deps/jemalloc/include/jemalloc/internal/prof_structs.h index 34ed482..0d58ae1 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof_structs.h +++ b/deps/jemalloc/include/jemalloc/internal/prof_structs.h @@ -169,6 +169,7 @@ struct prof_tdata_s { /* Sampling state. */ uint64_t prng_state; + uint64_t bytes_until_sample; /* State used to avoid dumping while operating on prof internals. */ bool enq; diff --git a/deps/jemalloc/include/jemalloc/internal/quantum.h b/deps/jemalloc/include/jemalloc/internal/quantum.h deleted file mode 100644 index 821086e..0000000 --- a/deps/jemalloc/include/jemalloc/internal/quantum.h +++ /dev/null @@ -1,77 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_QUANTUM_H -#define JEMALLOC_INTERNAL_QUANTUM_H - -/* - * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size - * classes). - */ -#ifndef LG_QUANTUM -# if (defined(__i386__) || defined(_M_IX86)) -# define LG_QUANTUM 4 -# endif -# ifdef __ia64__ -# define LG_QUANTUM 4 -# endif -# ifdef __alpha__ -# define LG_QUANTUM 4 -# endif -# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) -# define LG_QUANTUM 4 -# endif -# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) -# define LG_QUANTUM 4 -# endif -# ifdef __arm__ -# define LG_QUANTUM 3 -# endif -# ifdef __aarch64__ -# define LG_QUANTUM 4 -# endif -# ifdef __hppa__ -# define LG_QUANTUM 4 -# endif -# ifdef __m68k__ -# define LG_QUANTUM 3 -# endif -# ifdef __mips__ -# define LG_QUANTUM 3 -# endif -# ifdef __nios2__ -# define LG_QUANTUM 3 -# endif -# ifdef __or1k__ -# define LG_QUANTUM 3 -# endif -# ifdef __powerpc__ -# define LG_QUANTUM 4 -# endif -# if defined(__riscv) || defined(__riscv__) -# define LG_QUANTUM 4 -# endif -# ifdef __s390__ -# define LG_QUANTUM 4 -# endif -# if (defined (__SH3E__) || defined(__SH4_SINGLE__) || defined(__SH4__) || \ - defined(__SH4_SINGLE_ONLY__)) -# define LG_QUANTUM 4 -# endif -# ifdef __tile__ -# define LG_QUANTUM 4 -# endif -# ifdef __le32__ -# define LG_QUANTUM 4 -# endif -# ifndef LG_QUANTUM -# error "Unknown minimum alignment for architecture; specify via " - "--with-lg-quantum" -# endif -#endif - -#define QUANTUM ((size_t)(1U << LG_QUANTUM)) -#define QUANTUM_MASK (QUANTUM - 1) - -/* Return the smallest quantum multiple that is >= a. */ -#define QUANTUM_CEILING(a) \ - (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) - -#endif /* JEMALLOC_INTERNAL_QUANTUM_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/rtree.h b/deps/jemalloc/include/jemalloc/internal/rtree.h index 16ccbeb..b59d33a 100644 --- a/deps/jemalloc/include/jemalloc/internal/rtree.h +++ b/deps/jemalloc/include/jemalloc/internal/rtree.h @@ -4,7 +4,7 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree_tsd.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/tsd.h" /* @@ -31,7 +31,7 @@ # error Unsupported number of significant virtual address bits #endif /* Use compact leaf representation if virtual address encoding allows. */ -#if RTREE_NHIB >= LG_CEIL(SC_NSIZES) +#if RTREE_NHIB >= LG_CEIL_NSIZES # define RTREE_LEAF_COMPACT #endif @@ -170,8 +170,8 @@ rtree_subkey(uintptr_t key, unsigned level) { */ # ifdef RTREE_LEAF_COMPACT JEMALLOC_ALWAYS_INLINE uintptr_t -rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, bool dependent) { +rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + bool dependent) { return (uintptr_t)atomic_load_p(&elm->le_bits, dependent ? ATOMIC_RELAXED : ATOMIC_ACQUIRE); } @@ -208,7 +208,7 @@ rtree_leaf_elm_bits_slab_get(uintptr_t bits) { # endif JEMALLOC_ALWAYS_INLINE extent_t * -rtree_leaf_elm_extent_read(tsdn_t *tsdn, rtree_t *rtree, +rtree_leaf_elm_extent_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); @@ -221,7 +221,7 @@ rtree_leaf_elm_extent_read(tsdn_t *tsdn, rtree_t *rtree, } JEMALLOC_ALWAYS_INLINE szind_t -rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree, +rtree_leaf_elm_szind_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); @@ -233,7 +233,7 @@ rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree, } JEMALLOC_ALWAYS_INLINE bool -rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, +rtree_leaf_elm_slab_read(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); @@ -245,7 +245,7 @@ rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, } static inline void -rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, +rtree_leaf_elm_extent_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, rtree_leaf_elm_t *elm, extent_t *extent) { #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, true); @@ -259,9 +259,9 @@ rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, } static inline void -rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, +rtree_leaf_elm_szind_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, rtree_leaf_elm_t *elm, szind_t szind) { - assert(szind <= SC_NSIZES); + assert(szind <= NSIZES); #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, @@ -277,7 +277,7 @@ rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, } static inline void -rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, +rtree_leaf_elm_slab_write(UNUSED tsdn_t *tsdn, UNUSED rtree_t *rtree, rtree_leaf_elm_t *elm, bool slab) { #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, @@ -292,8 +292,8 @@ rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, } static inline void -rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, - rtree_leaf_elm_t *elm, extent_t *extent, szind_t szind, bool slab) { +rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, + extent_t *extent, szind_t szind, bool slab) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = ((uintptr_t)szind << LG_VADDR) | ((uintptr_t)extent & (((uintptr_t)0x1 << LG_VADDR) - 1)) | @@ -313,7 +313,7 @@ rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, static inline void rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, szind_t szind, bool slab) { - assert(!slab || szind < SC_NBINS); + assert(!slab || szind < NBINS); /* * The caller implicitly assures that it is the only writer to the szind @@ -429,7 +429,7 @@ rtree_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, dependent); if (!dependent && elm == NULL) { - return SC_NSIZES; + return NSIZES; } return rtree_leaf_elm_szind_read(tsdn, rtree, elm, dependent); } @@ -452,42 +452,6 @@ rtree_extent_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, return false; } -/* - * Try to read szind_slab from the L1 cache. Returns true on a hit, - * and fills in r_szind and r_slab. Otherwise returns false. - * - * Key is allowed to be NULL in order to save an extra branch on the - * fastpath. returns false in this case. - */ -JEMALLOC_ALWAYS_INLINE bool -rtree_szind_slab_read_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, - uintptr_t key, szind_t *r_szind, bool *r_slab) { - rtree_leaf_elm_t *elm; - - size_t slot = rtree_cache_direct_map(key); - uintptr_t leafkey = rtree_leafkey(key); - assert(leafkey != RTREE_LEAFKEY_INVALID); - - if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) { - rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf; - assert(leaf != NULL); - uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1); - elm = &leaf[subkey]; - -#ifdef RTREE_LEAF_COMPACT - uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, - elm, true); - *r_szind = rtree_leaf_elm_bits_szind_get(bits); - *r_slab = rtree_leaf_elm_bits_slab_get(bits); -#else - *r_szind = rtree_leaf_elm_szind_read(tsdn, rtree, elm, true); - *r_slab = rtree_leaf_elm_slab_read(tsdn, rtree, elm, true); -#endif - return true; - } else { - return false; - } -} JEMALLOC_ALWAYS_INLINE bool rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab) { @@ -510,7 +474,7 @@ rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, static inline void rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, szind_t szind, bool slab) { - assert(!slab || szind < SC_NBINS); + assert(!slab || szind < NBINS); rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true); rtree_leaf_elm_szind_slab_update(tsdn, rtree, elm, szind, slab); @@ -522,7 +486,7 @@ rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true); assert(rtree_leaf_elm_extent_read(tsdn, rtree, elm, false) != NULL); - rtree_leaf_elm_write(tsdn, rtree, elm, NULL, SC_NSIZES, false); + rtree_leaf_elm_write(tsdn, rtree, elm, NULL, NSIZES, false); } #endif /* JEMALLOC_INTERNAL_RTREE_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/rtree_tsd.h b/deps/jemalloc/include/jemalloc/internal/rtree_tsd.h index 562e292..93a7517 100644 --- a/deps/jemalloc/include/jemalloc/internal/rtree_tsd.h +++ b/deps/jemalloc/include/jemalloc/internal/rtree_tsd.h @@ -26,7 +26,7 @@ * Zero initializer required for tsd initialization only. Proper initialization * done via rtree_ctx_data_init(). */ -#define RTREE_CTX_ZERO_INITIALIZER {{{0, 0}}, {{0, 0}}} +#define RTREE_CTX_ZERO_INITIALIZER {{{0}}, {{0}}} typedef struct rtree_leaf_elm_s rtree_leaf_elm_t; diff --git a/deps/jemalloc/include/jemalloc/internal/safety_check.h b/deps/jemalloc/include/jemalloc/internal/safety_check.h deleted file mode 100644 index 53339ac..0000000 --- a/deps/jemalloc/include/jemalloc/internal/safety_check.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_SAFETY_CHECK_H -#define JEMALLOC_INTERNAL_SAFETY_CHECK_H - -void safety_check_fail(const char *format, ...); -/* Can set to NULL for a default. */ -void safety_check_set_abort(void (*abort_fn)()); - -JEMALLOC_ALWAYS_INLINE void -safety_check_set_redzone(void *ptr, size_t usize, size_t bumped_usize) { - assert(usize < bumped_usize); - for (size_t i = usize; i < bumped_usize && i < usize + 32; ++i) { - *((unsigned char *)ptr + i) = 0xBC; - } -} - -JEMALLOC_ALWAYS_INLINE void -safety_check_verify_redzone(const void *ptr, size_t usize, size_t bumped_usize) -{ - for (size_t i = usize; i < bumped_usize && i < usize + 32; ++i) { - if (unlikely(*((unsigned char *)ptr + i) != 0xBC)) { - safety_check_fail("Use after free error\n"); - } - } -} - -#endif /*JEMALLOC_INTERNAL_SAFETY_CHECK_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/sc.h b/deps/jemalloc/include/jemalloc/internal/sc.h deleted file mode 100644 index 9a099d8..0000000 --- a/deps/jemalloc/include/jemalloc/internal/sc.h +++ /dev/null @@ -1,333 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_SC_H -#define JEMALLOC_INTERNAL_SC_H - -#include "jemalloc/internal/jemalloc_internal_types.h" - -/* - * Size class computations: - * - * These are a little tricky; we'll first start by describing how things - * generally work, and then describe some of the details. - * - * Ignore the first few size classes for a moment. We can then split all the - * remaining size classes into groups. The size classes in a group are spaced - * such that they cover allocation request sizes in a power-of-2 range. The - * power of two is called the base of the group, and the size classes in it - * satisfy allocations in the half-open range (base, base * 2]. There are - * SC_NGROUP size classes in each group, equally spaced in the range, so that - * each one covers allocations for base / SC_NGROUP possible allocation sizes. - * We call that value (base / SC_NGROUP) the delta of the group. Each size class - * is delta larger than the one before it (including the initial size class in a - * group, which is delta larger than base, the largest size class in the - * previous group). - * To make the math all work out nicely, we require that SC_NGROUP is a power of - * two, and define it in terms of SC_LG_NGROUP. We'll often talk in terms of - * lg_base and lg_delta. For each of these groups then, we have that - * lg_delta == lg_base - SC_LG_NGROUP. - * The size classes in a group with a given lg_base and lg_delta (which, recall, - * can be computed from lg_base for these groups) are therefore: - * base + 1 * delta - * which covers allocations in (base, base + 1 * delta] - * base + 2 * delta - * which covers allocations in (base + 1 * delta, base + 2 * delta]. - * base + 3 * delta - * which covers allocations in (base + 2 * delta, base + 3 * delta]. - * ... - * base + SC_NGROUP * delta ( == 2 * base) - * which covers allocations in (base + (SC_NGROUP - 1) * delta, 2 * base]. - * (Note that currently SC_NGROUP is always 4, so the "..." is empty in - * practice.) - * Note that the last size class in the group is the next power of two (after - * base), so that we've set up the induction correctly for the next group's - * selection of delta. - * - * Now, let's start considering the first few size classes. Two extra constants - * come into play here: LG_QUANTUM and SC_LG_TINY_MIN. LG_QUANTUM ensures - * correct platform alignment; all objects of size (1 << LG_QUANTUM) or larger - * are at least (1 << LG_QUANTUM) aligned; this can be used to ensure that we - * never return improperly aligned memory, by making (1 << LG_QUANTUM) equal the - * highest required alignment of a platform. For allocation sizes smaller than - * (1 << LG_QUANTUM) though, we can be more relaxed (since we don't support - * platforms with types with alignment larger than their size). To allow such - * allocations (without wasting space unnecessarily), we introduce tiny size - * classes; one per power of two, up until we hit the quantum size. There are - * therefore LG_QUANTUM - SC_LG_TINY_MIN such size classes. - * - * Next, we have a size class of size (1 << LG_QUANTUM). This can't be the - * start of a group in the sense we described above (covering a power of two - * range) since, if we divided into it to pick a value of delta, we'd get a - * delta smaller than (1 << LG_QUANTUM) for sizes >= (1 << LG_QUANTUM), which - * is against the rules. - * - * The first base we can divide by SC_NGROUP while still being at least - * (1 << LG_QUANTUM) is SC_NGROUP * (1 << LG_QUANTUM). We can get there by - * having SC_NGROUP size classes, spaced (1 << LG_QUANTUM) apart. These size - * classes are: - * 1 * (1 << LG_QUANTUM) - * 2 * (1 << LG_QUANTUM) - * 3 * (1 << LG_QUANTUM) - * ... (although, as above, this "..." is empty in practice) - * SC_NGROUP * (1 << LG_QUANTUM). - * - * There are SC_NGROUP of these size classes, so we can regard it as a sort of - * pseudo-group, even though it spans multiple powers of 2, is divided - * differently, and both starts and ends on a power of 2 (as opposed to just - * ending). SC_NGROUP is itself a power of two, so the first group after the - * pseudo-group has the power-of-two base SC_NGROUP * (1 << LG_QUANTUM), for a - * lg_base of LG_QUANTUM + SC_LG_NGROUP. We can divide this base into SC_NGROUP - * sizes without violating our LG_QUANTUM requirements, so we can safely set - * lg_delta = lg_base - SC_LG_GROUP (== LG_QUANTUM). - * - * So, in order, the size classes are: - * - * Tiny size classes: - * - Count: LG_QUANTUM - SC_LG_TINY_MIN. - * - Sizes: - * 1 << SC_LG_TINY_MIN - * 1 << (SC_LG_TINY_MIN + 1) - * 1 << (SC_LG_TINY_MIN + 2) - * ... - * 1 << (LG_QUANTUM - 1) - * - * Initial pseudo-group: - * - Count: SC_NGROUP - * - Sizes: - * 1 * (1 << LG_QUANTUM) - * 2 * (1 << LG_QUANTUM) - * 3 * (1 << LG_QUANTUM) - * ... - * SC_NGROUP * (1 << LG_QUANTUM) - * - * Regular group 0: - * - Count: SC_NGROUP - * - Sizes: - * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP and lg_delta of - * lg_base - SC_LG_NGROUP) - * (1 << lg_base) + 1 * (1 << lg_delta) - * (1 << lg_base) + 2 * (1 << lg_delta) - * (1 << lg_base) + 3 * (1 << lg_delta) - * ... - * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] - * - * Regular group 1: - * - Count: SC_NGROUP - * - Sizes: - * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP + 1 and lg_delta of - * lg_base - SC_LG_NGROUP) - * (1 << lg_base) + 1 * (1 << lg_delta) - * (1 << lg_base) + 2 * (1 << lg_delta) - * (1 << lg_base) + 3 * (1 << lg_delta) - * ... - * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] - * - * ... - * - * Regular group N: - * - Count: SC_NGROUP - * - Sizes: - * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP + N and lg_delta of - * lg_base - SC_LG_NGROUP) - * (1 << lg_base) + 1 * (1 << lg_delta) - * (1 << lg_base) + 2 * (1 << lg_delta) - * (1 << lg_base) + 3 * (1 << lg_delta) - * ... - * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] - * - * - * Representation of metadata: - * To make the math easy, we'll mostly work in lg quantities. We record lg_base, - * lg_delta, and ndelta (i.e. number of deltas above the base) on a - * per-size-class basis, and maintain the invariant that, across all size - * classes, size == (1 << lg_base) + ndelta * (1 << lg_delta). - * - * For regular groups (i.e. those with lg_base >= LG_QUANTUM + SC_LG_NGROUP), - * lg_delta is lg_base - SC_LG_NGROUP, and ndelta goes from 1 to SC_NGROUP. - * - * For the initial tiny size classes (if any), lg_base is lg(size class size). - * lg_delta is lg_base for the first size class, and lg_base - 1 for all - * subsequent ones. ndelta is always 0. - * - * For the pseudo-group, if there are no tiny size classes, then we set - * lg_base == LG_QUANTUM, lg_delta == LG_QUANTUM, and have ndelta range from 0 - * to SC_NGROUP - 1. (Note that delta == base, so base + (SC_NGROUP - 1) * delta - * is just SC_NGROUP * base, or (1 << (SC_LG_NGROUP + LG_QUANTUM)), so we do - * indeed get a power of two that way). If there *are* tiny size classes, then - * the first size class needs to have lg_delta relative to the largest tiny size - * class. We therefore set lg_base == LG_QUANTUM - 1, - * lg_delta == LG_QUANTUM - 1, and ndelta == 1, keeping the rest of the - * pseudo-group the same. - * - * - * Other terminology: - * "Small" size classes mean those that are allocated out of bins, which is the - * same as those that are slab allocated. - * "Large" size classes are those that are not small. The cutoff for counting as - * large is page size * group size. - */ - -/* - * Size class N + (1 << SC_LG_NGROUP) twice the size of size class N. - */ -#define SC_LG_NGROUP 2 -#define SC_LG_TINY_MIN 3 - -#if SC_LG_TINY_MIN == 0 -/* The div module doesn't support division by 1, which this would require. */ -#error "Unsupported LG_TINY_MIN" -#endif - -/* - * The definitions below are all determined by the above settings and system - * characteristics. - */ -#define SC_NGROUP (1ULL << SC_LG_NGROUP) -#define SC_PTR_BITS ((1ULL << LG_SIZEOF_PTR) * 8) -#define SC_NTINY (LG_QUANTUM - SC_LG_TINY_MIN) -#define SC_LG_TINY_MAXCLASS (LG_QUANTUM > SC_LG_TINY_MIN ? LG_QUANTUM - 1 : -1) -#define SC_NPSEUDO SC_NGROUP -#define SC_LG_FIRST_REGULAR_BASE (LG_QUANTUM + SC_LG_NGROUP) -/* - * We cap allocations to be less than 2 ** (ptr_bits - 1), so the highest base - * we need is 2 ** (ptr_bits - 2). (This also means that the last group is 1 - * size class shorter than the others). - * We could probably save some space in arenas by capping this at LG_VADDR size. - */ -#define SC_LG_BASE_MAX (SC_PTR_BITS - 2) -#define SC_NREGULAR (SC_NGROUP * \ - (SC_LG_BASE_MAX - SC_LG_FIRST_REGULAR_BASE + 1) - 1) -#define SC_NSIZES (SC_NTINY + SC_NPSEUDO + SC_NREGULAR) - -/* The number of size classes that are a multiple of the page size. */ -#define SC_NPSIZES ( \ - /* Start with all the size classes. */ \ - SC_NSIZES \ - /* Subtract out those groups with too small a base. */ \ - - (LG_PAGE - 1 - SC_LG_FIRST_REGULAR_BASE) * SC_NGROUP \ - /* And the pseudo-group. */ \ - - SC_NPSEUDO \ - /* And the tiny group. */ \ - - SC_NTINY \ - /* Sizes where ndelta*delta is not a multiple of the page size. */ \ - - (SC_LG_NGROUP * SC_NGROUP)) -/* - * Note that the last line is computed as the sum of the second column in the - * following table: - * lg(base) | count of sizes to exclude - * ------------------------------|----------------------------- - * LG_PAGE - 1 | SC_NGROUP - 1 - * LG_PAGE | SC_NGROUP - 1 - * LG_PAGE + 1 | SC_NGROUP - 2 - * LG_PAGE + 2 | SC_NGROUP - 4 - * ... | ... - * LG_PAGE + (SC_LG_NGROUP - 1) | SC_NGROUP - (SC_NGROUP / 2) - */ - -/* - * We declare a size class is binnable if size < page size * group. Or, in other - * words, lg(size) < lg(page size) + lg(group size). - */ -#define SC_NBINS ( \ - /* Sub-regular size classes. */ \ - SC_NTINY + SC_NPSEUDO \ - /* Groups with lg_regular_min_base <= lg_base <= lg_base_max */ \ - + SC_NGROUP * (LG_PAGE + SC_LG_NGROUP - SC_LG_FIRST_REGULAR_BASE) \ - /* Last SC of the last group hits the bound exactly; exclude it. */ \ - - 1) - -/* - * The size2index_tab lookup table uses uint8_t to encode each bin index, so we - * cannot support more than 256 small size classes. - */ -#if (SC_NBINS > 256) -# error "Too many small size classes" -#endif - -/* The largest size class in the lookup table. */ -#define SC_LOOKUP_MAXCLASS ((size_t)1 << 12) - -/* Internal, only used for the definition of SC_SMALL_MAXCLASS. */ -#define SC_SMALL_MAX_BASE ((size_t)1 << (LG_PAGE + SC_LG_NGROUP - 1)) -#define SC_SMALL_MAX_DELTA ((size_t)1 << (LG_PAGE - 1)) - -/* The largest size class allocated out of a slab. */ -#define SC_SMALL_MAXCLASS (SC_SMALL_MAX_BASE \ - + (SC_NGROUP - 1) * SC_SMALL_MAX_DELTA) - -/* The smallest size class not allocated out of a slab. */ -#define SC_LARGE_MINCLASS ((size_t)1ULL << (LG_PAGE + SC_LG_NGROUP)) -#define SC_LG_LARGE_MINCLASS (LG_PAGE + SC_LG_NGROUP) - -/* Internal; only used for the definition of SC_LARGE_MAXCLASS. */ -#define SC_MAX_BASE ((size_t)1 << (SC_PTR_BITS - 2)) -#define SC_MAX_DELTA ((size_t)1 << (SC_PTR_BITS - 2 - SC_LG_NGROUP)) - -/* The largest size class supported. */ -#define SC_LARGE_MAXCLASS (SC_MAX_BASE + (SC_NGROUP - 1) * SC_MAX_DELTA) - -typedef struct sc_s sc_t; -struct sc_s { - /* Size class index, or -1 if not a valid size class. */ - int index; - /* Lg group base size (no deltas added). */ - int lg_base; - /* Lg delta to previous size class. */ - int lg_delta; - /* Delta multiplier. size == 1<data) / sizeof(size_t)]; \ - buf[sizeof(buf) / sizeof(size_t) - 1] = 0; \ - memcpy(buf, src, sizeof(type)); \ - size_t old_seq = atomic_load_zu(&dst->seq, ATOMIC_RELAXED); \ - atomic_store_zu(&dst->seq, old_seq + 1, ATOMIC_RELAXED); \ - atomic_fence(ATOMIC_RELEASE); \ - for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) { \ - atomic_store_zu(&dst->data[i], buf[i], ATOMIC_RELAXED); \ - } \ - atomic_store_zu(&dst->seq, old_seq + 2, ATOMIC_RELEASE); \ -} \ - \ -/* Returns whether or not the read was consistent. */ \ -static inline bool \ -seq_try_load_##short_type(type *dst, seq_##short_type##_t *src) { \ - size_t buf[sizeof(src->data) / sizeof(size_t)]; \ - size_t seq1 = atomic_load_zu(&src->seq, ATOMIC_ACQUIRE); \ - if (seq1 % 2 != 0) { \ - return false; \ - } \ - for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) { \ - buf[i] = atomic_load_zu(&src->data[i], ATOMIC_RELAXED); \ - } \ - atomic_fence(ATOMIC_ACQUIRE); \ - size_t seq2 = atomic_load_zu(&src->seq, ATOMIC_RELAXED); \ - if (seq1 != seq2) { \ - return false; \ - } \ - memcpy(dst, buf, sizeof(type)); \ - return true; \ -} - -#endif /* JEMALLOC_INTERNAL_SEQ_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/size_classes.sh b/deps/jemalloc/include/jemalloc/internal/size_classes.sh new file mode 100755 index 0000000..998994d --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/size_classes.sh @@ -0,0 +1,361 @@ +#!/bin/sh +# +# Usage: size_classes.sh + +# The following limits are chosen such that they cover all supported platforms. + +# Pointer sizes. +lg_zarr="2 3" + +# Quanta. +lg_qarr=$1 + +# The range of tiny size classes is [2^lg_tmin..2^(lg_q-1)]. +lg_tmin=$2 + +# Maximum lookup size. +lg_kmax=12 + +# Page sizes. +lg_parr=`echo $3 | tr ',' ' '` + +# Size class group size (number of size classes for each size doubling). +lg_g=$4 + +pow2() { + e=$1 + pow2_result=1 + while [ ${e} -gt 0 ] ; do + pow2_result=$((${pow2_result} + ${pow2_result})) + e=$((${e} - 1)) + done +} + +lg() { + x=$1 + lg_result=0 + while [ ${x} -gt 1 ] ; do + lg_result=$((${lg_result} + 1)) + x=$((${x} / 2)) + done +} + +lg_ceil() { + y=$1 + lg ${y}; lg_floor=${lg_result} + pow2 ${lg_floor}; pow2_floor=${pow2_result} + if [ ${pow2_floor} -lt ${y} ] ; then + lg_ceil_result=$((${lg_floor} + 1)) + else + lg_ceil_result=${lg_floor} + fi +} + +reg_size_compute() { + lg_grp=$1 + lg_delta=$2 + ndelta=$3 + + pow2 ${lg_grp}; grp=${pow2_result} + pow2 ${lg_delta}; delta=${pow2_result} + reg_size=$((${grp} + ${delta}*${ndelta})) +} + +slab_size() { + lg_p=$1 + lg_grp=$2 + lg_delta=$3 + ndelta=$4 + + pow2 ${lg_p}; p=${pow2_result} + reg_size_compute ${lg_grp} ${lg_delta} ${ndelta} + + # Compute smallest slab size that is an integer multiple of reg_size. + try_slab_size=${p} + try_nregs=$((${try_slab_size} / ${reg_size})) + perfect=0 + while [ ${perfect} -eq 0 ] ; do + perfect_slab_size=${try_slab_size} + perfect_nregs=${try_nregs} + + try_slab_size=$((${try_slab_size} + ${p})) + try_nregs=$((${try_slab_size} / ${reg_size})) + if [ ${perfect_slab_size} -eq $((${perfect_nregs} * ${reg_size})) ] ; then + perfect=1 + fi + done + + slab_size_pgs=$((${perfect_slab_size} / ${p})) +} + +size_class() { + index=$1 + lg_grp=$2 + lg_delta=$3 + ndelta=$4 + lg_p=$5 + lg_kmax=$6 + + if [ ${lg_delta} -ge ${lg_p} ] ; then + psz="yes" + else + pow2 ${lg_p}; p=${pow2_result} + pow2 ${lg_grp}; grp=${pow2_result} + pow2 ${lg_delta}; delta=${pow2_result} + sz=$((${grp} + ${delta} * ${ndelta})) + npgs=$((${sz} / ${p})) + if [ ${sz} -eq $((${npgs} * ${p})) ] ; then + psz="yes" + else + psz="no" + fi + fi + + lg ${ndelta}; lg_ndelta=${lg_result}; pow2 ${lg_ndelta} + if [ ${pow2_result} -lt ${ndelta} ] ; then + rem="yes" + else + rem="no" + fi + + lg_size=${lg_grp} + if [ $((${lg_delta} + ${lg_ndelta})) -eq ${lg_grp} ] ; then + lg_size=$((${lg_grp} + 1)) + else + lg_size=${lg_grp} + rem="yes" + fi + + if [ ${lg_size} -lt $((${lg_p} + ${lg_g})) ] ; then + bin="yes" + slab_size ${lg_p} ${lg_grp} ${lg_delta} ${ndelta}; pgs=${slab_size_pgs} + else + bin="no" + pgs=0 + fi + if [ ${lg_size} -lt ${lg_kmax} \ + -o ${lg_size} -eq ${lg_kmax} -a ${rem} = "no" ] ; then + lg_delta_lookup=${lg_delta} + else + lg_delta_lookup="no" + fi + printf ' SC(%3d, %6d, %8d, %6d, %3s, %3s, %3d, %2s) \\\n' ${index} ${lg_grp} ${lg_delta} ${ndelta} ${psz} ${bin} ${pgs} ${lg_delta_lookup} + # Defined upon return: + # - psz ("yes" or "no") + # - bin ("yes" or "no") + # - pgs + # - lg_delta_lookup (${lg_delta} or "no") +} + +sep_line() { + echo " \\" +} + +size_classes() { + lg_z=$1 + lg_q=$2 + lg_t=$3 + lg_p=$4 + lg_g=$5 + + pow2 $((${lg_z} + 3)); ptr_bits=${pow2_result} + pow2 ${lg_g}; g=${pow2_result} + + echo "#define SIZE_CLASSES \\" + echo " /* index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup */ \\" + + ntbins=0 + nlbins=0 + lg_tiny_maxclass='"NA"' + nbins=0 + npsizes=0 + + # Tiny size classes. + ndelta=0 + index=0 + lg_grp=${lg_t} + lg_delta=${lg_grp} + while [ ${lg_grp} -lt ${lg_q} ] ; do + size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} + if [ ${lg_delta_lookup} != "no" ] ; then + nlbins=$((${index} + 1)) + fi + if [ ${psz} = "yes" ] ; then + npsizes=$((${npsizes} + 1)) + fi + if [ ${bin} != "no" ] ; then + nbins=$((${index} + 1)) + fi + ntbins=$((${ntbins} + 1)) + lg_tiny_maxclass=${lg_grp} # Final written value is correct. + index=$((${index} + 1)) + lg_delta=${lg_grp} + lg_grp=$((${lg_grp} + 1)) + done + + # First non-tiny group. + if [ ${ntbins} -gt 0 ] ; then + sep_line + # The first size class has an unusual encoding, because the size has to be + # split between grp and delta*ndelta. + lg_grp=$((${lg_grp} - 1)) + ndelta=1 + size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} + index=$((${index} + 1)) + lg_grp=$((${lg_grp} + 1)) + lg_delta=$((${lg_delta} + 1)) + if [ ${psz} = "yes" ] ; then + npsizes=$((${npsizes} + 1)) + fi + fi + while [ ${ndelta} -lt ${g} ] ; do + size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} + index=$((${index} + 1)) + ndelta=$((${ndelta} + 1)) + if [ ${psz} = "yes" ] ; then + npsizes=$((${npsizes} + 1)) + fi + done + + # All remaining groups. + lg_grp=$((${lg_grp} + ${lg_g})) + while [ ${lg_grp} -lt $((${ptr_bits} - 1)) ] ; do + sep_line + ndelta=1 + if [ ${lg_grp} -eq $((${ptr_bits} - 2)) ] ; then + ndelta_limit=$((${g} - 1)) + else + ndelta_limit=${g} + fi + while [ ${ndelta} -le ${ndelta_limit} ] ; do + size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} + if [ ${lg_delta_lookup} != "no" ] ; then + nlbins=$((${index} + 1)) + # Final written value is correct: + lookup_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" + fi + if [ ${psz} = "yes" ] ; then + npsizes=$((${npsizes} + 1)) + fi + if [ ${bin} != "no" ] ; then + nbins=$((${index} + 1)) + # Final written value is correct: + small_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" + if [ ${lg_g} -gt 0 ] ; then + lg_large_minclass=$((${lg_grp} + 1)) + else + lg_large_minclass=$((${lg_grp} + 2)) + fi + fi + # Final written value is correct: + large_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" + index=$((${index} + 1)) + ndelta=$((${ndelta} + 1)) + done + lg_grp=$((${lg_grp} + 1)) + lg_delta=$((${lg_delta} + 1)) + done + echo + nsizes=${index} + lg_ceil ${nsizes}; lg_ceil_nsizes=${lg_ceil_result} + + # Defined upon completion: + # - ntbins + # - nlbins + # - nbins + # - nsizes + # - lg_ceil_nsizes + # - npsizes + # - lg_tiny_maxclass + # - lookup_maxclass + # - small_maxclass + # - lg_large_minclass + # - large_maxclass +} + +cat < 256) +# error "Too many small size classes" +#endif + +#endif /* JEMALLOC_INTERNAL_SIZE_CLASSES_H */ +EOF diff --git a/deps/jemalloc/include/jemalloc/internal/stats.h b/deps/jemalloc/include/jemalloc/internal/stats.h index 3b9e0ea..852e342 100644 --- a/deps/jemalloc/include/jemalloc/internal/stats.h +++ b/deps/jemalloc/include/jemalloc/internal/stats.h @@ -10,8 +10,7 @@ OPTION('a', unmerged, config_stats, false) \ OPTION('b', bins, true, false) \ OPTION('l', large, true, false) \ - OPTION('x', mutex, true, false) \ - OPTION('e', extents, true, false) + OPTION('x', mutex, true, false) enum { #define OPTION(o, v, d, s) stats_print_option_num_##v, diff --git a/deps/jemalloc/include/jemalloc/internal/sz.h b/deps/jemalloc/include/jemalloc/internal/sz.h index 68e558a..9794628 100644 --- a/deps/jemalloc/include/jemalloc/internal/sz.h +++ b/deps/jemalloc/include/jemalloc/internal/sz.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/pages.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/util.h" /* @@ -26,18 +26,18 @@ * sz_pind2sz_tab encodes the same information as could be computed by * sz_pind2sz_compute(). */ -extern size_t sz_pind2sz_tab[SC_NPSIZES + 1]; +extern size_t const sz_pind2sz_tab[NPSIZES+1]; /* * sz_index2size_tab encodes the same information as could be computed (at * unacceptable cost in some code paths) by sz_index2size_compute(). */ -extern size_t sz_index2size_tab[SC_NSIZES]; +extern size_t const sz_index2size_tab[NSIZES]; /* * sz_size2index_tab is a compact lookup table that rounds request sizes up to * size classes. In order to reduce cache footprint, the table is compressed, * and all accesses are via sz_size2index(). */ -extern uint8_t sz_size2index_tab[]; +extern uint8_t const sz_size2index_tab[]; static const size_t sz_large_pad = #ifdef JEMALLOC_CACHE_OBLIVIOUS @@ -47,47 +47,49 @@ static const size_t sz_large_pad = #endif ; -extern void sz_boot(const sc_data_t *sc_data); - JEMALLOC_ALWAYS_INLINE pszind_t sz_psz2ind(size_t psz) { - if (unlikely(psz > SC_LARGE_MAXCLASS)) { - return SC_NPSIZES; + if (unlikely(psz > LARGE_MAXCLASS)) { + return NPSIZES; } - pszind_t x = lg_floor((psz<<1)-1); - pszind_t shift = (x < SC_LG_NGROUP + LG_PAGE) ? - 0 : x - (SC_LG_NGROUP + LG_PAGE); - pszind_t grp = shift << SC_LG_NGROUP; + { + pszind_t x = lg_floor((psz<<1)-1); + pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x - + (LG_SIZE_CLASS_GROUP + LG_PAGE); + pszind_t grp = shift << LG_SIZE_CLASS_GROUP; - pszind_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1) ? - LG_PAGE : x - SC_LG_NGROUP - 1; + pszind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? + LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; - size_t delta_inverse_mask = ZU(-1) << lg_delta; - pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << SC_LG_NGROUP) - 1); + size_t delta_inverse_mask = ZU(-1) << lg_delta; + pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & + ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); - pszind_t ind = grp + mod; - return ind; + pszind_t ind = grp + mod; + return ind; + } } static inline size_t sz_pind2sz_compute(pszind_t pind) { - if (unlikely(pind == SC_NPSIZES)) { - return SC_LARGE_MAXCLASS + PAGE; + if (unlikely(pind == NPSIZES)) { + return LARGE_MAXCLASS + PAGE; } - size_t grp = pind >> SC_LG_NGROUP; - size_t mod = pind & ((ZU(1) << SC_LG_NGROUP) - 1); + { + size_t grp = pind >> LG_SIZE_CLASS_GROUP; + size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); - size_t grp_size_mask = ~((!!grp)-1); - size_t grp_size = ((ZU(1) << (LG_PAGE + (SC_LG_NGROUP-1))) << grp) - & grp_size_mask; + size_t grp_size_mask = ~((!!grp)-1); + size_t grp_size = ((ZU(1) << (LG_PAGE + + (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; - size_t shift = (grp == 0) ? 1 : grp; - size_t lg_delta = shift + (LG_PAGE-1); - size_t mod_size = (mod+1) << lg_delta; + size_t shift = (grp == 0) ? 1 : grp; + size_t lg_delta = shift + (LG_PAGE-1); + size_t mod_size = (mod+1) << lg_delta; - size_t sz = grp_size + mod_size; - return sz; + size_t sz = grp_size + mod_size; + return sz; + } } static inline size_t @@ -99,70 +101,70 @@ sz_pind2sz_lookup(pszind_t pind) { static inline size_t sz_pind2sz(pszind_t pind) { - assert(pind < SC_NPSIZES + 1); + assert(pind < NPSIZES+1); return sz_pind2sz_lookup(pind); } static inline size_t sz_psz2u(size_t psz) { - if (unlikely(psz > SC_LARGE_MAXCLASS)) { - return SC_LARGE_MAXCLASS + PAGE; + if (unlikely(psz > LARGE_MAXCLASS)) { + return LARGE_MAXCLASS + PAGE; + } + { + size_t x = lg_floor((psz<<1)-1); + size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? + LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; + size_t delta = ZU(1) << lg_delta; + size_t delta_mask = delta - 1; + size_t usize = (psz + delta_mask) & ~delta_mask; + return usize; } - size_t x = lg_floor((psz<<1)-1); - size_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1) ? - LG_PAGE : x - SC_LG_NGROUP - 1; - size_t delta = ZU(1) << lg_delta; - size_t delta_mask = delta - 1; - size_t usize = (psz + delta_mask) & ~delta_mask; - return usize; } static inline szind_t sz_size2index_compute(size_t size) { - if (unlikely(size > SC_LARGE_MAXCLASS)) { - return SC_NSIZES; + if (unlikely(size > LARGE_MAXCLASS)) { + return NSIZES; } - - if (size == 0) { - return 0; - } -#if (SC_NTINY != 0) - if (size <= (ZU(1) << SC_LG_TINY_MAXCLASS)) { - szind_t lg_tmin = SC_LG_TINY_MAXCLASS - SC_NTINY + 1; +#if (NTBINS != 0) + if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { + szind_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; szind_t lg_ceil = lg_floor(pow2_ceil_zu(size)); return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); } #endif { szind_t x = lg_floor((size<<1)-1); - szind_t shift = (x < SC_LG_NGROUP + LG_QUANTUM) ? 0 : - x - (SC_LG_NGROUP + LG_QUANTUM); - szind_t grp = shift << SC_LG_NGROUP; + szind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : + x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); + szind_t grp = shift << LG_SIZE_CLASS_GROUP; - szind_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - SC_LG_NGROUP - 1; + szind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; size_t delta_inverse_mask = ZU(-1) << lg_delta; szind_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << SC_LG_NGROUP) - 1); + ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); - szind_t index = SC_NTINY + grp + mod; + szind_t index = NTBINS + grp + mod; return index; } } JEMALLOC_ALWAYS_INLINE szind_t sz_size2index_lookup(size_t size) { - assert(size <= SC_LOOKUP_MAXCLASS); - szind_t ret = (sz_size2index_tab[(size + (ZU(1) << SC_LG_TINY_MIN) - 1) - >> SC_LG_TINY_MIN]); - assert(ret == sz_size2index_compute(size)); - return ret; + assert(size <= LOOKUP_MAXCLASS); + { + szind_t ret = (sz_size2index_tab[(size-1) >> LG_TINY_MIN]); + assert(ret == sz_size2index_compute(size)); + return ret; + } } JEMALLOC_ALWAYS_INLINE szind_t sz_size2index(size_t size) { - if (likely(size <= SC_LOOKUP_MAXCLASS)) { + assert(size > 0); + if (likely(size <= LOOKUP_MAXCLASS)) { return sz_size2index_lookup(size); } return sz_size2index_compute(size); @@ -170,20 +172,20 @@ sz_size2index(size_t size) { static inline size_t sz_index2size_compute(szind_t index) { -#if (SC_NTINY > 0) - if (index < SC_NTINY) { - return (ZU(1) << (SC_LG_TINY_MAXCLASS - SC_NTINY + 1 + index)); +#if (NTBINS > 0) + if (index < NTBINS) { + return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + index)); } #endif { - size_t reduced_index = index - SC_NTINY; - size_t grp = reduced_index >> SC_LG_NGROUP; - size_t mod = reduced_index & ((ZU(1) << SC_LG_NGROUP) - + size_t reduced_index = index - NTBINS; + size_t grp = reduced_index >> LG_SIZE_CLASS_GROUP; + size_t mod = reduced_index & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); size_t grp_size_mask = ~((!!grp)-1); size_t grp_size = ((ZU(1) << (LG_QUANTUM + - (SC_LG_NGROUP-1))) << grp) & grp_size_mask; + (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; size_t shift = (grp == 0) ? 1 : grp; size_t lg_delta = shift + (LG_QUANTUM-1); @@ -203,22 +205,18 @@ sz_index2size_lookup(szind_t index) { JEMALLOC_ALWAYS_INLINE size_t sz_index2size(szind_t index) { - assert(index < SC_NSIZES); + assert(index < NSIZES); return sz_index2size_lookup(index); } JEMALLOC_ALWAYS_INLINE size_t sz_s2u_compute(size_t size) { - if (unlikely(size > SC_LARGE_MAXCLASS)) { + if (unlikely(size > LARGE_MAXCLASS)) { return 0; } - - if (size == 0) { - size++; - } -#if (SC_NTINY > 0) - if (size <= (ZU(1) << SC_LG_TINY_MAXCLASS)) { - size_t lg_tmin = SC_LG_TINY_MAXCLASS - SC_NTINY + 1; +#if (NTBINS > 0) + if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { + size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; size_t lg_ceil = lg_floor(pow2_ceil_zu(size)); return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : (ZU(1) << lg_ceil)); @@ -226,8 +224,8 @@ sz_s2u_compute(size_t size) { #endif { size_t x = lg_floor((size<<1)-1); - size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - SC_LG_NGROUP - 1; + size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; size_t delta = ZU(1) << lg_delta; size_t delta_mask = delta - 1; size_t usize = (size + delta_mask) & ~delta_mask; @@ -249,7 +247,8 @@ sz_s2u_lookup(size_t size) { */ JEMALLOC_ALWAYS_INLINE size_t sz_s2u(size_t size) { - if (likely(size <= SC_LOOKUP_MAXCLASS)) { + assert(size > 0); + if (likely(size <= LOOKUP_MAXCLASS)) { return sz_s2u_lookup(size); } return sz_s2u_compute(size); @@ -266,7 +265,7 @@ sz_sa2u(size_t size, size_t alignment) { assert(alignment != 0 && ((alignment - 1) & alignment) == 0); /* Try for a small size class. */ - if (size <= SC_SMALL_MAXCLASS && alignment < PAGE) { + if (size <= SMALL_MAXCLASS && alignment < PAGE) { /* * Round size up to the nearest multiple of alignment. * @@ -282,20 +281,20 @@ sz_sa2u(size_t size, size_t alignment) { * 192 | 11000000 | 64 */ usize = sz_s2u(ALIGNMENT_CEILING(size, alignment)); - if (usize < SC_LARGE_MINCLASS) { + if (usize < LARGE_MINCLASS) { return usize; } } /* Large size class. Beware of overflow. */ - if (unlikely(alignment > SC_LARGE_MAXCLASS)) { + if (unlikely(alignment > LARGE_MAXCLASS)) { return 0; } /* Make sure result is a large size class. */ - if (size <= SC_LARGE_MINCLASS) { - usize = SC_LARGE_MINCLASS; + if (size <= LARGE_MINCLASS) { + usize = LARGE_MINCLASS; } else { usize = sz_s2u(size); if (usize < size) { diff --git a/deps/jemalloc/include/jemalloc/internal/tcache_externs.h b/deps/jemalloc/include/jemalloc/internal/tcache_externs.h index d63eafd..790367b 100644 --- a/deps/jemalloc/include/jemalloc/internal/tcache_externs.h +++ b/deps/jemalloc/include/jemalloc/internal/tcache_externs.h @@ -1,13 +1,15 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H #define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H +#include "jemalloc/internal/size_classes.h" + extern bool opt_tcache; extern ssize_t opt_lg_tcache_max; extern cache_bin_info_t *tcache_bin_info; /* - * Number of tcache bins. There are SC_NBINS small-object bins, plus 0 or more + * Number of tcache bins. There are NBINS small-object bins, plus 0 or more * large-object bins. */ extern unsigned nhbins; diff --git a/deps/jemalloc/include/jemalloc/internal/tcache_inlines.h b/deps/jemalloc/include/jemalloc/internal/tcache_inlines.h index 5eca20e..0f6ab8c 100644 --- a/deps/jemalloc/include/jemalloc/internal/tcache_inlines.h +++ b/deps/jemalloc/include/jemalloc/internal/tcache_inlines.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/bin.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/util.h" @@ -40,13 +40,13 @@ tcache_event(tsd_t *tsd, tcache_t *tcache) { JEMALLOC_ALWAYS_INLINE void * tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, - size_t size, szind_t binind, bool zero, bool slow_path) { + UNUSED size_t size, szind_t binind, bool zero, bool slow_path) { void *ret; cache_bin_t *bin; bool tcache_success; size_t usize JEMALLOC_CC_SILENCE_INIT(0); - assert(binind < SC_NBINS); + assert(binind < NBINS); bin = tcache_small_bin_get(tcache, binind); ret = cache_bin_alloc_easy(bin, &tcache_success); assert(tcache_success == (ret != NULL)); @@ -107,7 +107,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, cache_bin_t *bin; bool tcache_success; - assert(binind >= SC_NBINS &&binind < nhbins); + assert(binind >= NBINS &&binind < nhbins); bin = tcache_large_bin_get(tcache, binind); ret = cache_bin_alloc_easy(bin, &tcache_success); assert(tcache_success == (ret != NULL)); @@ -166,8 +166,7 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, cache_bin_t *bin; cache_bin_info_t *bin_info; - assert(tcache_salloc(tsd_tsdn(tsd), ptr) - <= SC_SMALL_MAXCLASS); + assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SMALL_MAXCLASS); if (slow_path && config_fill && unlikely(opt_junk_free)) { arena_dalloc_junk_small(ptr, &bin_infos[binind]); @@ -175,12 +174,13 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bin = tcache_small_bin_get(tcache, binind); bin_info = &tcache_bin_info[binind]; - if (unlikely(!cache_bin_dalloc_easy(bin, bin_info, ptr))) { + if (unlikely(bin->ncached == bin_info->ncached_max)) { tcache_bin_flush_small(tsd, tcache, bin, binind, (bin_info->ncached_max >> 1)); - bool ret = cache_bin_dalloc_easy(bin, bin_info, ptr); - assert(ret); } + assert(bin->ncached < bin_info->ncached_max); + bin->ncached++; + *(bin->avail - bin->ncached) = ptr; tcache_event(tsd, tcache); } @@ -191,8 +191,7 @@ tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, cache_bin_t *bin; cache_bin_info_t *bin_info; - assert(tcache_salloc(tsd_tsdn(tsd), ptr) - > SC_SMALL_MAXCLASS); + assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SMALL_MAXCLASS); assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_maxclass); if (slow_path && config_fill && unlikely(opt_junk_free)) { @@ -216,9 +215,6 @@ JEMALLOC_ALWAYS_INLINE tcache_t * tcaches_get(tsd_t *tsd, unsigned ind) { tcaches_t *elm = &tcaches[ind]; if (unlikely(elm->tcache == NULL)) { - malloc_printf(": invalid tcache id (%u).\n", ind); - abort(); - } else if (unlikely(elm->tcache == TCACHES_ELM_NEED_REINIT)) { elm->tcache = tcache_create_explicit(tsd); } return elm->tcache; diff --git a/deps/jemalloc/include/jemalloc/internal/tcache_structs.h b/deps/jemalloc/include/jemalloc/internal/tcache_structs.h index 172ef90..07b7387 100644 --- a/deps/jemalloc/include/jemalloc/internal/tcache_structs.h +++ b/deps/jemalloc/include/jemalloc/internal/tcache_structs.h @@ -1,14 +1,10 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_STRUCTS_H #define JEMALLOC_INTERNAL_TCACHE_STRUCTS_H -#include "jemalloc/internal/cache_bin.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/cache_bin.h" #include "jemalloc/internal/ticker.h" -#include "jemalloc/internal/tsd_types.h" - -/* Various uses of this struct need it to be a named type. */ -typedef ql_elm(tsd_t) tsd_link_t; struct tcache_s { /* @@ -25,7 +21,7 @@ struct tcache_s { * During tcache initialization, the avail pointer in each element of * tbins is initialized to point to the proper offset within this array. */ - cache_bin_t bins_small[SC_NBINS]; + cache_bin_t bins_small[NBINS]; /* * This data is less hot; we can be a little less careful with our @@ -33,11 +29,6 @@ struct tcache_s { */ /* Lets us track all the tcaches in an arena. */ ql_elm(tcache_t) link; - - /* Logically scoped to tsd, but put here for cache layout reasons. */ - ql_elm(tsd_t) tsd_link; - bool in_hook; - /* * The descriptor lets the arena find our cache bins without seeing the * tcache definition. This enables arenas to aggregate stats across @@ -50,13 +41,13 @@ struct tcache_s { /* Next bin to GC. */ szind_t next_gc_bin; /* For small bins, fill (ncached_max >> lg_fill_div). */ - uint8_t lg_fill_div[SC_NBINS]; + uint8_t lg_fill_div[NBINS]; /* * We put the cache bins for large size classes at the end of the * struct, since some of them might not get used. This might end up * letting us avoid touching an extra page if we don't have to. */ - cache_bin_t bins_large[SC_NSIZES-SC_NBINS]; + cache_bin_t bins_large[NSIZES-NBINS]; }; /* Linkage for list of available (previously used) explicit tcache IDs. */ diff --git a/deps/jemalloc/include/jemalloc/internal/tcache_types.h b/deps/jemalloc/include/jemalloc/internal/tcache_types.h index dce6938..e49bc9d 100644 --- a/deps/jemalloc/include/jemalloc/internal/tcache_types.h +++ b/deps/jemalloc/include/jemalloc/internal/tcache_types.h @@ -1,7 +1,7 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_TYPES_H #define JEMALLOC_INTERNAL_TCACHE_TYPES_H -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" typedef struct tcache_s tcache_t; typedef struct tcaches_s tcaches_t; @@ -45,7 +45,7 @@ typedef struct tcaches_s tcaches_t; /* Number of tcache allocation/deallocation events between incremental GCs. */ #define TCACHE_GC_INCR \ - ((TCACHE_GC_SWEEP / SC_NBINS) + ((TCACHE_GC_SWEEP / SC_NBINS == 0) ? 0 : 1)) + ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1)) /* Used in TSD static initializer only. Real init in tcache_data_init(). */ #define TCACHE_ZERO_INITIALIZER {0} @@ -53,7 +53,4 @@ typedef struct tcaches_s tcaches_t; /* Used in TSD static initializer only. Will be initialized to opt_tcache. */ #define TCACHE_ENABLED_ZERO_INITIALIZER false -/* Used for explicit tcache only. Means flushed but not destroyed. */ -#define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1) - #endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/test_hooks.h b/deps/jemalloc/include/jemalloc/internal/test_hooks.h deleted file mode 100644 index a6351e5..0000000 --- a/deps/jemalloc/include/jemalloc/internal/test_hooks.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_TEST_HOOKS_H -#define JEMALLOC_INTERNAL_TEST_HOOKS_H - -extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(); -extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(); - -#define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) - -#define open JEMALLOC_HOOK(open, test_hooks_libc_hook) -#define read JEMALLOC_HOOK(read, test_hooks_libc_hook) -#define write JEMALLOC_HOOK(write, test_hooks_libc_hook) -#define readlink JEMALLOC_HOOK(readlink, test_hooks_libc_hook) -#define close JEMALLOC_HOOK(close, test_hooks_libc_hook) -#define creat JEMALLOC_HOOK(creat, test_hooks_libc_hook) -#define secure_getenv JEMALLOC_HOOK(secure_getenv, test_hooks_libc_hook) -/* Note that this is undef'd and re-define'd in src/prof.c. */ -#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) - -#endif /* JEMALLOC_INTERNAL_TEST_HOOKS_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/ticker.h b/deps/jemalloc/include/jemalloc/internal/ticker.h index 52d0db4..4b36047 100644 --- a/deps/jemalloc/include/jemalloc/internal/ticker.h +++ b/deps/jemalloc/include/jemalloc/internal/ticker.h @@ -75,17 +75,4 @@ ticker_tick(ticker_t *ticker) { return ticker_ticks(ticker, 1); } -/* - * Try to tick. If ticker would fire, return true, but rely on - * slowpath to reset ticker. - */ -static inline bool -ticker_trytick(ticker_t *ticker) { - --ticker->tick; - if (unlikely(ticker->tick < 0)) { - return true; - } - return false; -} - #endif /* JEMALLOC_INTERNAL_TICKER_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/tsd.h b/deps/jemalloc/include/jemalloc/internal/tsd.h index 9ba2600..0b9841a 100644 --- a/deps/jemalloc/include/jemalloc/internal/tsd.h +++ b/deps/jemalloc/include/jemalloc/internal/tsd.h @@ -3,7 +3,6 @@ #include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/assert.h" -#include "jemalloc/internal/bin_types.h" #include "jemalloc/internal/jemalloc_internal_externs.h" #include "jemalloc/internal/prof_types.h" #include "jemalloc/internal/ql.h" @@ -69,19 +68,17 @@ typedef void (*test_callback_t)(int *); O(offset_state, uint64_t, uint64_t) \ O(thread_allocated, uint64_t, uint64_t) \ O(thread_deallocated, uint64_t, uint64_t) \ - O(bytes_until_sample, int64_t, int64_t) \ O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \ O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) \ O(iarena, arena_t *, arena_t *) \ O(arena, arena_t *, arena_t *) \ O(arenas_tdata, arena_tdata_t *, arena_tdata_t *)\ - O(binshards, tsd_binshards_t, tsd_binshards_t)\ O(tcache, tcache_t, tcache_t) \ O(witness_tsd, witness_tsd_t, witness_tsdn_t) \ MALLOC_TEST_TSD #define TSD_INITIALIZER { \ - ATOMIC_INIT(tsd_state_uninitialized), \ + tsd_state_uninitialized, \ TCACHE_ENABLED_ZERO_INITIALIZER, \ false, \ 0, \ @@ -89,97 +86,29 @@ typedef void (*test_callback_t)(int *); 0, \ 0, \ 0, \ - 0, \ NULL, \ RTREE_CTX_ZERO_INITIALIZER, \ NULL, \ NULL, \ NULL, \ - TSD_BINSHARDS_ZERO_INITIALIZER, \ TCACHE_ZERO_INITIALIZER, \ WITNESS_TSD_INITIALIZER \ MALLOC_TEST_TSD_INITIALIZER \ } -void *malloc_tsd_malloc(size_t size); -void malloc_tsd_dalloc(void *wrapper); -void malloc_tsd_cleanup_register(bool (*f)(void)); -tsd_t *malloc_tsd_boot0(void); -void malloc_tsd_boot1(void); -void tsd_cleanup(void *arg); -tsd_t *tsd_fetch_slow(tsd_t *tsd, bool internal); -void tsd_state_set(tsd_t *tsd, uint8_t new_state); -void tsd_slow_update(tsd_t *tsd); -void tsd_prefork(tsd_t *tsd); -void tsd_postfork_parent(tsd_t *tsd); -void tsd_postfork_child(tsd_t *tsd); - -/* - * Call ..._inc when your module wants to take all threads down the slow paths, - * and ..._dec when it no longer needs to. - */ -void tsd_global_slow_inc(tsdn_t *tsdn); -void tsd_global_slow_dec(tsdn_t *tsdn); -bool tsd_global_slow(); - enum { - /* Common case --> jnz. */ - tsd_state_nominal = 0, - /* Initialized but on slow path. */ - tsd_state_nominal_slow = 1, - /* - * Some thread has changed global state in such a way that all nominal - * threads need to recompute their fast / slow status the next time they - * get a chance. - * - * Any thread can change another thread's status *to* recompute, but - * threads are the only ones who can change their status *from* - * recompute. - */ - tsd_state_nominal_recompute = 2, - /* - * The above nominal states should be lower values. We use - * tsd_nominal_max to separate nominal states from threads in the - * process of being born / dying. - */ - tsd_state_nominal_max = 2, - - /* - * A thread might free() during its death as its only allocator action; - * in such scenarios, we need tsd, but set up in such a way that no - * cleanup is necessary. - */ - tsd_state_minimal_initialized = 3, - /* States during which we know we're in thread death. */ - tsd_state_purgatory = 4, - tsd_state_reincarnated = 5, - /* - * What it says on the tin; tsd that hasn't been initialized. Note - * that even when the tsd struct lives in TLS, when need to keep track - * of stuff like whether or not our pthread destructors have been - * scheduled, so this really truly is different than the nominal state. - */ - tsd_state_uninitialized = 6 + tsd_state_nominal = 0, /* Common case --> jnz. */ + tsd_state_nominal_slow = 1, /* Initialized but on slow path. */ + /* the above 2 nominal states should be lower values. */ + tsd_state_nominal_max = 1, /* used for comparison only. */ + tsd_state_minimal_initialized = 2, + tsd_state_purgatory = 3, + tsd_state_reincarnated = 4, + tsd_state_uninitialized = 5 }; -/* - * Some TSD accesses can only be done in a nominal state. To enforce this, we - * wrap TSD member access in a function that asserts on TSD state, and mangle - * field names to prevent touching them accidentally. - */ -#define TSD_MANGLE(n) cant_access_tsd_items_directly_use_a_getter_or_setter_##n - -#ifdef JEMALLOC_U8_ATOMICS -# define tsd_state_t atomic_u8_t -# define tsd_atomic_load atomic_load_u8 -# define tsd_atomic_store atomic_store_u8 -# define tsd_atomic_exchange atomic_exchange_u8 -#else -# define tsd_state_t atomic_u32_t -# define tsd_atomic_load atomic_load_u32 -# define tsd_atomic_store atomic_store_u32 -# define tsd_atomic_exchange atomic_exchange_u32 -#endif +/* Manually limit tsd_state_t to a single byte. */ +typedef uint8_t tsd_state_t; /* The actual tsd. */ struct tsd_s { @@ -188,29 +117,13 @@ struct tsd_s { * module. Access any thread-local state through the getters and * setters below. */ - - /* - * We manually limit the state to just a single byte. Unless the 8-bit - * atomics are unavailable (which is rare). - */ - tsd_state_t state; + tsd_state_t state; #define O(n, t, nt) \ - t TSD_MANGLE(n); + t use_a_getter_or_setter_instead_##n; MALLOC_TSD #undef O }; -JEMALLOC_ALWAYS_INLINE uint8_t -tsd_state_get(tsd_t *tsd) { - /* - * This should be atomic. Unfortunately, compilers right now can't tell - * that this can be done as a memory comparison, and forces a load into - * a register that hurts fast-path performance. - */ - /* return atomic_load_u8(&tsd->state, ATOMIC_RELAXED); */ - return *(uint8_t *)&tsd->state; -} - /* * Wrapper around tsd_t that makes it possible to avoid implicit conversion * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be @@ -237,6 +150,15 @@ tsdn_tsd(tsdn_t *tsdn) { return &tsdn->tsd; } +void *malloc_tsd_malloc(size_t size); +void malloc_tsd_dalloc(void *wrapper); +void malloc_tsd_cleanup_register(bool (*f)(void)); +tsd_t *malloc_tsd_boot0(void); +void malloc_tsd_boot1(void); +void tsd_cleanup(void *arg); +tsd_t *tsd_fetch_slow(tsd_t *tsd, bool internal); +void tsd_slow_update(tsd_t *tsd); + /* * We put the platform-specific data declarations and inlines into their own * header files to avoid cluttering this file. They define tsd_boot0, @@ -260,7 +182,7 @@ tsdn_tsd(tsdn_t *tsdn) { #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get_unsafe(tsd_t *tsd) { \ - return &tsd->TSD_MANGLE(n); \ + return &tsd->use_a_getter_or_setter_instead_##n; \ } MALLOC_TSD #undef O @@ -269,16 +191,10 @@ MALLOC_TSD #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get(tsd_t *tsd) { \ - /* \ - * Because the state might change asynchronously if it's \ - * nominal, we need to make sure that we only read it once. \ - */ \ - uint8_t state = tsd_state_get(tsd); \ - assert(state == tsd_state_nominal || \ - state == tsd_state_nominal_slow || \ - state == tsd_state_nominal_recompute || \ - state == tsd_state_reincarnated || \ - state == tsd_state_minimal_initialized); \ + assert(tsd->state == tsd_state_nominal || \ + tsd->state == tsd_state_nominal_slow || \ + tsd->state == tsd_state_reincarnated || \ + tsd->state == tsd_state_minimal_initialized); \ return tsd_##n##p_get_unsafe(tsd); \ } MALLOC_TSD @@ -313,8 +229,8 @@ MALLOC_TSD #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE void \ tsd_##n##_set(tsd_t *tsd, t val) { \ - assert(tsd_state_get(tsd) != tsd_state_reincarnated && \ - tsd_state_get(tsd) != tsd_state_minimal_initialized); \ + assert(tsd->state != tsd_state_reincarnated && \ + tsd->state != tsd_state_minimal_initialized); \ *tsd_##n##p_get(tsd) = val; \ } MALLOC_TSD @@ -322,18 +238,13 @@ MALLOC_TSD JEMALLOC_ALWAYS_INLINE void tsd_assert_fast(tsd_t *tsd) { - /* - * Note that our fastness assertion does *not* include global slowness - * counters; it's not in general possible to ensure that they won't - * change asynchronously from underneath us. - */ assert(!malloc_slow && tsd_tcache_enabled_get(tsd) && tsd_reentrancy_level_get(tsd) == 0); } JEMALLOC_ALWAYS_INLINE bool tsd_fast(tsd_t *tsd) { - bool fast = (tsd_state_get(tsd) == tsd_state_nominal); + bool fast = (tsd->state == tsd_state_nominal); if (fast) { tsd_assert_fast(tsd); } @@ -350,7 +261,7 @@ tsd_fetch_impl(bool init, bool minimal) { } assert(tsd != NULL); - if (unlikely(tsd_state_get(tsd) != tsd_state_nominal)) { + if (unlikely(tsd->state != tsd_state_nominal)) { return tsd_fetch_slow(tsd, minimal); } assert(tsd_fast(tsd)); @@ -370,7 +281,7 @@ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_internal_fetch(void) { tsd_t *tsd = tsd_fetch_min(); /* Use reincarnated state to prevent full initialization. */ - tsd_state_set(tsd, tsd_state_reincarnated); + tsd->state = tsd_state_reincarnated; return tsd; } @@ -382,7 +293,7 @@ tsd_fetch(void) { static inline bool tsd_nominal(tsd_t *tsd) { - return (tsd_state_get(tsd) <= tsd_state_nominal_max); + return (tsd->state <= tsd_state_nominal_max); } JEMALLOC_ALWAYS_INLINE tsdn_t * diff --git a/deps/jemalloc/include/jemalloc/internal/tsd_generic.h b/deps/jemalloc/include/jemalloc/internal/tsd_generic.h index cf73c0c..1e52ef7 100644 --- a/deps/jemalloc/include/jemalloc/internal/tsd_generic.h +++ b/deps/jemalloc/include/jemalloc/internal/tsd_generic.h @@ -77,10 +77,7 @@ tsd_wrapper_get(bool init) { abort(); } else { wrapper->initialized = false; - JEMALLOC_DIAGNOSTIC_PUSH - JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS tsd_t initializer = TSD_INITIALIZER; - JEMALLOC_DIAGNOSTIC_POP wrapper->val = initializer; } tsd_wrapper_set(wrapper); @@ -110,10 +107,7 @@ tsd_boot1(void) { tsd_boot_wrapper.initialized = false; tsd_cleanup(&tsd_boot_wrapper.val); wrapper->initialized = false; - JEMALLOC_DIAGNOSTIC_PUSH - JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS tsd_t initializer = TSD_INITIALIZER; - JEMALLOC_DIAGNOSTIC_POP wrapper->val = initializer; tsd_wrapper_set(wrapper); } diff --git a/deps/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h b/deps/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h index 65852d5..beb467a 100644 --- a/deps/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h +++ b/deps/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h @@ -3,10 +3,8 @@ #endif #define JEMALLOC_INTERNAL_TSD_MALLOC_THREAD_CLEANUP_H -#define JEMALLOC_TSD_TYPE_ATTR(type) __thread type JEMALLOC_TLS_MODEL - -extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; -extern JEMALLOC_TSD_TYPE_ATTR(bool) tsd_initialized; +extern __thread tsd_t tsd_tls; +extern __thread bool tsd_initialized; extern bool tsd_booted; /* Initialization/cleanup. */ @@ -49,6 +47,7 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { + assert(tsd_booted); return &tsd_tls; } JEMALLOC_ALWAYS_INLINE void diff --git a/deps/jemalloc/include/jemalloc/internal/tsd_tls.h b/deps/jemalloc/include/jemalloc/internal/tsd_tls.h index 7d6c805..0de64b7 100644 --- a/deps/jemalloc/include/jemalloc/internal/tsd_tls.h +++ b/deps/jemalloc/include/jemalloc/internal/tsd_tls.h @@ -3,9 +3,7 @@ #endif #define JEMALLOC_INTERNAL_TSD_TLS_H -#define JEMALLOC_TSD_TYPE_ATTR(type) __thread type JEMALLOC_TLS_MODEL - -extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; +extern __thread tsd_t tsd_tls; extern pthread_key_t tsd_tsd; extern bool tsd_booted; @@ -41,7 +39,8 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * -tsd_get(bool init) { +tsd_get(UNUSED bool init) { + assert(tsd_booted); return &tsd_tls; } diff --git a/deps/jemalloc/include/jemalloc/internal/witness.h b/deps/jemalloc/include/jemalloc/internal/witness.h index fff9e98..7ace8ae 100644 --- a/deps/jemalloc/include/jemalloc/internal/witness.h +++ b/deps/jemalloc/include/jemalloc/internal/witness.h @@ -27,9 +27,9 @@ #define WITNESS_RANK_PROF_BT2GCTX 6U #define WITNESS_RANK_PROF_TDATAS 7U #define WITNESS_RANK_PROF_TDATA 8U -#define WITNESS_RANK_PROF_LOG 9U -#define WITNESS_RANK_PROF_GCTX 10U -#define WITNESS_RANK_BACKGROUND_THREAD 11U +#define WITNESS_RANK_PROF_GCTX 9U + +#define WITNESS_RANK_BACKGROUND_THREAD 10U /* * Used as an argument to witness_assert_depth_to_rank() in order to validate @@ -37,19 +37,18 @@ * witness_assert_depth_to_rank() is inclusive rather than exclusive, this * definition can have the same value as the minimally ranked core lock. */ -#define WITNESS_RANK_CORE 12U +#define WITNESS_RANK_CORE 11U -#define WITNESS_RANK_DECAY 12U -#define WITNESS_RANK_TCACHE_QL 13U -#define WITNESS_RANK_EXTENT_GROW 14U -#define WITNESS_RANK_EXTENTS 15U -#define WITNESS_RANK_EXTENT_AVAIL 16U +#define WITNESS_RANK_DECAY 11U +#define WITNESS_RANK_TCACHE_QL 12U +#define WITNESS_RANK_EXTENT_GROW 13U +#define WITNESS_RANK_EXTENTS 14U +#define WITNESS_RANK_EXTENT_AVAIL 15U -#define WITNESS_RANK_EXTENT_POOL 17U -#define WITNESS_RANK_RTREE 18U -#define WITNESS_RANK_BASE 19U -#define WITNESS_RANK_ARENA_LARGE 20U -#define WITNESS_RANK_HOOK 21U +#define WITNESS_RANK_EXTENT_POOL 16U +#define WITNESS_RANK_RTREE 17U +#define WITNESS_RANK_BASE 18U +#define WITNESS_RANK_ARENA_LARGE 19U #define WITNESS_RANK_LEAF 0xffffffffU #define WITNESS_RANK_BIN WITNESS_RANK_LEAF diff --git a/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in b/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in index 11c3918..6d89435 100644 --- a/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in +++ b/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in @@ -4,9 +4,6 @@ /* Defined if alloc_size attribute is supported. */ #undef JEMALLOC_HAVE_ATTR_ALLOC_SIZE -/* Defined if format_arg(...) attribute is supported. */ -#undef JEMALLOC_HAVE_ATTR_FORMAT_ARG - /* Defined if format(gnu_printf, ...) attribute is supported. */ #undef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF diff --git a/deps/jemalloc/include/jemalloc/jemalloc_macros.h.in b/deps/jemalloc/include/jemalloc/jemalloc_macros.h.in index 59e2955..daf9e57 100644 --- a/deps/jemalloc/include/jemalloc/jemalloc_macros.h.in +++ b/deps/jemalloc/include/jemalloc/jemalloc_macros.h.in @@ -10,7 +10,6 @@ #define JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@ #define JEMALLOC_VERSION_NREV @jemalloc_version_nrev@ #define JEMALLOC_VERSION_GID "@jemalloc_version_gid@" -#define JEMALLOC_VERSION_GID_IDENT @jemalloc_version_gid@ #define MALLOCX_LG_ALIGN(la) ((int)(la)) #if LG_SIZEOF_PTR == 2 @@ -69,7 +68,6 @@ # define JEMALLOC_EXPORT __declspec(dllimport) # endif # endif -# define JEMALLOC_FORMAT_ARG(i) # define JEMALLOC_FORMAT_PRINTF(s, i) # define JEMALLOC_NOINLINE __declspec(noinline) # ifdef __cplusplus @@ -97,11 +95,6 @@ # ifndef JEMALLOC_EXPORT # define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) # endif -# ifdef JEMALLOC_HAVE_ATTR_FORMAT_ARG -# define JEMALLOC_FORMAT_ARG(i) JEMALLOC_ATTR(__format_arg__(3)) -# else -# define JEMALLOC_FORMAT_ARG(i) -# endif # ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF # define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i)) # elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF) @@ -127,3 +120,7 @@ # define JEMALLOC_RESTRICT_RETURN # define JEMALLOC_ALLOCATOR #endif + +/* This version of Jemalloc, modified for Redis, has the je_get_defrag_hint() + * function. */ +#define JEMALLOC_FRAG_HINT diff --git a/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj index 228e8be..f7b175b 100644 --- a/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj +++ b/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj @@ -47,7 +47,7 @@ - + @@ -59,14 +59,12 @@ - - {8D6BB292-9E1C-413D-9F98-4864BDC1514A} @@ -347,4 +345,4 @@ - + \ No newline at end of file diff --git a/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters index d839515..11cfcd0 100644 --- a/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters +++ b/deps/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters @@ -37,7 +37,7 @@ Source Files - + Source Files @@ -70,9 +70,6 @@ Source Files - - Source Files - Source Files @@ -100,8 +97,5 @@ Source Files - - Source Files - - + \ No newline at end of file diff --git a/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj index edcceed..ed71de8 100644 --- a/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj +++ b/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj @@ -47,7 +47,7 @@ - + @@ -59,15 +59,12 @@ - - - {8D6BB292-9E1C-413D-9F98-4864BDC1514A} @@ -347,4 +344,4 @@ - + \ No newline at end of file diff --git a/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters index 6df7260..11cfcd0 100644 --- a/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters +++ b/deps/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters @@ -37,7 +37,7 @@ Source Files - + Source Files @@ -70,9 +70,6 @@ Source Files - - Source Files - Source Files @@ -100,11 +97,5 @@ Source Files - - Source Files - - - Source Files - - + \ No newline at end of file diff --git a/deps/jemalloc/scripts/gen_run_tests.py b/deps/jemalloc/scripts/gen_run_tests.py index a414f81..a87ecff 100755 --- a/deps/jemalloc/scripts/gen_run_tests.py +++ b/deps/jemalloc/scripts/gen_run_tests.py @@ -4,7 +4,6 @@ import sys from itertools import combinations from os import uname from multiprocessing import cpu_count -from subprocess import call # Later, we want to test extended vaddr support. Apparently, the "real" way of # checking this is flaky on OS X. @@ -14,25 +13,13 @@ nparallel = cpu_count() * 2 uname = uname()[0] -if "BSD" in uname: - make_cmd = 'gmake' -else: - make_cmd = 'make' - def powerset(items): result = [] for i in xrange(len(items) + 1): result += combinations(items, i) return result -possible_compilers = [] -for cc, cxx in (['gcc', 'g++'], ['clang', 'clang++']): - try: - cmd_ret = call([cc, "-v"]) - if cmd_ret == 0: - possible_compilers.append((cc, cxx)) - except: - pass +possible_compilers = [('gcc', 'g++'), ('clang', 'clang++')] possible_compiler_opts = [ '-m32', ] @@ -40,7 +27,6 @@ possible_config_opts = [ '--enable-debug', '--enable-prof', '--disable-stats', - '--enable-opt-safety-checks', ] if bits_64: possible_config_opts.append('--with-lg-vaddr=56') @@ -53,7 +39,7 @@ possible_malloc_conf_opts = [ ] print 'set -e' -print 'if [ -f Makefile ] ; then %(make_cmd)s relclean ; fi' % {'make_cmd': make_cmd} +print 'if [ -f Makefile ] ; then make relclean ; fi' print 'autoconf' print 'rm -rf run_tests.out' print 'mkdir run_tests.out' @@ -116,11 +102,11 @@ cd run_test_%(ind)d.out echo "==> %(config_line)s" >> run_test.log %(config_line)s >> run_test.log 2>&1 || abort -run_cmd %(make_cmd)s all tests -run_cmd %(make_cmd)s check -run_cmd %(make_cmd)s distclean +run_cmd make all tests +run_cmd make check +run_cmd make distclean EOF -chmod 755 run_test_%(ind)d.sh""" % {'ind': ind, 'config_line': config_line, 'make_cmd': make_cmd} +chmod 755 run_test_%(ind)d.sh""" % {'ind': ind, 'config_line': config_line} ind += 1 print 'for i in `seq 0 %(last_ind)d` ; do echo run_test_${i}.sh ; done | xargs -P %(nparallel)d -n 1 sh' % {'last_ind': ind-1, 'nparallel': nparallel} diff --git a/deps/jemalloc/scripts/gen_travis.py b/deps/jemalloc/scripts/gen_travis.py index f1478c6..6dd3929 100755 --- a/deps/jemalloc/scripts/gen_travis.py +++ b/deps/jemalloc/scripts/gen_travis.py @@ -4,7 +4,6 @@ from itertools import combinations travis_template = """\ language: generic -dist: precise matrix: include: @@ -12,7 +11,6 @@ matrix: before_script: - autoconf - - scripts/gen_travis.py > travis_script && diff .travis.yml travis_script - ./configure ${COMPILER_FLAGS:+ \ CC="$CC $COMPILER_FLAGS" \ CXX="$CXX $COMPILER_FLAGS" } \ @@ -45,8 +43,6 @@ configure_flag_unusuals = [ '--enable-debug', '--enable-prof', '--disable-stats', - '--disable-libdl', - '--enable-opt-safety-checks', ] malloc_conf_unusuals = [ @@ -65,85 +61,47 @@ unusual_combinations_to_test = [] for i in xrange(MAX_UNUSUAL_OPTIONS + 1): unusual_combinations_to_test += combinations(all_unusuals, i) -gcc_multilib_set = False -# Formats a job from a combination of flags -def format_job(combination): - global gcc_multilib_set +include_rows = "" +for unusual_combination in unusual_combinations_to_test: + os = os_default + if os_unusual in unusual_combination: + os = os_unusual - os = os_unusual if os_unusual in combination else os_default - compilers = compilers_unusual if compilers_unusual in combination else compilers_default + compilers = compilers_default + if compilers_unusual in unusual_combination: + compilers = compilers_unusual - compiler_flags = [x for x in combination if x in compiler_flag_unusuals] - configure_flags = [x for x in combination if x in configure_flag_unusuals] - malloc_conf = [x for x in combination if x in malloc_conf_unusuals] + compiler_flags = [ + x for x in unusual_combination if x in compiler_flag_unusuals] + configure_flags = [ + x for x in unusual_combination if x in configure_flag_unusuals] + + malloc_conf = [ + x for x in unusual_combination if x in malloc_conf_unusuals] # Filter out unsupported configurations on OS X. if os == 'osx' and ('dss:primary' in malloc_conf or \ 'percpu_arena:percpu' in malloc_conf or 'background_thread:true' \ in malloc_conf): - return "" + continue if len(malloc_conf) > 0: configure_flags.append('--with-malloc-conf=' + ",".join(malloc_conf)) # Filter out an unsupported configuration - heap profiling on OS X. if os == 'osx' and '--enable-prof' in configure_flags: - return "" + continue # We get some spurious errors when -Warray-bounds is enabled. env_string = ('{} COMPILER_FLAGS="{}" CONFIGURE_FLAGS="{}" ' 'EXTRA_CFLAGS="-Werror -Wno-array-bounds"').format( compilers, " ".join(compiler_flags), " ".join(configure_flags)) - job = "" - job += ' - os: %s\n' % os - job += ' env: %s\n' % env_string - if '-m32' in combination and os == 'linux': - job += ' addons:' - if gcc_multilib_set: - job += ' *gcc_multilib\n' - else: - job += ' &gcc_multilib\n' - job += ' apt:\n' - job += ' packages:\n' - job += ' - gcc-multilib\n' - gcc_multilib_set = True - return job - -include_rows = "" -for combination in unusual_combinations_to_test: - include_rows += format_job(combination) - -# Development build -include_rows += '''\ - # Development build - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" -''' - -# Enable-expermental-smallocx -include_rows += '''\ - # --enable-expermental-smallocx: - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" -''' - -# Valgrind build bots -include_rows += ''' - # Valgrind - - os: linux - env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind" - addons: - apt: - packages: - - valgrind -''' - -# To enable valgrind on macosx add: -# -# - os: osx -# env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind" -# install: brew install valgrind -# -# It currently fails due to: https://github.com/jemalloc/jemalloc/issues/1274 + include_rows += ' - os: %s\n' % os + include_rows += ' env: %s\n' % env_string + if '-m32' in unusual_combination and os == 'linux': + include_rows += ' addons:\n' + include_rows += ' apt:\n' + include_rows += ' packages:\n' + include_rows += ' - gcc-multilib\n' print travis_template % include_rows diff --git a/deps/jemalloc/src/arena.c b/deps/jemalloc/src/arena.c index ba50e41..5d55bf1 100644 --- a/deps/jemalloc/src/arena.c +++ b/deps/jemalloc/src/arena.c @@ -8,11 +8,9 @@ #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/safety_check.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/util.h" -JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS - /******************************************************************************/ /* Data. */ @@ -42,11 +40,7 @@ const uint64_t h_steps[SMOOTHSTEP_NSTEPS] = { #undef STEP }; -static div_info_t arena_binind_div_info[SC_NBINS]; - -size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; -size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; -static unsigned huge_arena_ind; +static div_info_t arena_binind_div_info[NBINS]; /******************************************************************************/ /* @@ -67,7 +61,7 @@ static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, /******************************************************************************/ void -arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, +arena_basic_stats_merge(UNUSED tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy) { *nthreads += arena_nthreads_get(arena, false); @@ -83,8 +77,7 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, - bin_stats_t *bstats, arena_stats_large_t *lstats, - arena_stats_extents_t *estats) { + bin_stats_t *bstats, arena_stats_large_t *lstats) { cassert(config_stats); arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms, @@ -101,10 +94,6 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_accum_zu(&astats->retained, extents_npages_get(&arena->extents_retained) << LG_PAGE); - atomic_store_zu(&astats->extent_avail, - atomic_load_zu(&arena->extent_avail_cnt, ATOMIC_RELAXED), - ATOMIC_RELAXED); - arena_stats_accum_u64(&astats->decay_dirty.npurge, arena_stats_read_u64(tsdn, &arena->stats, &arena->stats.decay_dirty.npurge)); @@ -132,10 +121,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, (((atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) + extents_npages_get(&arena->extents_dirty) + extents_npages_get(&arena->extents_muzzy)) << LG_PAGE))); - arena_stats_accum_zu(&astats->abandoned_vm, atomic_load_zu( - &arena->stats.abandoned_vm, ATOMIC_RELAXED)); - for (szind_t i = 0; i < SC_NSIZES - SC_NBINS; i++) { + for (szind_t i = 0; i < NSIZES - NBINS; i++) { uint64_t nmalloc = arena_stats_read_u64(tsdn, &arena->stats, &arena->stats.lstats[i].nmalloc); arena_stats_accum_u64(&lstats[i].nmalloc, nmalloc); @@ -153,43 +140,12 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_accum_u64(&astats->nrequests_large, nmalloc + nrequests); - /* nfill == nmalloc for large currently. */ - arena_stats_accum_u64(&lstats[i].nfills, nmalloc); - arena_stats_accum_u64(&astats->nfills_large, nmalloc); - - uint64_t nflush = arena_stats_read_u64(tsdn, &arena->stats, - &arena->stats.lstats[i].nflushes); - arena_stats_accum_u64(&lstats[i].nflushes, nflush); - arena_stats_accum_u64(&astats->nflushes_large, nflush); - assert(nmalloc >= ndalloc); assert(nmalloc - ndalloc <= SIZE_T_MAX); size_t curlextents = (size_t)(nmalloc - ndalloc); lstats[i].curlextents += curlextents; arena_stats_accum_zu(&astats->allocated_large, - curlextents * sz_index2size(SC_NBINS + i)); - } - - for (pszind_t i = 0; i < SC_NPSIZES; i++) { - size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes, - retained_bytes; - dirty = extents_nextents_get(&arena->extents_dirty, i); - muzzy = extents_nextents_get(&arena->extents_muzzy, i); - retained = extents_nextents_get(&arena->extents_retained, i); - dirty_bytes = extents_nbytes_get(&arena->extents_dirty, i); - muzzy_bytes = extents_nbytes_get(&arena->extents_muzzy, i); - retained_bytes = - extents_nbytes_get(&arena->extents_retained, i); - - atomic_store_zu(&estats[i].ndirty, dirty, ATOMIC_RELAXED); - atomic_store_zu(&estats[i].nmuzzy, muzzy, ATOMIC_RELAXED); - atomic_store_zu(&estats[i].nretained, retained, ATOMIC_RELAXED); - atomic_store_zu(&estats[i].dirty_bytes, dirty_bytes, - ATOMIC_RELAXED); - atomic_store_zu(&estats[i].muzzy_bytes, muzzy_bytes, - ATOMIC_RELAXED); - atomic_store_zu(&estats[i].retained_bytes, retained_bytes, - ATOMIC_RELAXED); + curlextents * sz_index2size(NBINS + i)); } arena_stats_unlock(tsdn, &arena->stats); @@ -200,7 +156,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, cache_bin_array_descriptor_t *descriptor; ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) { szind_t i = 0; - for (; i < SC_NBINS; i++) { + for (; i < NBINS; i++) { cache_bin_t *tbin = &descriptor->bins_small[i]; arena_stats_accum_zu(&astats->tcache_bytes, tbin->ncached * sz_index2size(i)); @@ -244,11 +200,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, nstime_update(&astats->uptime); nstime_subtract(&astats->uptime, &arena->create_time); - for (szind_t i = 0; i < SC_NBINS; i++) { - for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { - bin_stats_merge(tsdn, &bstats[i], - &arena->bins[i].bin_shards[j]); - } + for (szind_t i = 0; i < NBINS; i++) { + bin_stats_merge(tsdn, &bstats[i], &arena->bins[i]); } } @@ -283,54 +236,6 @@ arena_slab_reg_alloc(extent_t *slab, const bin_info_t *bin_info) { return ret; } -static void -arena_slab_reg_alloc_batch(extent_t *slab, const bin_info_t *bin_info, - unsigned cnt, void** ptrs) { - arena_slab_data_t *slab_data = extent_slab_data_get(slab); - - assert(extent_nfree_get(slab) >= cnt); - assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info)); - -#if (! defined JEMALLOC_INTERNAL_POPCOUNTL) || (defined BITMAP_USE_TREE) - for (unsigned i = 0; i < cnt; i++) { - size_t regind = bitmap_sfu(slab_data->bitmap, - &bin_info->bitmap_info); - *(ptrs + i) = (void *)((uintptr_t)extent_addr_get(slab) + - (uintptr_t)(bin_info->reg_size * regind)); - } -#else - unsigned group = 0; - bitmap_t g = slab_data->bitmap[group]; - unsigned i = 0; - while (i < cnt) { - while (g == 0) { - g = slab_data->bitmap[++group]; - } - size_t shift = group << LG_BITMAP_GROUP_NBITS; - size_t pop = popcount_lu(g); - if (pop > (cnt - i)) { - pop = cnt - i; - } - - /* - * Load from memory locations only once, outside the - * hot loop below. - */ - uintptr_t base = (uintptr_t)extent_addr_get(slab); - uintptr_t regsize = (uintptr_t)bin_info->reg_size; - while (pop--) { - size_t bit = cfs_lu(&g); - size_t regind = shift + bit; - *(ptrs + i) = (void *)(base + regsize * regind); - - i++; - } - slab_data->bitmap[group] = g; - } -#endif - extent_nfree_sub(slab, cnt); -} - #ifndef JEMALLOC_JET static #endif @@ -386,11 +291,11 @@ arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { cassert(config_stats); - if (usize < SC_LARGE_MINCLASS) { - usize = SC_LARGE_MINCLASS; + if (usize < LARGE_MINCLASS) { + usize = LARGE_MINCLASS; } index = sz_size2index(usize); - hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; + hindex = (index >= NBINS) ? index - NBINS : 0; arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.lstats[hindex].nmalloc, 1); @@ -402,11 +307,11 @@ arena_large_dalloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { cassert(config_stats); - if (usize < SC_LARGE_MINCLASS) { - usize = SC_LARGE_MINCLASS; + if (usize < LARGE_MINCLASS) { + usize = LARGE_MINCLASS; } index = sz_size2index(usize); - hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; + hindex = (index >= NBINS) ? index - NBINS : 0; arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.lstats[hindex].ndalloc, 1); @@ -419,11 +324,6 @@ arena_large_ralloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t oldusize, arena_large_malloc_stats_update(tsdn, arena, usize); } -static bool -arena_may_have_muzzy(arena_t *arena) { - return (pages_can_purge_lazy && (arena_muzzy_decay_ms_get(arena) != 0)); -} - extent_t * arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool *zero) { @@ -438,7 +338,7 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, extent_t *extent = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit); - if (extent == NULL && arena_may_have_muzzy(arena)) { + if (extent == NULL) { extent = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit); @@ -843,7 +743,7 @@ static size_t arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, arena_decay_t *decay, extents_t *extents, bool all, extent_list_t *decay_extents, bool is_background_thread) { - size_t nmadvise, nunmapped; + UNUSED size_t nmadvise, nunmapped; size_t npurged; if (config_stats) { @@ -934,7 +834,7 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, size_t npurge = arena_stash_decayed(tsdn, arena, &extent_hooks, extents, npages_limit, npages_decay_max, &decay_extents); if (npurge != 0) { - size_t npurged = arena_decay_stashed(tsdn, arena, + UNUSED size_t npurged = arena_decay_stashed(tsdn, arena, &extent_hooks, decay, extents, all, &decay_extents, is_background_thread); assert(npurged == npurge); @@ -963,7 +863,7 @@ arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, bool epoch_advanced = arena_maybe_decay(tsdn, arena, decay, extents, is_background_thread); - size_t npages_new; + UNUSED size_t npages_new; if (epoch_advanced) { /* Backlog is updated on epoch advance. */ npages_new = decay->backlog[SMOOTHSTEP_NSTEPS-1]; @@ -1013,17 +913,11 @@ static void arena_bin_slabs_nonfull_insert(bin_t *bin, extent_t *slab) { assert(extent_nfree_get(slab) > 0); extent_heap_insert(&bin->slabs_nonfull, slab); - if (config_stats) { - bin->stats.nonfull_slabs++; - } } static void arena_bin_slabs_nonfull_remove(bin_t *bin, extent_t *slab) { extent_heap_remove(&bin->slabs_nonfull, slab); - if (config_stats) { - bin->stats.nonfull_slabs--; - } } static extent_t * @@ -1034,7 +928,6 @@ arena_bin_slabs_nonfull_tryget(bin_t *bin) { } if (config_stats) { bin->stats.reslabs++; - bin->stats.nonfull_slabs--; } return slab; } @@ -1061,37 +954,6 @@ arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, extent_t *slab) { extent_list_remove(&bin->slabs_full, slab); } -static void -arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) { - extent_t *slab; - - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - if (bin->slabcur != NULL) { - slab = bin->slabcur; - bin->slabcur = NULL; - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - } - while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) != NULL) { - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - } - for (slab = extent_list_first(&bin->slabs_full); slab != NULL; - slab = extent_list_first(&bin->slabs_full)) { - arena_bin_slabs_full_remove(arena, bin, slab); - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - } - if (config_stats) { - bin->stats.curregs = 0; - bin->stats.curslabs = 0; - } - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); -} - void arena_reset(tsd_t *tsd, arena_t *arena) { /* @@ -1121,7 +983,7 @@ arena_reset(tsd_t *tsd, arena_t *arena) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != SC_NSIZES); + assert(alloc_ctx.szind != NSIZES); if (config_stats || (config_prof && opt_prof)) { usize = sz_index2size(alloc_ctx.szind); @@ -1137,11 +999,35 @@ arena_reset(tsd_t *tsd, arena_t *arena) { malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); /* Bins. */ - for (unsigned i = 0; i < SC_NBINS; i++) { - for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { - arena_bin_reset(tsd, arena, - &arena->bins[i].bin_shards[j]); + for (unsigned i = 0; i < NBINS; i++) { + extent_t *slab; + bin_t *bin = &arena->bins[i]; + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + if (bin->slabcur != NULL) { + slab = bin->slabcur; + bin->slabcur = NULL; + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); } + while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) != + NULL) { + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + } + for (slab = extent_list_first(&bin->slabs_full); slab != NULL; + slab = extent_list_first(&bin->slabs_full)) { + arena_bin_slabs_full_remove(arena, bin, slab); + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + } + if (config_stats) { + bin->stats.curregs = 0; + bin->stats.curslabs = 0; + } + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); } atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED); @@ -1226,7 +1112,7 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, } static extent_t * -arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard, +arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, const bin_info_t *bin_info) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); @@ -1238,7 +1124,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard extent_t *slab = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit); - if (slab == NULL && arena_may_have_muzzy(arena)) { + if (slab == NULL) { slab = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit); @@ -1254,7 +1140,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard /* Initialize slab internals. */ arena_slab_data_t *slab_data = extent_slab_data_get(slab); - extent_nfree_binshard_set(slab, bin_info->nregs, binshard); + extent_nfree_set(slab, bin_info->nregs); bitmap_init(slab_data->bitmap, &bin_info->bitmap_info, false); arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE); @@ -1264,7 +1150,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard static extent_t * arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - szind_t binind, unsigned binshard) { + szind_t binind) { extent_t *slab; const bin_info_t *bin_info; @@ -1280,7 +1166,7 @@ arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, /* Allocate a new slab. */ malloc_mutex_unlock(tsdn, &bin->lock); /******************************/ - slab = arena_slab_alloc(tsdn, arena, binind, binshard, bin_info); + slab = arena_slab_alloc(tsdn, arena, binind, bin_info); /********************************/ malloc_mutex_lock(tsdn, &bin->lock); if (slab != NULL) { @@ -1307,7 +1193,7 @@ arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, /* Re-fill bin->slabcur, then call arena_slab_reg_alloc(). */ static void * arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - szind_t binind, unsigned binshard) { + szind_t binind) { const bin_info_t *bin_info; extent_t *slab; @@ -1316,7 +1202,7 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, arena_bin_slabs_full_insert(arena, bin, bin->slabcur); bin->slabcur = NULL; } - slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind, binshard); + slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind); if (bin->slabcur != NULL) { /* * Another thread updated slabcur while this one ran without the @@ -1360,75 +1246,46 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, return arena_slab_reg_alloc(slab, bin_info); } -/* Choose a bin shard and return the locked bin. */ -bin_t * -arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, - unsigned *binshard) { - bin_t *bin; - if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) { - *binshard = 0; - } else { - *binshard = tsd_binshardsp_get(tsdn_tsd(tsdn))->binshard[binind]; - } - assert(*binshard < bin_infos[binind].n_shards); - bin = &arena->bins[binind].bin_shards[*binshard]; - malloc_mutex_lock(tsdn, &bin->lock); - - return bin; -} - void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, cache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes) { - unsigned i, nfill, cnt; + unsigned i, nfill; + bin_t *bin; assert(tbin->ncached == 0); if (config_prof && arena_prof_accum(tsdn, arena, prof_accumbytes)) { prof_idump(tsdn); } - - unsigned binshard; - bin_t *bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard); - + bin = &arena->bins[binind]; + malloc_mutex_lock(tsdn, &bin->lock); for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> - tcache->lg_fill_div[binind]); i < nfill; i += cnt) { + tcache->lg_fill_div[binind]); i < nfill; i++) { extent_t *slab; + void *ptr; if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > 0) { - unsigned tofill = nfill - i; - cnt = tofill < extent_nfree_get(slab) ? - tofill : extent_nfree_get(slab); - arena_slab_reg_alloc_batch( - slab, &bin_infos[binind], cnt, - tbin->avail - nfill + i); + ptr = arena_slab_reg_alloc(slab, &bin_infos[binind]); } else { - cnt = 1; - void *ptr = arena_bin_malloc_hard(tsdn, arena, bin, - binind, binshard); + ptr = arena_bin_malloc_hard(tsdn, arena, bin, binind); + } + if (ptr == NULL) { /* * OOM. tbin->avail isn't yet filled down to its first * element, so the successful allocations (if any) must * be moved just before tbin->avail before bailing out. */ - if (ptr == NULL) { - if (i > 0) { - memmove(tbin->avail - i, - tbin->avail - nfill, - i * sizeof(void *)); - } - break; + if (i > 0) { + memmove(tbin->avail - i, tbin->avail - nfill, + i * sizeof(void *)); } - /* Insert such that low regions get used first. */ - *(tbin->avail - nfill + i) = ptr; + break; } if (config_fill && unlikely(opt_junk_alloc)) { - for (unsigned j = 0; j < cnt; j++) { - void* ptr = *(tbin->avail - nfill + i + j); - arena_alloc_junk_small(ptr, &bin_infos[binind], - true); - } + arena_alloc_junk_small(ptr, &bin_infos[binind], true); } + /* Insert such that low regions get used first. */ + *(tbin->avail - nfill + i) = ptr; } if (config_stats) { bin->stats.nmalloc += i; @@ -1463,15 +1320,15 @@ arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) { size_t usize; extent_t *slab; - assert(binind < SC_NBINS); + assert(binind < NBINS); + bin = &arena->bins[binind]; usize = sz_index2size(binind); - unsigned binshard; - bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard); + malloc_mutex_lock(tsdn, &bin->lock); if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > 0) { ret = arena_slab_reg_alloc(slab, &bin_infos[binind]); } else { - ret = arena_bin_malloc_hard(tsdn, arena, bin, binind, binshard); + ret = arena_bin_malloc_hard(tsdn, arena, bin, binind); } if (ret == NULL) { @@ -1516,13 +1373,13 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, assert(!tsdn_null(tsdn) || arena != NULL); if (likely(!tsdn_null(tsdn))) { - arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, size); + arena = arena_choose(tsdn_tsd(tsdn), arena); } if (unlikely(arena == NULL)) { return NULL; } - if (likely(size <= SC_SMALL_MAXCLASS)) { + if (likely(size <= SMALL_MAXCLASS)) { return arena_malloc_small(tsdn, arena, ind, zero); } return large_malloc(tsdn, arena, sz_index2size(ind), zero); @@ -1533,9 +1390,8 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero, tcache_t *tcache) { void *ret; - if (usize <= SC_SMALL_MAXCLASS - && (alignment < PAGE - || (alignment == PAGE && (usize & PAGE_MASK) == 0))) { + if (usize <= SMALL_MAXCLASS && (alignment < PAGE || (alignment == PAGE + && (usize & PAGE_MASK) == 0))) { /* Small; alignment doesn't require special slab placement. */ ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize), zero, tcache, true); @@ -1550,15 +1406,11 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, } void -arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize) { +arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize) { cassert(config_prof); assert(ptr != NULL); - assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); - assert(usize <= SC_SMALL_MAXCLASS); - - if (config_opt_safety_checks) { - safety_check_set_redzone(ptr, usize, SC_LARGE_MINCLASS); - } + assert(isalloc(tsdn, ptr) == LARGE_MINCLASS); + assert(usize <= SMALL_MAXCLASS); rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); @@ -1582,15 +1434,15 @@ arena_prof_demote(tsdn_t *tsdn, extent_t *extent, const void *ptr) { cassert(config_prof); assert(ptr != NULL); - extent_szind_set(extent, SC_NBINS); + extent_szind_set(extent, NBINS); rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, - SC_NBINS, false); + NBINS, false); - assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); + assert(isalloc(tsdn, ptr) == LARGE_MINCLASS); - return SC_LARGE_MINCLASS; + return LARGE_MINCLASS; } void @@ -1600,19 +1452,10 @@ arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, assert(opt_prof); extent_t *extent = iealloc(tsdn, ptr); - size_t usize = extent_usize_get(extent); - size_t bumped_usize = arena_prof_demote(tsdn, extent, ptr); - if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) { - /* - * Currently, we only do redzoning for small sampled - * allocations. - */ - assert(bumped_usize == SC_LARGE_MINCLASS); - safety_check_verify_redzone(ptr, usize, bumped_usize); - } - if (bumped_usize <= tcache_maxclass && tcache != NULL) { + size_t usize = arena_prof_demote(tsdn, extent, ptr); + if (usize <= tcache_maxclass) { tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, - sz_size2index(bumped_usize), slow_path); + sz_size2index(usize), slow_path); } else { large_dalloc(tsdn, extent); } @@ -1656,7 +1499,7 @@ arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, } static void -arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, +arena_bin_lower_slab(UNUSED tsdn_t *tsdn, arena_t *arena, extent_t *slab, bin_t *bin) { assert(extent_nfree_get(slab) > 0); @@ -1683,9 +1526,11 @@ arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, } static void -arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - szind_t binind, extent_t *slab, void *ptr, bool junked) { +arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, extent_t *slab, + void *ptr, bool junked) { arena_slab_data_t *slab_data = extent_slab_data_get(slab); + szind_t binind = extent_szind_get(slab); + bin_t *bin = &arena->bins[binind]; const bin_info_t *bin_info = &bin_infos[binind]; if (!junked && config_fill && unlikely(opt_junk_free)) { @@ -1709,21 +1554,18 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, bin_t *bin, } void -arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin, - szind_t binind, extent_t *extent, void *ptr) { - arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr, - true); +arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, extent_t *extent, + void *ptr) { + arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, true); } static void arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr) { szind_t binind = extent_szind_get(extent); - unsigned binshard = extent_binshard_get(extent); - bin_t *bin = &arena->bins[binind].bin_shards[binshard]; + bin_t *bin = &arena->bins[binind]; malloc_mutex_lock(tsdn, &bin->lock); - arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr, - false); + arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, false); malloc_mutex_unlock(tsdn, &bin->lock); } @@ -1738,48 +1580,38 @@ arena_dalloc_small(tsdn_t *tsdn, void *ptr) { bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero, size_t *newsize) { - bool ret; + size_t extra, bool zero) { /* Calls with non-zero extra had to clamp extra. */ - assert(extra == 0 || size + extra <= SC_LARGE_MAXCLASS); + assert(extra == 0 || size + extra <= LARGE_MAXCLASS); - extent_t *extent = iealloc(tsdn, ptr); - if (unlikely(size > SC_LARGE_MAXCLASS)) { - ret = true; - goto done; + if (unlikely(size > LARGE_MAXCLASS)) { + return true; } + extent_t *extent = iealloc(tsdn, ptr); size_t usize_min = sz_s2u(size); size_t usize_max = sz_s2u(size + extra); - if (likely(oldsize <= SC_SMALL_MAXCLASS && usize_min - <= SC_SMALL_MAXCLASS)) { + if (likely(oldsize <= SMALL_MAXCLASS && usize_min <= SMALL_MAXCLASS)) { /* * Avoid moving the allocation if the size class can be left the * same. */ assert(bin_infos[sz_size2index(oldsize)].reg_size == oldsize); - if ((usize_max > SC_SMALL_MAXCLASS - || sz_size2index(usize_max) != sz_size2index(oldsize)) - && (size > oldsize || usize_max < oldsize)) { - ret = true; - goto done; + if ((usize_max > SMALL_MAXCLASS || sz_size2index(usize_max) != + sz_size2index(oldsize)) && (size > oldsize || usize_max < + oldsize)) { + return true; } arena_decay_tick(tsdn, extent_arena_get(extent)); - ret = false; - } else if (oldsize >= SC_LARGE_MINCLASS - && usize_max >= SC_LARGE_MINCLASS) { - ret = large_ralloc_no_move(tsdn, extent, usize_min, usize_max, + return false; + } else if (oldsize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS) { + return large_ralloc_no_move(tsdn, extent, usize_min, usize_max, zero); - } else { - ret = true; } -done: - assert(extent == iealloc(tsdn, ptr)); - *newsize = extent_usize_get(extent); - return ret; + return true; } static void * @@ -1790,7 +1622,7 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, zero, tcache, true); } usize = sz_sa2u(usize, alignment); - if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { return NULL; } return ipalloct(tsdn, usize, alignment, zero, tcache, arena); @@ -1798,30 +1630,22 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, void * arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t alignment, bool zero, tcache_t *tcache, - hook_ralloc_args_t *hook_args) { + size_t size, size_t alignment, bool zero, tcache_t *tcache) { size_t usize = sz_s2u(size); - if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || size > LARGE_MAXCLASS)) { return NULL; } - if (likely(usize <= SC_SMALL_MAXCLASS)) { + if (likely(usize <= SMALL_MAXCLASS)) { /* Try to avoid moving the allocation. */ - UNUSED size_t newsize; - if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero, - &newsize)) { - hook_invoke_expand(hook_args->is_realloc - ? hook_expand_realloc : hook_expand_rallocx, - ptr, oldsize, usize, (uintptr_t)ptr, - hook_args->args); + if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero)) { return ptr; } } - if (oldsize >= SC_LARGE_MINCLASS - && usize >= SC_LARGE_MINCLASS) { - return large_ralloc(tsdn, arena, ptr, usize, - alignment, zero, tcache, hook_args); + if (oldsize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS) { + return large_ralloc(tsdn, arena, iealloc(tsdn, ptr), usize, + alignment, zero, tcache); } /* @@ -1834,16 +1658,11 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, return NULL; } - hook_invoke_alloc(hook_args->is_realloc - ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, - hook_args->args); - hook_invoke_dalloc(hook_args->is_realloc - ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); - /* * Junk/zero-filling were already done by * ipalloc()/arena_malloc(). */ + size_t copysize = (usize < oldsize) ? usize : oldsize; memcpy(ret, ptr, copysize); isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); @@ -1901,7 +1720,8 @@ arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit, if (new_limit != NULL) { size_t limit = *new_limit; /* Grow no more than the new limit. */ - if ((new_ind = sz_psz2ind(limit + 1) - 1) >= SC_NPSIZES) { + if ((new_ind = sz_psz2ind(limit + 1) - 1) > + EXTENT_GROW_MAX_PIND) { return true; } } @@ -1953,12 +1773,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } } - unsigned nbins_total = 0; - for (i = 0; i < SC_NBINS; i++) { - nbins_total += bin_infos[i].n_shards; - } - size_t arena_size = sizeof(arena_t) + sizeof(bin_t) * nbins_total; - arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE); + arena = (arena_t *)base_alloc(tsdn, base, sizeof(arena_t), CACHELINE); if (arena == NULL) { goto label_error; } @@ -2050,7 +1865,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } arena->extent_grow_next = sz_psz2ind(HUGEPAGE); - arena->retain_grow_limit = sz_psz2ind(SC_LARGE_MAXCLASS); + arena->retain_grow_limit = EXTENT_GROW_MAX_PIND; if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow", WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { goto label_error; @@ -2063,20 +1878,12 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } /* Initialize bins. */ - uintptr_t bin_addr = (uintptr_t)arena + sizeof(arena_t); - atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE); - for (i = 0; i < SC_NBINS; i++) { - unsigned nshards = bin_infos[i].n_shards; - arena->bins[i].bin_shards = (bin_t *)bin_addr; - bin_addr += nshards * sizeof(bin_t); - for (unsigned j = 0; j < nshards; j++) { - bool err = bin_init(&arena->bins[i].bin_shards[j]); - if (err) { - goto label_error; - } + for (i = 0; i < NBINS; i++) { + bool err = bin_init(&arena->bins[i]); + if (err) { + goto label_error; } } - assert(bin_addr == (uintptr_t)arena + arena_size); arena->base = base; /* Set arena before creating background threads. */ @@ -2093,8 +1900,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { */ assert(!tsdn_null(tsdn)); pre_reentrancy(tsdn_tsd(tsdn), arena); - if (test_hooks_arena_new_hook) { - test_hooks_arena_new_hook(); + if (hooks_arena_new_hook) { + hooks_arena_new_hook(); } post_reentrancy(tsdn_tsd(tsdn)); } @@ -2107,75 +1914,20 @@ label_error: return NULL; } -arena_t * -arena_choose_huge(tsd_t *tsd) { - /* huge_arena_ind can be 0 during init (will use a0). */ - if (huge_arena_ind == 0) { - assert(!malloc_initialized()); - } - - arena_t *huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, false); - if (huge_arena == NULL) { - /* Create the huge arena on demand. */ - assert(huge_arena_ind != 0); - huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, true); - if (huge_arena == NULL) { - return NULL; - } - /* - * Purge eagerly for huge allocations, because: 1) number of - * huge allocations is usually small, which means ticker based - * decay is not reliable; and 2) less immediate reuse is - * expected for huge allocations. - */ - if (arena_dirty_decay_ms_default_get() > 0) { - arena_dirty_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0); - } - if (arena_muzzy_decay_ms_default_get() > 0) { - arena_muzzy_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0); - } - } - - return huge_arena; -} - -bool -arena_init_huge(void) { - bool huge_enabled; - - /* The threshold should be large size class. */ - if (opt_oversize_threshold > SC_LARGE_MAXCLASS || - opt_oversize_threshold < SC_LARGE_MINCLASS) { - opt_oversize_threshold = 0; - oversize_threshold = SC_LARGE_MAXCLASS + PAGE; - huge_enabled = false; - } else { - /* Reserve the index for the huge arena. */ - huge_arena_ind = narenas_total_get(); - oversize_threshold = opt_oversize_threshold; - huge_enabled = true; - } - - return huge_enabled; -} - -bool -arena_is_huge(unsigned arena_ind) { - if (huge_arena_ind == 0) { - return false; - } - return (arena_ind == huge_arena_ind); -} - void -arena_boot(sc_data_t *sc_data) { +arena_boot(void) { arena_dirty_decay_ms_default_set(opt_dirty_decay_ms); arena_muzzy_decay_ms_default_set(opt_muzzy_decay_ms); - for (unsigned i = 0; i < SC_NBINS; i++) { - sc_t *sc = &sc_data->sc[i]; - div_init(&arena_binind_div_info[i], - (1U << sc->lg_base) + (sc->ndelta << sc->lg_delta)); - } +#define REGIND_bin_yes(index, reg_size) \ + div_init(&arena_binind_div_info[(index)], (reg_size)); +#define REGIND_bin_no(index, reg_size) +#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, \ + lg_delta_lookup) \ + REGIND_bin_##bin(index, (1U<bins[i].bin_shards[j]); - } + for (unsigned i = 0; i < NBINS; i++) { + bin_prefork(tsdn, &arena->bins[i]); } } @@ -2231,11 +1981,8 @@ void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { unsigned i; - for (i = 0; i < SC_NBINS; i++) { - for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { - bin_postfork_parent(tsdn, - &arena->bins[i].bin_shards[j]); - } + for (i = 0; i < NBINS; i++) { + bin_postfork_parent(tsdn, &arena->bins[i]); } malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); base_postfork_parent(tsdn, arena->base); @@ -2278,10 +2025,8 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { } } - for (i = 0; i < SC_NBINS; i++) { - for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { - bin_postfork_child(tsdn, &arena->bins[i].bin_shards[j]); - } + for (i = 0; i < NBINS; i++) { + bin_postfork_child(tsdn, &arena->bins[i]); } malloc_mutex_postfork_child(tsdn, &arena->large_mtx); base_postfork_child(tsdn, arena->base); diff --git a/deps/jemalloc/src/background_thread.c b/deps/jemalloc/src/background_thread.c index 57b9b25..457669c 100644 --- a/deps/jemalloc/src/background_thread.c +++ b/deps/jemalloc/src/background_thread.c @@ -4,8 +4,6 @@ #include "jemalloc/internal/assert.h" -JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS - /******************************************************************************/ /* Data. */ @@ -13,7 +11,7 @@ JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS #define BACKGROUND_THREAD_DEFAULT false /* Read-only after initialization. */ bool opt_background_thread = BACKGROUND_THREAD_DEFAULT; -size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1; +size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT; /* Used for thread creation, termination and stats. */ malloc_mutex_t background_thread_lock; @@ -24,9 +22,13 @@ size_t max_background_threads; /* Thread info per-index. */ background_thread_info_t *background_thread_info; +/* False if no necessary runtime support. */ +bool can_enable_background_thread; + /******************************************************************************/ #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER +#include static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, void *(*)(void *), void *__restrict); @@ -79,7 +81,7 @@ background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) { } static inline bool -set_current_thread_affinity(int cpu) { +set_current_thread_affinity(UNUSED int cpu) { #if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) cpu_set_t cpuset; CPU_ZERO(&cpuset); @@ -508,8 +510,6 @@ background_thread_entry(void *ind_arg) { assert(thread_ind < max_background_threads); #ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP pthread_setname_np(pthread_self(), "jemalloc_bg_thd"); -#elif defined(__FreeBSD__) - pthread_set_name_np(pthread_self(), "jemalloc_bg_thd"); #endif if (opt_percpu_arena != percpu_arena_disabled) { set_current_thread_affinity((int)thread_ind); @@ -534,8 +534,9 @@ background_thread_init(tsd_t *tsd, background_thread_info_t *info) { n_background_threads++; } -static bool -background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { +/* Create a new background thread if needed. */ +bool +background_thread_create(tsd_t *tsd, unsigned arena_ind) { assert(have_background_thread); malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); @@ -588,19 +589,6 @@ background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { return false; } -/* Create a new background thread if needed. */ -bool -background_thread_create(tsd_t *tsd, unsigned arena_ind) { - assert(have_background_thread); - - bool ret; - malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); - ret = background_thread_create_locked(tsd, arena_ind); - malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); - - return ret; -} - bool background_threads_enable(tsd_t *tsd) { assert(n_background_threads == 0); @@ -634,7 +622,7 @@ background_threads_enable(tsd_t *tsd) { } } - return background_thread_create_locked(tsd, 0); + return background_thread_create(tsd, 0); } bool @@ -825,34 +813,21 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { #undef BILLION #undef BACKGROUND_THREAD_MIN_INTERVAL_NS -#ifdef JEMALLOC_HAVE_DLSYM -#include -#endif - static bool pthread_create_fptr_init(void) { if (pthread_create_fptr != NULL) { return false; } - /* - * Try the next symbol first, because 1) when use lazy_lock we have a - * wrapper for pthread_create; and 2) application may define its own - * wrapper as well (and can call malloc within the wrapper). - */ -#ifdef JEMALLOC_HAVE_DLSYM pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); -#else - pthread_create_fptr = NULL; -#endif if (pthread_create_fptr == NULL) { - if (config_lazy_lock) { + can_enable_background_thread = false; + if (config_lazy_lock || opt_background_thread) { malloc_write(": Error in dlsym(RTLD_NEXT, " "\"pthread_create\")\n"); abort(); - } else { - /* Fall back to the default symbol. */ - pthread_create_fptr = pthread_create; } + } else { + can_enable_background_thread = true; } return false; @@ -897,8 +872,9 @@ background_thread_boot1(tsdn_t *tsdn) { assert(have_background_thread); assert(narenas_total_get() > 0); - if (opt_max_background_threads > MAX_BACKGROUND_THREAD_LIMIT) { - opt_max_background_threads = DEFAULT_NUM_BACKGROUND_THREAD; + if (opt_max_background_threads == MAX_BACKGROUND_THREAD_LIMIT && + ncpus < MAX_BACKGROUND_THREAD_LIMIT) { + opt_max_background_threads = ncpus; } max_background_threads = opt_max_background_threads; diff --git a/deps/jemalloc/src/base.c b/deps/jemalloc/src/base.c index f3c6166..b0324b5 100644 --- a/deps/jemalloc/src/base.c +++ b/deps/jemalloc/src/base.c @@ -262,8 +262,8 @@ base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks, */ size_t min_block_size = HUGEPAGE_CEILING(sz_psz2u(header_size + gap_size + usize)); - pszind_t pind_next = (*pind_last + 1 < sz_psz2ind(SC_LARGE_MAXCLASS)) ? - *pind_last + 1 : *pind_last; + pszind_t pind_next = (*pind_last + 1 < NPSIZES) ? *pind_last + 1 : + *pind_last; size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next)); size_t block_size = (min_block_size > next_block_size) ? min_block_size : next_block_size; @@ -372,7 +372,7 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { base->extent_sn_next = extent_sn_next; base->blocks = block; base->auto_thp_switched = false; - for (szind_t i = 0; i < SC_NSIZES; i++) { + for (szind_t i = 0; i < NSIZES; i++) { extent_heap_new(&base->avail[i]); } if (config_stats) { @@ -426,7 +426,7 @@ base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment, extent_t *extent = NULL; malloc_mutex_lock(tsdn, &base->mtx); - for (szind_t i = sz_size2index(asize); i < SC_NSIZES; i++) { + for (szind_t i = sz_size2index(asize); i < NSIZES; i++) { extent = extent_heap_remove_first(&base->avail[i]); if (extent != NULL) { /* Use existing space. */ diff --git a/deps/jemalloc/src/bin.c b/deps/jemalloc/src/bin.c index bca6b12..0886bc4 100644 --- a/deps/jemalloc/src/bin.c +++ b/deps/jemalloc/src/bin.c @@ -1,68 +1,23 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/jemalloc_internal_includes.h" -#include "jemalloc/internal/assert.h" #include "jemalloc/internal/bin.h" -#include "jemalloc/internal/sc.h" #include "jemalloc/internal/witness.h" -bin_info_t bin_infos[SC_NBINS]; - -static void -bin_infos_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], - bin_info_t bin_infos[SC_NBINS]) { - for (unsigned i = 0; i < SC_NBINS; i++) { - bin_info_t *bin_info = &bin_infos[i]; - sc_t *sc = &sc_data->sc[i]; - bin_info->reg_size = ((size_t)1U << sc->lg_base) - + ((size_t)sc->ndelta << sc->lg_delta); - bin_info->slab_size = (sc->pgs << LG_PAGE); - bin_info->nregs = - (uint32_t)(bin_info->slab_size / bin_info->reg_size); - bin_info->n_shards = bin_shard_sizes[i]; - bitmap_info_t bitmap_info = BITMAP_INFO_INITIALIZER( - bin_info->nregs); - bin_info->bitmap_info = bitmap_info; - } -} - -bool -bin_update_shard_size(unsigned bin_shard_sizes[SC_NBINS], size_t start_size, - size_t end_size, size_t nshards) { - if (nshards > BIN_SHARDS_MAX || nshards == 0) { - return true; - } - - if (start_size > SC_SMALL_MAXCLASS) { - return false; - } - if (end_size > SC_SMALL_MAXCLASS) { - end_size = SC_SMALL_MAXCLASS; - } - - /* Compute the index since this may happen before sz init. */ - szind_t ind1 = sz_size2index_compute(start_size); - szind_t ind2 = sz_size2index_compute(end_size); - for (unsigned i = ind1; i <= ind2; i++) { - bin_shard_sizes[i] = (unsigned)nshards; - } - - return false; -} - -void -bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) { - /* Load the default number of shards. */ - for (unsigned i = 0; i < SC_NBINS; i++) { - bin_shard_sizes[i] = N_BIN_SHARDS_DEFAULT; - } -} - -void -bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { - assert(sc_data->initialized); - bin_infos_init(sc_data, bin_shard_sizes, bin_infos); -} +const bin_info_t bin_infos[NBINS] = { +#define BIN_INFO_bin_yes(reg_size, slab_size, nregs) \ + {reg_size, slab_size, nregs, BITMAP_INFO_INITIALIZER(nregs)}, +#define BIN_INFO_bin_no(reg_size, slab_size, nregs) +#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, \ + lg_delta_lookup) \ + BIN_INFO_bin_##bin((1U< SC_LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { ret = true; goto label_return; } @@ -321,7 +320,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) { lg_prevbuckets = ckh->lg_curbuckets; lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1; usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); - if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { return; } tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, NULL, @@ -397,7 +396,7 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh->keycomp = keycomp; usize = sz_sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE); - if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { ret = true; goto label_return; } diff --git a/deps/jemalloc/src/ctl.c b/deps/jemalloc/src/ctl.c index 48afaa6..1e713a3 100644 --- a/deps/jemalloc/src/ctl.c +++ b/deps/jemalloc/src/ctl.c @@ -8,7 +8,7 @@ #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/nstime.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/util.h" /******************************************************************************/ @@ -72,7 +72,6 @@ CTL_PROTO(config_debug) CTL_PROTO(config_fill) CTL_PROTO(config_lazy_lock) CTL_PROTO(config_malloc_conf) -CTL_PROTO(config_opt_safety_checks) CTL_PROTO(config_prof) CTL_PROTO(config_prof_libgcc) CTL_PROTO(config_prof_libunwind) @@ -81,13 +80,11 @@ CTL_PROTO(config_utrace) CTL_PROTO(config_xmalloc) CTL_PROTO(opt_abort) CTL_PROTO(opt_abort_conf) -CTL_PROTO(opt_confirm_conf) CTL_PROTO(opt_metadata_thp) CTL_PROTO(opt_retain) CTL_PROTO(opt_dss) CTL_PROTO(opt_narenas) CTL_PROTO(opt_percpu_arena) -CTL_PROTO(opt_oversize_threshold) CTL_PROTO(opt_background_thread) CTL_PROTO(opt_max_background_threads) CTL_PROTO(opt_dirty_decay_ms) @@ -129,7 +126,6 @@ INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) CTL_PROTO(arenas_bin_i_slab_size) -CTL_PROTO(arenas_bin_i_nshards) INDEX_PROTO(arenas_bin_i) CTL_PROTO(arenas_lextent_i_size) INDEX_PROTO(arenas_lextent_i) @@ -151,20 +147,14 @@ CTL_PROTO(prof_gdump) CTL_PROTO(prof_reset) CTL_PROTO(prof_interval) CTL_PROTO(lg_prof_sample) -CTL_PROTO(prof_log_start) -CTL_PROTO(prof_log_stop) CTL_PROTO(stats_arenas_i_small_allocated) CTL_PROTO(stats_arenas_i_small_nmalloc) CTL_PROTO(stats_arenas_i_small_ndalloc) CTL_PROTO(stats_arenas_i_small_nrequests) -CTL_PROTO(stats_arenas_i_small_nfills) -CTL_PROTO(stats_arenas_i_small_nflushes) CTL_PROTO(stats_arenas_i_large_allocated) CTL_PROTO(stats_arenas_i_large_nmalloc) CTL_PROTO(stats_arenas_i_large_ndalloc) CTL_PROTO(stats_arenas_i_large_nrequests) -CTL_PROTO(stats_arenas_i_large_nfills) -CTL_PROTO(stats_arenas_i_large_nflushes) CTL_PROTO(stats_arenas_i_bins_j_nmalloc) CTL_PROTO(stats_arenas_i_bins_j_ndalloc) CTL_PROTO(stats_arenas_i_bins_j_nrequests) @@ -174,20 +164,12 @@ CTL_PROTO(stats_arenas_i_bins_j_nflushes) CTL_PROTO(stats_arenas_i_bins_j_nslabs) CTL_PROTO(stats_arenas_i_bins_j_nreslabs) CTL_PROTO(stats_arenas_i_bins_j_curslabs) -CTL_PROTO(stats_arenas_i_bins_j_nonfull_slabs) INDEX_PROTO(stats_arenas_i_bins_j) CTL_PROTO(stats_arenas_i_lextents_j_nmalloc) CTL_PROTO(stats_arenas_i_lextents_j_ndalloc) CTL_PROTO(stats_arenas_i_lextents_j_nrequests) CTL_PROTO(stats_arenas_i_lextents_j_curlextents) INDEX_PROTO(stats_arenas_i_lextents_j) -CTL_PROTO(stats_arenas_i_extents_j_ndirty) -CTL_PROTO(stats_arenas_i_extents_j_nmuzzy) -CTL_PROTO(stats_arenas_i_extents_j_nretained) -CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes) -CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes) -CTL_PROTO(stats_arenas_i_extents_j_retained_bytes) -INDEX_PROTO(stats_arenas_i_extents_j) CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_uptime) CTL_PROTO(stats_arenas_i_dss) @@ -198,7 +180,6 @@ CTL_PROTO(stats_arenas_i_pdirty) CTL_PROTO(stats_arenas_i_pmuzzy) CTL_PROTO(stats_arenas_i_mapped) CTL_PROTO(stats_arenas_i_retained) -CTL_PROTO(stats_arenas_i_extent_avail) CTL_PROTO(stats_arenas_i_dirty_npurge) CTL_PROTO(stats_arenas_i_dirty_nmadvise) CTL_PROTO(stats_arenas_i_dirty_purged) @@ -210,7 +191,6 @@ CTL_PROTO(stats_arenas_i_internal) CTL_PROTO(stats_arenas_i_metadata_thp) CTL_PROTO(stats_arenas_i_tcache_bytes) CTL_PROTO(stats_arenas_i_resident) -CTL_PROTO(stats_arenas_i_abandoned_vm) INDEX_PROTO(stats_arenas_i) CTL_PROTO(stats_allocated) CTL_PROTO(stats_active) @@ -222,12 +202,6 @@ CTL_PROTO(stats_metadata_thp) CTL_PROTO(stats_resident) CTL_PROTO(stats_mapped) CTL_PROTO(stats_retained) -CTL_PROTO(experimental_hooks_install) -CTL_PROTO(experimental_hooks_remove) -CTL_PROTO(experimental_utilization_query) -CTL_PROTO(experimental_utilization_batch_query) -CTL_PROTO(experimental_arenas_i_pactivep) -INDEX_PROTO(experimental_arenas_i) #define MUTEX_STATS_CTL_PROTO_GEN(n) \ CTL_PROTO(stats_##n##_num_ops) \ @@ -296,7 +270,6 @@ static const ctl_named_node_t config_node[] = { {NAME("fill"), CTL(config_fill)}, {NAME("lazy_lock"), CTL(config_lazy_lock)}, {NAME("malloc_conf"), CTL(config_malloc_conf)}, - {NAME("opt_safety_checks"), CTL(config_opt_safety_checks)}, {NAME("prof"), CTL(config_prof)}, {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, @@ -308,13 +281,11 @@ static const ctl_named_node_t config_node[] = { static const ctl_named_node_t opt_node[] = { {NAME("abort"), CTL(opt_abort)}, {NAME("abort_conf"), CTL(opt_abort_conf)}, - {NAME("confirm_conf"), CTL(opt_confirm_conf)}, {NAME("metadata_thp"), CTL(opt_metadata_thp)}, {NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)}, {NAME("narenas"), CTL(opt_narenas)}, {NAME("percpu_arena"), CTL(opt_percpu_arena)}, - {NAME("oversize_threshold"), CTL(opt_oversize_threshold)}, {NAME("background_thread"), CTL(opt_background_thread)}, {NAME("max_background_threads"), CTL(opt_max_background_threads)}, {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)}, @@ -370,8 +341,7 @@ static const ctl_indexed_node_t arena_node[] = { static const ctl_named_node_t arenas_bin_i_node[] = { {NAME("size"), CTL(arenas_bin_i_size)}, {NAME("nregs"), CTL(arenas_bin_i_nregs)}, - {NAME("slab_size"), CTL(arenas_bin_i_slab_size)}, - {NAME("nshards"), CTL(arenas_bin_i_nshards)} + {NAME("slab_size"), CTL(arenas_bin_i_slab_size)} }; static const ctl_named_node_t super_arenas_bin_i_node[] = { {NAME(""), CHILD(named, arenas_bin_i)} @@ -415,26 +385,21 @@ static const ctl_named_node_t prof_node[] = { {NAME("gdump"), CTL(prof_gdump)}, {NAME("reset"), CTL(prof_reset)}, {NAME("interval"), CTL(prof_interval)}, - {NAME("lg_sample"), CTL(lg_prof_sample)}, - {NAME("log_start"), CTL(prof_log_start)}, - {NAME("log_stop"), CTL(prof_log_stop)} + {NAME("lg_sample"), CTL(lg_prof_sample)} }; + static const ctl_named_node_t stats_arenas_i_small_node[] = { {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)}, - {NAME("nfills"), CTL(stats_arenas_i_small_nfills)}, - {NAME("nflushes"), CTL(stats_arenas_i_small_nflushes)} + {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} }; static const ctl_named_node_t stats_arenas_i_large_node[] = { {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)}, - {NAME("nfills"), CTL(stats_arenas_i_large_nfills)}, - {NAME("nflushes"), CTL(stats_arenas_i_large_nflushes)} + {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} }; #define MUTEX_PROF_DATA_NODE(prefix) \ @@ -468,7 +433,6 @@ static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { {NAME("nslabs"), CTL(stats_arenas_i_bins_j_nslabs)}, {NAME("nreslabs"), CTL(stats_arenas_i_bins_j_nreslabs)}, {NAME("curslabs"), CTL(stats_arenas_i_bins_j_curslabs)}, - {NAME("nonfull_slabs"), CTL(stats_arenas_i_bins_j_nonfull_slabs)}, {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)} }; @@ -494,23 +458,6 @@ static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { {INDEX(stats_arenas_i_lextents_j)} }; -static const ctl_named_node_t stats_arenas_i_extents_j_node[] = { - {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)}, - {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)}, - {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)}, - {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)}, - {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)}, - {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)} -}; - -static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = { - {NAME(""), CHILD(named, stats_arenas_i_extents_j)} -}; - -static const ctl_indexed_node_t stats_arenas_i_extents_node[] = { - {INDEX(stats_arenas_i_extents_j)} -}; - #define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx) MUTEX_PROF_ARENA_MUTEXES #undef OP @@ -532,7 +479,6 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)}, {NAME("mapped"), CTL(stats_arenas_i_mapped)}, {NAME("retained"), CTL(stats_arenas_i_retained)}, - {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)}, {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)}, {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)}, {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)}, @@ -544,12 +490,10 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("metadata_thp"), CTL(stats_arenas_i_metadata_thp)}, {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)}, {NAME("resident"), CTL(stats_arenas_i_resident)}, - {NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)}, {NAME("small"), CHILD(named, stats_arenas_i_small)}, {NAME("large"), CHILD(named, stats_arenas_i_large)}, {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, - {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)}, {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)} }; static const ctl_named_node_t super_stats_arenas_i_node[] = { @@ -592,33 +536,6 @@ static const ctl_named_node_t stats_node[] = { {NAME("arenas"), CHILD(indexed, stats_arenas)} }; -static const ctl_named_node_t experimental_hooks_node[] = { - {NAME("install"), CTL(experimental_hooks_install)}, - {NAME("remove"), CTL(experimental_hooks_remove)} -}; - -static const ctl_named_node_t experimental_utilization_node[] = { - {NAME("query"), CTL(experimental_utilization_query)}, - {NAME("batch_query"), CTL(experimental_utilization_batch_query)} -}; - -static const ctl_named_node_t experimental_arenas_i_node[] = { - {NAME("pactivep"), CTL(experimental_arenas_i_pactivep)} -}; -static const ctl_named_node_t super_experimental_arenas_i_node[] = { - {NAME(""), CHILD(named, experimental_arenas_i)} -}; - -static const ctl_indexed_node_t experimental_arenas_node[] = { - {INDEX(experimental_arenas_i)} -}; - -static const ctl_named_node_t experimental_node[] = { - {NAME("hooks"), CHILD(named, experimental_hooks)}, - {NAME("utilization"), CHILD(named, experimental_utilization)}, - {NAME("arenas"), CHILD(indexed, experimental_arenas)} -}; - static const ctl_named_node_t root_node[] = { {NAME("version"), CTL(version)}, {NAME("epoch"), CTL(epoch)}, @@ -631,8 +548,7 @@ static const ctl_named_node_t root_node[] = { {NAME("arena"), CHILD(indexed, arena)}, {NAME("arenas"), CHILD(named, arenas)}, {NAME("prof"), CHILD(named, prof)}, - {NAME("stats"), CHILD(named, stats)}, - {NAME("experimental"), CHILD(named, experimental)} + {NAME("stats"), CHILD(named, stats)} }; static const ctl_named_node_t super_root_node[] = { {NAME(""), CHILD(named, root)} @@ -780,14 +696,10 @@ ctl_arena_clear(ctl_arena_t *ctl_arena) { ctl_arena->astats->nmalloc_small = 0; ctl_arena->astats->ndalloc_small = 0; ctl_arena->astats->nrequests_small = 0; - ctl_arena->astats->nfills_small = 0; - ctl_arena->astats->nflushes_small = 0; - memset(ctl_arena->astats->bstats, 0, SC_NBINS * + memset(ctl_arena->astats->bstats, 0, NBINS * sizeof(bin_stats_t)); - memset(ctl_arena->astats->lstats, 0, (SC_NSIZES - SC_NBINS) * + memset(ctl_arena->astats->lstats, 0, (NSIZES - NBINS) * sizeof(arena_stats_large_t)); - memset(ctl_arena->astats->estats, 0, SC_NPSIZES * - sizeof(arena_stats_extents_t)); } } @@ -801,9 +713,9 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, &ctl_arena->pdirty, &ctl_arena->pmuzzy, &ctl_arena->astats->astats, ctl_arena->astats->bstats, - ctl_arena->astats->lstats, ctl_arena->astats->estats); + ctl_arena->astats->lstats); - for (i = 0; i < SC_NBINS; i++) { + for (i = 0; i < NBINS; i++) { ctl_arena->astats->allocated_small += ctl_arena->astats->bstats[i].curregs * sz_index2size(i); @@ -813,10 +725,6 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { ctl_arena->astats->bstats[i].ndalloc; ctl_arena->astats->nrequests_small += ctl_arena->astats->bstats[i].nrequests; - ctl_arena->astats->nfills_small += - ctl_arena->astats->bstats[i].nfills; - ctl_arena->astats->nflushes_small += - ctl_arena->astats->bstats[i].nflushes; } } else { arena_basic_stats_merge(tsdn, arena, &ctl_arena->nthreads, @@ -852,8 +760,6 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, &astats->astats.mapped); accum_atomic_zu(&sdstats->astats.retained, &astats->astats.retained); - accum_atomic_zu(&sdstats->astats.extent_avail, - &astats->astats.extent_avail); } ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge, @@ -899,8 +805,6 @@ MUTEX_PROF_ARENA_MUTEXES sdstats->nmalloc_small += astats->nmalloc_small; sdstats->ndalloc_small += astats->ndalloc_small; sdstats->nrequests_small += astats->nrequests_small; - sdstats->nfills_small += astats->nfills_small; - sdstats->nflushes_small += astats->nflushes_small; if (!destroyed) { accum_atomic_zu(&sdstats->astats.allocated_large, @@ -915,8 +819,6 @@ MUTEX_PROF_ARENA_MUTEXES &astats->astats.ndalloc_large); ctl_accum_arena_stats_u64(&sdstats->astats.nrequests_large, &astats->astats.nrequests_large); - accum_atomic_zu(&sdstats->astats.abandoned_vm, - &astats->astats.abandoned_vm); accum_atomic_zu(&sdstats->astats.tcache_bytes, &astats->astats.tcache_bytes); @@ -925,8 +827,7 @@ MUTEX_PROF_ARENA_MUTEXES sdstats->astats.uptime = astats->astats.uptime; } - /* Merge bin stats. */ - for (i = 0; i < SC_NBINS; i++) { + for (i = 0; i < NBINS; i++) { sdstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; sdstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; sdstats->bstats[i].nrequests += @@ -945,18 +846,14 @@ MUTEX_PROF_ARENA_MUTEXES if (!destroyed) { sdstats->bstats[i].curslabs += astats->bstats[i].curslabs; - sdstats->bstats[i].nonfull_slabs += - astats->bstats[i].nonfull_slabs; } else { assert(astats->bstats[i].curslabs == 0); - assert(astats->bstats[i].nonfull_slabs == 0); } malloc_mutex_prof_merge(&sdstats->bstats[i].mutex_data, &astats->bstats[i].mutex_data); } - /* Merge stats for large allocations. */ - for (i = 0; i < SC_NSIZES - SC_NBINS; i++) { + for (i = 0; i < NSIZES - NBINS; i++) { ctl_accum_arena_stats_u64(&sdstats->lstats[i].nmalloc, &astats->lstats[i].nmalloc); ctl_accum_arena_stats_u64(&sdstats->lstats[i].ndalloc, @@ -970,22 +867,6 @@ MUTEX_PROF_ARENA_MUTEXES assert(astats->lstats[i].curlextents == 0); } } - - /* Merge extents stats. */ - for (i = 0; i < SC_NPSIZES; i++) { - accum_atomic_zu(&sdstats->estats[i].ndirty, - &astats->estats[i].ndirty); - accum_atomic_zu(&sdstats->estats[i].nmuzzy, - &astats->estats[i].nmuzzy); - accum_atomic_zu(&sdstats->estats[i].nretained, - &astats->estats[i].nretained); - accum_atomic_zu(&sdstats->estats[i].dirty_bytes, - &astats->estats[i].dirty_bytes); - accum_atomic_zu(&sdstats->estats[i].muzzy_bytes, - &astats->estats[i].muzzy_bytes); - accum_atomic_zu(&sdstats->estats[i].retained_bytes, - &astats->estats[i].retained_bytes); - } } } @@ -1497,8 +1378,8 @@ label_return: \ #define CTL_RO_CGEN(c, n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ + size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1540,8 +1421,8 @@ label_return: \ */ #define CTL_RO_NL_CGEN(c, n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ + size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1559,8 +1440,8 @@ label_return: \ #define CTL_RO_NL_GEN(n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ + size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1594,8 +1475,8 @@ label_return: \ #define CTL_RO_CONFIG_GEN(n, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ + size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1613,8 +1494,8 @@ label_return: \ CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) static int -epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; UNUSED uint64_t newval; @@ -1632,9 +1513,8 @@ label_return: } static int -background_thread_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) { +background_thread_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -1664,6 +1544,13 @@ background_thread_ctl(tsd_t *tsd, const size_t *mib, background_thread_enabled_set(tsd_tsdn(tsd), newval); if (newval) { + if (!can_enable_background_thread) { + malloc_printf(": Error in dlsym(" + "RTLD_NEXT, \"pthread_create\"). Cannot " + "enable background_thread\n"); + ret = EFAULT; + goto label_return; + } if (background_threads_enable(tsd)) { ret = EFAULT; goto label_return; @@ -1684,9 +1571,8 @@ label_return: } static int -max_background_threads_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +max_background_threads_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; size_t oldval; @@ -1719,6 +1605,13 @@ max_background_threads_ctl(tsd_t *tsd, const size_t *mib, } if (background_thread_enabled()) { + if (!can_enable_background_thread) { + malloc_printf(": Error in dlsym(" + "RTLD_NEXT, \"pthread_create\"). Cannot " + "enable background_thread\n"); + ret = EFAULT; + goto label_return; + } background_thread_enabled_set(tsd_tsdn(tsd), false); if (background_threads_disable(tsd)) { ret = EFAULT; @@ -1749,7 +1642,6 @@ CTL_RO_CONFIG_GEN(config_debug, bool) CTL_RO_CONFIG_GEN(config_fill, bool) CTL_RO_CONFIG_GEN(config_lazy_lock, bool) CTL_RO_CONFIG_GEN(config_malloc_conf, const char *) -CTL_RO_CONFIG_GEN(config_opt_safety_checks, bool) CTL_RO_CONFIG_GEN(config_prof, bool) CTL_RO_CONFIG_GEN(config_prof_libgcc, bool) CTL_RO_CONFIG_GEN(config_prof_libunwind, bool) @@ -1761,7 +1653,6 @@ CTL_RO_CONFIG_GEN(config_xmalloc, bool) CTL_RO_NL_GEN(opt_abort, opt_abort, bool) CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool) -CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool) CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *) CTL_RO_NL_GEN(opt_retain, opt_retain, bool) @@ -1769,7 +1660,6 @@ CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned) CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena], const char *) -CTL_RO_NL_GEN(opt_oversize_threshold, opt_oversize_threshold, size_t) CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool) CTL_RO_NL_GEN(opt_max_background_threads, opt_max_background_threads, size_t) CTL_RO_NL_GEN(opt_dirty_decay_ms, opt_dirty_decay_ms, ssize_t) @@ -1800,8 +1690,8 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) /******************************************************************************/ static int -thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; arena_t *oldarena; unsigned newind, oldind; @@ -1865,9 +1755,8 @@ CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp, tsd_thread_deallocatedp_get, uint64_t *) static int -thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -1887,9 +1776,8 @@ label_return: } static int -thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; if (!tcache_available(tsd)) { @@ -1908,9 +1796,8 @@ label_return: } static int -thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; if (!config_prof) { @@ -1940,9 +1827,8 @@ label_return: } static int -thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -1971,8 +1857,8 @@ label_return: /******************************************************************************/ static int -tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -1989,8 +1875,8 @@ label_return: } static int -tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -2009,8 +1895,8 @@ label_return: } static int -tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -2158,8 +2044,9 @@ arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) { if (have_background_thread) { malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); if (background_thread_enabled()) { + unsigned ind = arena_ind % ncpus; background_thread_info_t *info = - background_thread_info_get(arena_ind); + &background_thread_info[ind]; assert(info->state == background_thread_started); malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); info->state = background_thread_paused; @@ -2172,8 +2059,9 @@ static void arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) { if (have_background_thread) { if (background_thread_enabled()) { + unsigned ind = arena_ind % ncpus; background_thread_info_t *info = - background_thread_info_get(arena_ind); + &background_thread_info[ind]; assert(info->state == background_thread_paused); malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); info->state = background_thread_started; @@ -2329,17 +2217,6 @@ arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, ret = EINVAL; goto label_return; } - if (arena_is_huge(arena_ind) && *(ssize_t *)newp > 0) { - /* - * By default the huge arena purges eagerly. If it is - * set to non-zero decay time afterwards, background - * thread might be needed. - */ - if (background_thread_create(tsd, arena_ind)) { - ret = EFAULT; - goto label_return; - } - } if (dirty ? arena_dirty_decay_ms_set(tsd_tsdn(tsd), arena, *(ssize_t *)newp) : arena_muzzy_decay_ms_set(tsd_tsdn(tsd), arena, *(ssize_t *)newp)) { @@ -2423,9 +2300,8 @@ label_return: } static int -arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned arena_ind; arena_t *arena; @@ -2460,8 +2336,7 @@ label_return: } static const ctl_named_node_t * -arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t i) { +arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { const ctl_named_node_t *ret; malloc_mutex_lock(tsdn, &ctl_mtx); @@ -2486,8 +2361,8 @@ label_return: /******************************************************************************/ static int -arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned narenas; @@ -2507,9 +2382,8 @@ label_return: } static int -arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen, bool dirty) { +arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { int ret; if (oldp != NULL && oldlenp != NULL) { @@ -2551,36 +2425,34 @@ arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) CTL_RO_NL_GEN(arenas_page, PAGE, size_t) CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) -CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned) +CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t) CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t) -CTL_RO_NL_GEN(arenas_bin_i_nshards, bin_infos[mib[2]].n_shards, uint32_t) static const ctl_named_node_t * -arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t i) { - if (i > SC_NBINS) { +arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { + if (i > NBINS) { return NULL; } return super_arenas_bin_i_node; } -CTL_RO_NL_GEN(arenas_nlextents, SC_NSIZES - SC_NBINS, unsigned) -CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(SC_NBINS+(szind_t)mib[2]), +CTL_RO_NL_GEN(arenas_nlextents, NSIZES - NBINS, unsigned) +CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(NBINS+(szind_t)mib[2]), size_t) static const ctl_named_node_t * -arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t i) { - if (i > SC_NSIZES - SC_NBINS) { +arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, + size_t i) { + if (i > NSIZES - NBINS) { return NULL; } return super_arenas_lextent_i_node; } static int -arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; extent_hooks_t *extent_hooks; unsigned arena_ind; @@ -2602,9 +2474,8 @@ label_return: } static int -arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned arena_ind; void *ptr; @@ -2635,9 +2506,8 @@ label_return: /******************************************************************************/ static int -prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) { +prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -2663,8 +2533,8 @@ label_return: } static int -prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -2689,8 +2559,8 @@ label_return: } static int -prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; const char *filename = NULL; @@ -2712,8 +2582,8 @@ label_return: } static int -prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -2738,8 +2608,8 @@ label_return: } static int -prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; size_t lg_sample = lg_prof_sample; @@ -2763,44 +2633,6 @@ label_return: CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) -static int -prof_log_start_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { - int ret; - - const char *filename = NULL; - - if (!config_prof) { - return ENOENT; - } - - WRITEONLY(); - WRITE(filename, const char *); - - if (prof_log_start(tsd_tsdn(tsd), filename)) { - ret = EFAULT; - goto label_return; - } - - ret = 0; -label_return: - return ret; -} - -static int -prof_log_stop_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { - if (!config_prof) { - return ENOENT; - } - - if (prof_log_stop(tsd_tsdn(tsd))) { - return EFAULT; - } - - return 0; -} - /******************************************************************************/ CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t) @@ -2835,10 +2667,6 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, CTL_RO_CGEN(config_stats, stats_arenas_i_retained, atomic_load_zu(&arenas_i(mib[2])->astats->astats.retained, ATOMIC_RELAXED), size_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail, - atomic_load_zu(&arenas_i(mib[2])->astats->astats.extent_avail, - ATOMIC_RELAXED), - size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge, ctl_arena_stats_read_u64( @@ -2875,9 +2703,6 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes, CTL_RO_CGEN(config_stats, stats_arenas_i_resident, atomic_load_zu(&arenas_i(mib[2])->astats->astats.resident, ATOMIC_RELAXED), size_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_abandoned_vm, - atomic_load_zu(&arenas_i(mib[2])->astats->astats.abandoned_vm, - ATOMIC_RELAXED), size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, arenas_i(mib[2])->astats->allocated_small, size_t) @@ -2887,10 +2712,6 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc, arenas_i(mib[2])->astats->ndalloc_small, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests, arenas_i(mib[2])->astats->nrequests_small, uint64_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_small_nfills, - arenas_i(mib[2])->astats->nfills_small, uint64_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_small_nflushes, - arenas_i(mib[2])->astats->nflushes_small, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated, atomic_load_zu(&arenas_i(mib[2])->astats->astats.allocated_large, ATOMIC_RELAXED), size_t) @@ -2900,19 +2721,12 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc, CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc, ctl_arena_stats_read_u64( &arenas_i(mib[2])->astats->astats.ndalloc_large), uint64_t) +/* + * Note: "nmalloc" here instead of "nrequests" in the read. This is intentional. + */ CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, ctl_arena_stats_read_u64( - &arenas_i(mib[2])->astats->astats.nrequests_large), uint64_t) -/* - * Note: "nmalloc_large" here instead of "nfills" in the read. This is - * intentional (large has no batch fill). - */ -CTL_RO_CGEN(config_stats, stats_arenas_i_large_nfills, - ctl_arena_stats_read_u64( - &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_large_nflushes, - ctl_arena_stats_read_u64( - &arenas_i(mib[2])->astats->astats.nflushes_large), uint64_t) + &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t) /* Intentional. */ /* Lock profiling related APIs below. */ #define RO_MUTEX_CTL_GEN(n, l) \ @@ -2951,9 +2765,8 @@ RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, /* Resets all mutex stats, including global, arena and bin mutexes. */ static int -stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) { +stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { if (!config_stats) { return ENOENT; } @@ -2993,11 +2806,9 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, MUTEX_PROF_RESET(arena->tcache_ql_mtx); MUTEX_PROF_RESET(arena->base->mtx); - for (szind_t i = 0; i < SC_NBINS; i++) { - for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { - bin_t *bin = &arena->bins[i].bin_shards[j]; - MUTEX_PROF_RESET(bin->lock); - } + for (szind_t i = 0; i < NBINS; i++) { + bin_t *bin = &arena->bins[i]; + MUTEX_PROF_RESET(bin->lock); } } #undef MUTEX_PROF_RESET @@ -3022,13 +2833,11 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreslabs, arenas_i(mib[2])->astats->bstats[mib[4]].reslabs, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs, arenas_i(mib[2])->astats->bstats[mib[4]].curslabs, size_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nonfull_slabs, - arenas_i(mib[2])->astats->bstats[mib[4]].nonfull_slabs, size_t) static const ctl_named_node_t * -stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t j) { - if (j > SC_NBINS) { +stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, + size_t j) { + if (j > NBINS) { return NULL; } return super_stats_arenas_i_bins_j_node; @@ -3047,65 +2856,22 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t) static const ctl_named_node_t * -stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t j) { - if (j > SC_NSIZES - SC_NBINS) { +stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, + size_t j) { + if (j > NSIZES - NBINS) { return NULL; } return super_stats_arenas_i_lextents_j_node; } -CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty, - atomic_load_zu( - &arenas_i(mib[2])->astats->estats[mib[4]].ndirty, - ATOMIC_RELAXED), size_t); -CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy, - atomic_load_zu( - &arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, - ATOMIC_RELAXED), size_t); -CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained, - atomic_load_zu( - &arenas_i(mib[2])->astats->estats[mib[4]].nretained, - ATOMIC_RELAXED), size_t); -CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes, - atomic_load_zu( - &arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, - ATOMIC_RELAXED), size_t); -CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes, - atomic_load_zu( - &arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, - ATOMIC_RELAXED), size_t); -CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes, - atomic_load_zu( - &arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, - ATOMIC_RELAXED), size_t); - static const ctl_named_node_t * -stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t j) { - if (j >= SC_NPSIZES) { - return NULL; - } - return super_stats_arenas_i_extents_j_node; -} - -static bool -ctl_arenas_i_verify(size_t i) { - size_t a = arenas_i2a_impl(i, true, true); - if (a == UINT_MAX || !ctl_arenas->arenas[a]->initialized) { - return true; - } - - return false; -} - -static const ctl_named_node_t * -stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t i) { +stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { const ctl_named_node_t *ret; + size_t a; malloc_mutex_lock(tsdn, &ctl_mtx); - if (ctl_arenas_i_verify(i)) { + a = arenas_i2a_impl(i, true, true); + if (a == UINT_MAX || !ctl_arenas->arenas[a]->initialized) { ret = NULL; goto label_return; } @@ -3115,321 +2881,3 @@ label_return: malloc_mutex_unlock(tsdn, &ctl_mtx); return ret; } - -static int -experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - if (oldp == NULL || oldlenp == NULL|| newp == NULL) { - ret = EINVAL; - goto label_return; - } - /* - * Note: this is a *private* struct. This is an experimental interface; - * forcing the user to know the jemalloc internals well enough to - * extract the ABI hopefully ensures nobody gets too comfortable with - * this API, which can change at a moment's notice. - */ - hooks_t hooks; - WRITE(hooks, hooks_t); - void *handle = hook_install(tsd_tsdn(tsd), &hooks); - if (handle == NULL) { - ret = EAGAIN; - goto label_return; - } - READ(handle, void *); - - ret = 0; -label_return: - return ret; -} - -static int -experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - WRITEONLY(); - void *handle = NULL; - WRITE(handle, void *); - if (handle == NULL) { - ret = EINVAL; - goto label_return; - } - hook_remove(tsd_tsdn(tsd), handle); - ret = 0; -label_return: - return ret; -} - -/* - * Output six memory utilization entries for an input pointer, the first one of - * type (void *) and the remaining five of type size_t, describing the following - * (in the same order): - * - * (a) memory address of the extent a potential reallocation would go into, - * == the five fields below describe about the extent the pointer resides in == - * (b) number of free regions in the extent, - * (c) number of regions in the extent, - * (d) size of the extent in terms of bytes, - * (e) total number of free regions in the bin the extent belongs to, and - * (f) total number of regions in the bin the extent belongs to. - * - * Note that "(e)" and "(f)" are only available when stats are enabled; - * otherwise their values are undefined. - * - * This API is mainly intended for small class allocations, where extents are - * used as slab. - * - * In case of large class allocations, "(a)" will be NULL, and "(e)" and "(f)" - * will be zero (if stats are enabled; otherwise undefined). The other three - * fields will be properly set though the values are trivial: "(b)" will be 0, - * "(c)" will be 1, and "(d)" will be the usable size. - * - * The input pointer and size are respectively passed in by newp and newlen, - * and the output fields and size are respectively oldp and *oldlenp. - * - * It can be beneficial to define the following macros to make it easier to - * access the output: - * - * #define SLABCUR_READ(out) (*(void **)out) - * #define COUNTS(out) ((size_t *)((void **)out + 1)) - * #define NFREE_READ(out) COUNTS(out)[0] - * #define NREGS_READ(out) COUNTS(out)[1] - * #define SIZE_READ(out) COUNTS(out)[2] - * #define BIN_NFREE_READ(out) COUNTS(out)[3] - * #define BIN_NREGS_READ(out) COUNTS(out)[4] - * - * and then write e.g. NFREE_READ(oldp) to fetch the output. See the unit test - * test_query in test/unit/extent_util.c for an example. - * - * For a typical defragmentation workflow making use of this API for - * understanding the fragmentation level, please refer to the comment for - * experimental_utilization_batch_query_ctl. - * - * It's up to the application how to determine the significance of - * fragmentation relying on the outputs returned. Possible choices are: - * - * (a) if extent utilization ratio is below certain threshold, - * (b) if extent memory consumption is above certain threshold, - * (c) if extent utilization ratio is significantly below bin utilization ratio, - * (d) if input pointer deviates a lot from potential reallocation address, or - * (e) some selection/combination of the above. - * - * The caller needs to make sure that the input/output arguments are valid, - * in particular, that the size of the output is correct, i.e.: - * - * *oldlenp = sizeof(void *) + sizeof(size_t) * 5 - * - * Otherwise, the function immediately returns EINVAL without touching anything. - * - * In the rare case where there's no associated extent found for the input - * pointer, the function zeros out all output fields and return. Please refer - * to the comment for experimental_utilization_batch_query_ctl to understand the - * motivation from C++. - */ -static int -experimental_utilization_query_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - - assert(sizeof(extent_util_stats_verbose_t) - == sizeof(void *) + sizeof(size_t) * 5); - - if (oldp == NULL || oldlenp == NULL - || *oldlenp != sizeof(extent_util_stats_verbose_t) - || newp == NULL) { - ret = EINVAL; - goto label_return; - } - - void *ptr = NULL; - WRITE(ptr, void *); - extent_util_stats_verbose_t *util_stats - = (extent_util_stats_verbose_t *)oldp; - extent_util_stats_verbose_get(tsd_tsdn(tsd), ptr, - &util_stats->nfree, &util_stats->nregs, &util_stats->size, - &util_stats->bin_nfree, &util_stats->bin_nregs, - &util_stats->slabcur_addr); - ret = 0; - -label_return: - return ret; -} - -/* - * Given an input array of pointers, output three memory utilization entries of - * type size_t for each input pointer about the extent it resides in: - * - * (a) number of free regions in the extent, - * (b) number of regions in the extent, and - * (c) size of the extent in terms of bytes. - * - * This API is mainly intended for small class allocations, where extents are - * used as slab. In case of large class allocations, the outputs are trivial: - * "(a)" will be 0, "(b)" will be 1, and "(c)" will be the usable size. - * - * Note that multiple input pointers may reside on a same extent so the output - * fields may contain duplicates. - * - * The format of the input/output looks like: - * - * input[0]: 1st_pointer_to_query | output[0]: 1st_extent_n_free_regions - * | output[1]: 1st_extent_n_regions - * | output[2]: 1st_extent_size - * input[1]: 2nd_pointer_to_query | output[3]: 2nd_extent_n_free_regions - * | output[4]: 2nd_extent_n_regions - * | output[5]: 2nd_extent_size - * ... | ... - * - * The input array and size are respectively passed in by newp and newlen, and - * the output array and size are respectively oldp and *oldlenp. - * - * It can be beneficial to define the following macros to make it easier to - * access the output: - * - * #define NFREE_READ(out, i) out[(i) * 3] - * #define NREGS_READ(out, i) out[(i) * 3 + 1] - * #define SIZE_READ(out, i) out[(i) * 3 + 2] - * - * and then write e.g. NFREE_READ(oldp, i) to fetch the output. See the unit - * test test_batch in test/unit/extent_util.c for a concrete example. - * - * A typical workflow would be composed of the following steps: - * - * (1) flush tcache: mallctl("thread.tcache.flush", ...) - * (2) initialize input array of pointers to query fragmentation - * (3) allocate output array to hold utilization statistics - * (4) query utilization: mallctl("experimental.utilization.batch_query", ...) - * (5) (optional) decide if it's worthwhile to defragment; otherwise stop here - * (6) disable tcache: mallctl("thread.tcache.enabled", ...) - * (7) defragment allocations with significant fragmentation, e.g.: - * for each allocation { - * if it's fragmented { - * malloc(...); - * memcpy(...); - * free(...); - * } - * } - * (8) enable tcache: mallctl("thread.tcache.enabled", ...) - * - * The application can determine the significance of fragmentation themselves - * relying on the statistics returned, both at the overall level i.e. step "(5)" - * and at individual allocation level i.e. within step "(7)". Possible choices - * are: - * - * (a) whether memory utilization ratio is below certain threshold, - * (b) whether memory consumption is above certain threshold, or - * (c) some combination of the two. - * - * The caller needs to make sure that the input/output arrays are valid and - * their sizes are proper as well as matched, meaning: - * - * (a) newlen = n_pointers * sizeof(const void *) - * (b) *oldlenp = n_pointers * sizeof(size_t) * 3 - * (c) n_pointers > 0 - * - * Otherwise, the function immediately returns EINVAL without touching anything. - * - * In the rare case where there's no associated extent found for some pointers, - * rather than immediately terminating the computation and raising an error, - * the function simply zeros out the corresponding output fields and continues - * the computation until all input pointers are handled. The motivations of - * such a design are as follows: - * - * (a) The function always either processes nothing or processes everything, and - * never leaves the output half touched and half untouched. - * - * (b) It facilitates usage needs especially common in C++. A vast variety of - * C++ objects are instantiated with multiple dynamic memory allocations. For - * example, std::string and std::vector typically use at least two allocations, - * one for the metadata and one for the actual content. Other types may use - * even more allocations. When inquiring about utilization statistics, the - * caller often wants to examine into all such allocations, especially internal - * one(s), rather than just the topmost one. The issue comes when some - * implementations do certain optimizations to reduce/aggregate some internal - * allocations, e.g. putting short strings directly into the metadata, and such - * decisions are not known to the caller. Therefore, we permit pointers to - * memory usages that may not be returned by previous malloc calls, and we - * provide the caller a convenient way to identify such cases. - */ -static int -experimental_utilization_batch_query_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret; - - assert(sizeof(extent_util_stats_t) == sizeof(size_t) * 3); - - const size_t len = newlen / sizeof(const void *); - if (oldp == NULL || oldlenp == NULL || newp == NULL || newlen == 0 - || newlen != len * sizeof(const void *) - || *oldlenp != len * sizeof(extent_util_stats_t)) { - ret = EINVAL; - goto label_return; - } - - void **ptrs = (void **)newp; - extent_util_stats_t *util_stats = (extent_util_stats_t *)oldp; - size_t i; - for (i = 0; i < len; ++i) { - extent_util_stats_get(tsd_tsdn(tsd), ptrs[i], - &util_stats[i].nfree, &util_stats[i].nregs, - &util_stats[i].size); - } - ret = 0; - -label_return: - return ret; -} - -static const ctl_named_node_t * -experimental_arenas_i_index(tsdn_t *tsdn, const size_t *mib, - size_t miblen, size_t i) { - const ctl_named_node_t *ret; - - malloc_mutex_lock(tsdn, &ctl_mtx); - if (ctl_arenas_i_verify(i)) { - ret = NULL; - goto label_return; - } - ret = super_experimental_arenas_i_node; -label_return: - malloc_mutex_unlock(tsdn, &ctl_mtx); - return ret; -} - -static int -experimental_arenas_i_pactivep_ctl(tsd_t *tsd, const size_t *mib, - size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - if (!config_stats) { - return ENOENT; - } - if (oldp == NULL || oldlenp == NULL || *oldlenp != sizeof(size_t *)) { - return EINVAL; - } - - unsigned arena_ind; - arena_t *arena; - int ret; - size_t *pactivep; - - malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); - READONLY(); - MIB_UNSIGNED(arena_ind, 2); - if (arena_ind < narenas_total_get() && (arena = - arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { -#if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) || \ - defined(JEMALLOC_GCC_SYNC_ATOMICS) || defined(_MSC_VER) - /* Expose the underlying counter for fast read. */ - pactivep = (size_t *)&(arena->nactive.repr); - READ(pactivep, size_t *); - ret = 0; -#else - ret = EFAULT; -#endif - } else { - ret = EFAULT; - } -label_return: - malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); - return ret; -} diff --git a/deps/jemalloc/src/extent.c b/deps/jemalloc/src/extent.c index 9237f90..09d6d77 100644 --- a/deps/jemalloc/src/extent.c +++ b/deps/jemalloc/src/extent.c @@ -20,7 +20,7 @@ mutex_pool_t extent_mutex_pool; size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT; static const bitmap_info_t extents_bitmap_info = - BITMAP_INFO_INITIALIZER(SC_NPSIZES+1); + BITMAP_INFO_INITIALIZER(NPSIZES+1); static void *extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, @@ -50,16 +50,20 @@ static bool extent_purge_forced_default(extent_hooks_t *extent_hooks, static bool extent_purge_forced_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t offset, size_t length, bool growing_retained); +#ifdef JEMALLOC_MAPS_COALESCE static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t size_a, size_t size_b, bool committed, unsigned arena_ind); +#endif static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b, bool growing_retained); +#ifdef JEMALLOC_MAPS_COALESCE static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, void *addr_b, size_t size_b, bool committed, unsigned arena_ind); +#endif static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b, bool growing_retained); @@ -84,9 +88,11 @@ const extent_hooks_t extent_hooks_default = { , NULL #endif +#ifdef JEMALLOC_MAPS_COALESCE , extent_split_default, extent_merge_default +#endif }; /* Used exclusively for gdump triggering. */ @@ -113,13 +119,9 @@ static void extent_record(tsdn_t *tsdn, arena_t *arena, /******************************************************************************/ -#define ATTR_NONE /* does nothing */ - -ph_gen(ATTR_NONE, extent_avail_, extent_tree_t, extent_t, ph_link, +ph_gen(UNUSED, extent_avail_, extent_tree_t, extent_t, ph_link, extent_esnead_comp) -#undef ATTR_NONE - typedef enum { lock_result_success, lock_result_failure, @@ -128,16 +130,13 @@ typedef enum { static lock_result_t extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm, - extent_t **result, bool inactive_only) { + extent_t **result) { extent_t *extent1 = rtree_leaf_elm_extent_read(tsdn, &extents_rtree, elm, true); - /* Slab implies active extents and should be skipped. */ - if (extent1 == NULL || (inactive_only && rtree_leaf_elm_slab_read(tsdn, - &extents_rtree, elm, true))) { + if (extent1 == NULL) { return lock_result_no_extent; } - /* * It's possible that the extent changed out from under us, and with it * the leaf->extent mapping. We have to recheck while holding the lock. @@ -160,8 +159,7 @@ extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm, * address, and NULL otherwise. */ static extent_t * -extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr, - bool inactive_only) { +extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) { extent_t *ret = NULL; rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)addr, false, false); @@ -170,8 +168,7 @@ extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr, } lock_result_t lock_result; do { - lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret, - inactive_only); + lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret); } while (lock_result == lock_result_failure); return ret; } @@ -185,7 +182,6 @@ extent_alloc(tsdn_t *tsdn, arena_t *arena) { return base_alloc_extent(tsdn, arena->base); } extent_avail_remove(&arena->extent_avail, extent); - atomic_fetch_sub_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED); malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); return extent; } @@ -194,7 +190,6 @@ void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); extent_avail_insert(&arena->extent_avail, extent); - atomic_fetch_add_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED); malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); } @@ -260,7 +255,7 @@ extent_size_quantize_ceil(size_t size) { size_t ret; assert(size > 0); - assert(size - sz_large_pad <= SC_LARGE_MAXCLASS); + assert(size - sz_large_pad <= LARGE_MAXCLASS); assert((size & PAGE_MASK) == 0); ret = extent_size_quantize_floor(size); @@ -289,7 +284,7 @@ extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, malloc_mutex_rank_exclusive)) { return true; } - for (unsigned i = 0; i < SC_NPSIZES + 1; i++) { + for (unsigned i = 0; i < NPSIZES+1; i++) { extent_heap_new(&extents->heaps[i]); } bitmap_init(extents->bitmap, &extents_bitmap_info, true); @@ -310,32 +305,6 @@ extents_npages_get(extents_t *extents) { return atomic_load_zu(&extents->npages, ATOMIC_RELAXED); } -size_t -extents_nextents_get(extents_t *extents, pszind_t pind) { - return atomic_load_zu(&extents->nextents[pind], ATOMIC_RELAXED); -} - -size_t -extents_nbytes_get(extents_t *extents, pszind_t pind) { - return atomic_load_zu(&extents->nbytes[pind], ATOMIC_RELAXED); -} - -static void -extents_stats_add(extents_t *extent, pszind_t pind, size_t sz) { - size_t cur = atomic_load_zu(&extent->nextents[pind], ATOMIC_RELAXED); - atomic_store_zu(&extent->nextents[pind], cur + 1, ATOMIC_RELAXED); - cur = atomic_load_zu(&extent->nbytes[pind], ATOMIC_RELAXED); - atomic_store_zu(&extent->nbytes[pind], cur + sz, ATOMIC_RELAXED); -} - -static void -extents_stats_sub(extents_t *extent, pszind_t pind, size_t sz) { - size_t cur = atomic_load_zu(&extent->nextents[pind], ATOMIC_RELAXED); - atomic_store_zu(&extent->nextents[pind], cur - 1, ATOMIC_RELAXED); - cur = atomic_load_zu(&extent->nbytes[pind], ATOMIC_RELAXED); - atomic_store_zu(&extent->nbytes[pind], cur - sz, ATOMIC_RELAXED); -} - static void extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { malloc_mutex_assert_owner(tsdn, &extents->mtx); @@ -349,11 +318,6 @@ extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { (size_t)pind); } extent_heap_insert(&extents->heaps[pind], extent); - - if (config_stats) { - extents_stats_add(extents, pind, size); - } - extent_list_append(&extents->lru, extent); size_t npages = size >> LG_PAGE; /* @@ -376,11 +340,6 @@ extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { size_t psz = extent_size_quantize_floor(size); pszind_t pind = sz_psz2ind(psz); extent_heap_remove(&extents->heaps[pind], extent); - - if (config_stats) { - extents_stats_sub(extents, pind, size); - } - if (extent_heap_empty(&extents->heaps[pind])) { bitmap_set(extents->bitmap, &extents_bitmap_info, (size_t)pind); @@ -412,7 +371,7 @@ extents_fit_alignment(extents_t *extents, size_t min_size, size_t max_size, &extents_bitmap_info, (size_t)pind); i < pind_max; i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, (size_t)i+1)) { - assert(i < SC_NPSIZES); + assert(i < NPSIZES); assert(!extent_heap_empty(&extents->heaps[i])); extent_t *extent = extent_heap_first(&extents->heaps[i]); uintptr_t base = (uintptr_t)extent_base_get(extent); @@ -435,6 +394,30 @@ extents_fit_alignment(extents_t *extents, size_t min_size, size_t max_size, return NULL; } +/* Do any-best-fit extent selection, i.e. select any extent that best fits. */ +static extent_t * +extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, + size_t size) { + pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); + pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, + (size_t)pind); + if (i < NPSIZES+1) { + /* + * In order to reduce fragmentation, avoid reusing and splitting + * large extents for much smaller sizes. + */ + if ((sz_pind2sz(i) >> opt_lg_extent_max_active_fit) > size) { + return NULL; + } + assert(!extent_heap_empty(&extents->heaps[i])); + extent_t *extent = extent_heap_first(&extents->heaps[i]); + assert(extent_size_get(extent) >= size); + return extent; + } + + return NULL; +} + /* * Do first-fit extent selection, i.e. select the oldest/lowest extent that is * large enough. @@ -445,49 +428,30 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, extent_t *ret = NULL; pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); - - if (!maps_coalesce && !opt_retain) { - /* - * No split / merge allowed (Windows w/o retain). Try exact fit - * only. - */ - return extent_heap_empty(&extents->heaps[pind]) ? NULL : - extent_heap_first(&extents->heaps[pind]); - } - for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, - &extents_bitmap_info, (size_t)pind); - i < SC_NPSIZES + 1; - i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, + &extents_bitmap_info, (size_t)pind); i < NPSIZES+1; i = + (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, (size_t)i+1)) { assert(!extent_heap_empty(&extents->heaps[i])); extent_t *extent = extent_heap_first(&extents->heaps[i]); assert(extent_size_get(extent) >= size); - /* - * In order to reduce fragmentation, avoid reusing and splitting - * large extents for much smaller sizes. - * - * Only do check for dirty extents (delay_coalesce). - */ - if (extents->delay_coalesce && - (sz_pind2sz(i) >> opt_lg_extent_max_active_fit) > size) { - break; - } if (ret == NULL || extent_snad_comp(extent, ret) < 0) { ret = extent; } - if (i == SC_NPSIZES) { + if (i == NPSIZES) { break; } - assert(i < SC_NPSIZES); + assert(i < NPSIZES); } return ret; } /* - * Do first-fit extent selection, where the selection policy choice is - * based on extents->delay_coalesce. + * Do {best,first}-fit extent selection, where the selection policy choice is + * based on extents->delay_coalesce. Best-fit selection requires less + * searching, but its layout policy is less stable and may cause higher virtual + * memory fragmentation as a side effect. */ static extent_t * extents_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, @@ -500,7 +464,8 @@ extents_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, return NULL; } - extent_t *extent = + extent_t *extent = extents->delay_coalesce ? + extents_best_fit_locked(tsdn, arena, extents, max_size) : extents_first_fit_locked(tsdn, arena, extents, max_size); if (alignment > PAGE && extent == NULL) { @@ -627,24 +592,16 @@ label_return: return extent; } -/* - * This can only happen when we fail to allocate a new extent struct (which - * indicates OOM), e.g. when trying to split an existing extent. - */ static void -extents_abandon_vm(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, +extents_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent, bool growing_retained) { - size_t sz = extent_size_get(extent); - if (config_stats) { - arena_stats_accum_zu(&arena->stats.abandoned_vm, sz); - } /* * Leak extent after making sure its pages have already been purged, so * that this is only a virtual memory leak. */ if (extents_state_get(extents) == extent_state_dirty) { if (extent_purge_lazy_impl(tsdn, arena, r_extent_hooks, - extent, 0, sz, growing_retained)) { + extent, 0, extent_size_get(extent), growing_retained)) { extent_purge_forced_impl(tsdn, arena, r_extent_hooks, extent, 0, extent_size_get(extent), growing_retained); @@ -791,7 +748,6 @@ extent_register_impl(tsdn_t *tsdn, extent_t *extent, bool gdump_add) { if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, false, true, &elm_a, &elm_b)) { - extent_unlock(tsdn, extent); return true; } @@ -861,7 +817,7 @@ extent_deregister_impl(tsdn_t *tsdn, extent_t *extent, bool gdump) { extent_lock(tsdn, extent); - extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, SC_NSIZES, false); + extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, NSIZES, false); if (extent_slab_get(extent)) { extent_interior_deregister(tsdn, rtree_ctx, extent); extent_slab_set(extent, false); @@ -918,8 +874,7 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, extent_hooks_assure_initialized(arena, r_extent_hooks); extent_t *extent; if (new_addr != NULL) { - extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr, - false); + extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr); if (extent != NULL) { /* * We might null-out extent to report an error, but we @@ -1003,7 +958,7 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, if (leadsize != 0) { *lead = *extent; *extent = extent_split_impl(tsdn, arena, r_extent_hooks, - *lead, leadsize, SC_NSIZES, false, esize + trailsize, szind, + *lead, leadsize, NSIZES, false, esize + trailsize, szind, slab, growing_retained); if (*extent == NULL) { *to_leak = *lead; @@ -1015,7 +970,7 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, /* Split the trail. */ if (trailsize != 0) { *trail = extent_split_impl(tsdn, arena, r_extent_hooks, *extent, - esize, szind, slab, trailsize, SC_NSIZES, false, + esize, szind, slab, trailsize, NSIZES, false, growing_retained); if (*trail == NULL) { *to_leak = *extent; @@ -1032,7 +987,7 @@ extent_split_interior(tsdn_t *tsdn, arena_t *arena, * splitting occurred. */ extent_szind_set(*extent, szind); - if (szind != SC_NSIZES) { + if (szind != NSIZES) { rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)extent_addr_get(*extent), szind, slab); if (slab && extent_size_get(*extent) > PAGE) { @@ -1068,17 +1023,6 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, &to_leak, &to_salvage, new_addr, size, pad, alignment, slab, szind, growing_retained); - if (!maps_coalesce && result != extent_split_interior_ok - && !opt_retain) { - /* - * Split isn't supported (implies Windows w/o retain). Avoid - * leaking the extents. - */ - assert(to_leak != NULL && lead == NULL && trail == NULL); - extent_deactivate(tsdn, arena, extents, to_leak); - return NULL; - } - if (result == extent_split_interior_ok) { if (lead != NULL) { extent_deactivate(tsdn, arena, extents, lead); @@ -1099,27 +1043,16 @@ extent_recycle_split(tsdn_t *tsdn, arena_t *arena, if (to_leak != NULL) { void *leak = extent_base_get(to_leak); extent_deregister_no_gdump_sub(tsdn, to_leak); - extents_abandon_vm(tsdn, arena, r_extent_hooks, extents, + extents_leak(tsdn, arena, r_extent_hooks, extents, to_leak, growing_retained); - assert(extent_lock_from_addr(tsdn, rtree_ctx, leak, - false) == NULL); + assert(extent_lock_from_addr(tsdn, rtree_ctx, leak) + == NULL); } return NULL; } unreachable(); } -static bool -extent_need_manual_zero(arena_t *arena) { - /* - * Need to manually zero the extent on repopulating if either; 1) non - * default extent hooks installed (in which case the purge semantics may - * change); or 2) transparent huge pages enabled. - */ - return (!arena_has_default_hooks(arena) || - (opt_thp == thp_mode_always)); -} - /* * Tries to satisfy the given allocation request by reusing one of the extents * in the given extents_t. @@ -1159,9 +1092,7 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent, growing_retained); return NULL; } - if (!extent_need_manual_zero(arena)) { - extent_zeroed_set(extent, true); - } + extent_zeroed_set(extent, true); } if (extent_committed_get(extent)) { @@ -1182,16 +1113,14 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, if (*zero) { void *addr = extent_base_get(extent); + size_t size = extent_size_get(extent); if (!extent_zeroed_get(extent)) { - size_t size = extent_size_get(extent); - if (extent_need_manual_zero(arena) || - pages_purge_forced(addr, size)) { + if (pages_purge_forced(addr, size)) { memset(addr, 0, size); } } else if (config_debug) { size_t *p = (size_t *)(uintptr_t)addr; - /* Check the first page only. */ - for (size_t i = 0; i < PAGE / sizeof(size_t); i++) { + for (size_t i = 0; i < size / sizeof(size_t); i++) { assert(p[i] == 0); } } @@ -1262,7 +1191,7 @@ extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, assert(arena != NULL); return extent_alloc_default_impl(tsdn, arena, new_addr, size, - ALIGNMENT_CEILING(alignment, PAGE), zero, commit); + alignment, zero, commit); } static void @@ -1315,11 +1244,11 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); while (alloc_size < alloc_size_min) { egn_skip++; - if (arena->extent_grow_next + egn_skip >= - sz_psz2ind(SC_LARGE_MAXCLASS)) { + if (arena->extent_grow_next + egn_skip == NPSIZES) { /* Outside legal range. */ goto label_err; } + assert(arena->extent_grow_next + egn_skip < NPSIZES); alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); } @@ -1342,16 +1271,17 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, extent_hook_post_reentrancy(tsdn); } - extent_init(extent, arena, ptr, alloc_size, false, SC_NSIZES, + extent_init(extent, arena, ptr, alloc_size, false, NSIZES, arena_extent_sn_next(arena), extent_state_active, zeroed, - committed, true, EXTENT_IS_HEAD); + committed, true); if (ptr == NULL) { extent_dalloc(tsdn, arena, extent); goto label_err; } if (extent_register_no_gdump_add(tsdn, extent)) { - extent_dalloc(tsdn, arena, extent); + extents_leak(tsdn, arena, r_extent_hooks, + &arena->extents_retained, extent, true); goto label_err; } @@ -1398,7 +1328,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, } if (to_leak != NULL) { extent_deregister_no_gdump_sub(tsdn, to_leak); - extents_abandon_vm(tsdn, arena, r_extent_hooks, + extents_leak(tsdn, arena, r_extent_hooks, &arena->extents_retained, to_leak, true); } goto label_err; @@ -1411,9 +1341,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, &arena->extents_retained, extent, true); goto label_err; } - if (!extent_need_manual_zero(arena)) { - extent_zeroed_set(extent, true); - } + extent_zeroed_set(extent, true); } /* @@ -1447,8 +1375,7 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, if (*zero && !extent_zeroed_get(extent)) { void *addr = extent_base_get(extent); size_t size = extent_size_get(extent); - if (extent_need_manual_zero(arena) || - pages_purge_forced(addr, size)) { + if (pages_purge_forced(addr, size)) { memset(addr, 0, size); } } @@ -1498,15 +1425,14 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, return NULL; } void *addr; - size_t palignment = ALIGNMENT_CEILING(alignment, PAGE); if (*r_extent_hooks == &extent_hooks_default) { /* Call directly to propagate tsdn. */ addr = extent_alloc_default_impl(tsdn, arena, new_addr, esize, - palignment, zero, commit); + alignment, zero, commit); } else { extent_hook_pre_reentrancy(tsdn, arena); addr = (*r_extent_hooks)->alloc(*r_extent_hooks, new_addr, - esize, palignment, zero, commit, arena_ind_get(arena)); + esize, alignment, zero, commit, arena_ind_get(arena)); extent_hook_post_reentrancy(tsdn); } if (addr == NULL) { @@ -1515,12 +1441,13 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, } extent_init(extent, arena, addr, esize, slab, szind, arena_extent_sn_next(arena), extent_state_active, *zero, *commit, - true, EXTENT_NOT_HEAD); + true); if (pad != 0) { extent_addr_randomize(tsdn, extent, alignment); } if (extent_register(tsdn, extent)) { - extent_dalloc(tsdn, arena, extent); + extents_leak(tsdn, arena, r_extent_hooks, + &arena->extents_retained, extent, false); return NULL; } @@ -1597,15 +1524,9 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, } static extent_t * -extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, +extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, - extent_t *extent, bool *coalesced, bool growing_retained, - bool inactive_only) { - /* - * We avoid checking / locking inactive neighbors for large size - * classes, since they are eagerly coalesced on deallocation which can - * cause lock contention. - */ + extent_t *extent, bool *coalesced, bool growing_retained) { /* * Continue attempting to coalesce until failure, to protect against * races with other threads that are thwarted by this one. @@ -1616,7 +1537,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, /* Try to coalesce forward. */ extent_t *next = extent_lock_from_addr(tsdn, rtree_ctx, - extent_past_get(extent), inactive_only); + extent_past_get(extent)); if (next != NULL) { /* * extents->mtx only protects against races for @@ -1642,7 +1563,7 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, /* Try to coalesce backward. */ extent_t *prev = extent_lock_from_addr(tsdn, rtree_ctx, - extent_before_get(extent), inactive_only); + extent_before_get(extent)); if (prev != NULL) { bool can_coalesce = extent_can_coalesce(arena, extents, extent, prev); @@ -1668,22 +1589,6 @@ extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, return extent; } -static extent_t * -extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, - extent_t *extent, bool *coalesced, bool growing_retained) { - return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx, - extents, extent, coalesced, growing_retained, false); -} - -static extent_t * -extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, - extent_t *extent, bool *coalesced, bool growing_retained) { - return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx, - extents, extent, coalesced, growing_retained, true); -} - /* * Does the metadata management portions of putting an unused extent into the * given extents_t (coalesces, deregisters slab interiors, the heap operations). @@ -1701,7 +1606,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, malloc_mutex_lock(tsdn, &extents->mtx); extent_hooks_assure_initialized(arena, r_extent_hooks); - extent_szind_set(extent, SC_NSIZES); + extent_szind_set(extent, NSIZES); if (extent_slab_get(extent)) { extent_interior_deregister(tsdn, rtree_ctx, extent); extent_slab_set(extent, false); @@ -1713,22 +1618,18 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, if (!extents->delay_coalesce) { extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, rtree_ctx, extents, extent, NULL, growing_retained); - } else if (extent_size_get(extent) >= SC_LARGE_MINCLASS) { - assert(extents == &arena->extents_dirty); + } else if (extent_size_get(extent) >= LARGE_MINCLASS) { /* Always coalesce large extents eagerly. */ bool coalesced; + size_t prev_size; do { + prev_size = extent_size_get(extent); assert(extent_state_get(extent) == extent_state_active); - extent = extent_try_coalesce_large(tsdn, arena, + extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, rtree_ctx, extents, extent, &coalesced, growing_retained); - } while (coalesced); - if (extent_size_get(extent) >= oversize_threshold) { - /* Shortcut to purge the oversize extent eagerly. */ - malloc_mutex_unlock(tsdn, &extents->mtx); - arena_decay_extent(tsdn, arena, r_extent_hooks, extent); - return; - } + } while (coalesced && + extent_size_get(extent) >= prev_size + LARGE_MINCLASS); } extent_deactivate_locked(tsdn, arena, extents, extent); @@ -1743,18 +1644,13 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { WITNESS_RANK_CORE, 0); if (extent_register(tsdn, extent)) { - extent_dalloc(tsdn, arena, extent); + extents_leak(tsdn, arena, &extent_hooks, + &arena->extents_retained, extent, false); return; } extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent); } -static bool -extent_may_dalloc(void) { - /* With retain enabled, the default dalloc always fails. */ - return !opt_retain; -} - static bool extent_dalloc_default_impl(void *addr, size_t size) { if (!have_dss || !extent_in_dss(addr)) { @@ -1810,20 +1706,16 @@ extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - /* Avoid calling the default extent_dalloc unless have to. */ - if (*r_extent_hooks != &extent_hooks_default || extent_may_dalloc()) { - /* - * Deregister first to avoid a race with other allocating - * threads, and reregister if deallocation fails. - */ - extent_deregister(tsdn, extent); - if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, - extent)) { - return; - } - extent_reregister(tsdn, extent); + /* + * Deregister first to avoid a race with other allocating threads, and + * reregister if deallocation fails. + */ + extent_deregister(tsdn, extent); + if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent)) { + return; } + extent_reregister(tsdn, extent); if (*r_extent_hooks != &extent_hooks_default) { extent_hook_pre_reentrancy(tsdn, arena); } @@ -2063,20 +1955,13 @@ extent_purge_forced_wrapper(tsdn_t *tsdn, arena_t *arena, offset, length, false); } +#ifdef JEMALLOC_MAPS_COALESCE static bool extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { - if (!maps_coalesce) { - /* - * Without retain, only whole regions can be purged (required by - * MEM_RELEASE on Windows) -- therefore disallow splitting. See - * comments in extent_head_no_merge(). - */ - return !opt_retain; - } - - return false; + return !maps_coalesce; } +#endif /* * Accepts the extent to split, and the characteristics of each side of the @@ -2108,8 +1993,7 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) + size_a), size_b, slab_b, szind_b, extent_sn_get(extent), extent_state_get(extent), extent_zeroed_get(extent), - extent_committed_get(extent), extent_dumpable_get(extent), - EXTENT_NOT_HEAD); + extent_committed_get(extent), extent_dumpable_get(extent)); rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); @@ -2120,8 +2004,7 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_init(&lead, arena, extent_addr_get(extent), size_a, slab_a, szind_a, extent_sn_get(extent), extent_state_get(extent), extent_zeroed_get(extent), - extent_committed_get(extent), extent_dumpable_get(extent), - EXTENT_NOT_HEAD); + extent_committed_get(extent), extent_dumpable_get(extent)); extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false, true, &lead_elm_a, &lead_elm_b); @@ -2179,7 +2062,7 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, static bool extent_merge_default_impl(void *addr_a, void *addr_b) { - if (!maps_coalesce && !opt_retain) { + if (!maps_coalesce) { return true; } if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) { @@ -2189,51 +2072,13 @@ extent_merge_default_impl(void *addr_a, void *addr_b) { return false; } -/* - * Returns true if the given extents can't be merged because of their head bit - * settings. Assumes the second extent has the higher address. - */ -static bool -extent_head_no_merge(extent_t *a, extent_t *b) { - assert(extent_base_get(a) < extent_base_get(b)); - /* - * When coalesce is not always allowed (Windows), only merge extents - * from the same VirtualAlloc region under opt.retain (in which case - * MEM_DECOMMIT is utilized for purging). - */ - if (maps_coalesce) { - return false; - } - if (!opt_retain) { - return true; - } - /* If b is a head extent, disallow the cross-region merge. */ - if (extent_is_head_get(b)) { - /* - * Additionally, sn should not overflow with retain; sanity - * check that different regions have unique sn. - */ - assert(extent_sn_comp(a, b) != 0); - return true; - } - assert(extent_sn_comp(a, b) == 0); - - return false; -} - +#ifdef JEMALLOC_MAPS_COALESCE static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { - if (!maps_coalesce) { - tsdn_t *tsdn = tsdn_fetch(); - extent_t *a = iealloc(tsdn, addr_a); - extent_t *b = iealloc(tsdn, addr_b); - if (extent_head_no_merge(a, b)) { - return true; - } - } return extent_merge_default_impl(addr_a, addr_b); } +#endif static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, @@ -2241,11 +2086,10 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); - assert(extent_base_get(a) < extent_base_get(b)); extent_hooks_assure_initialized(arena, r_extent_hooks); - if ((*r_extent_hooks)->merge == NULL || extent_head_no_merge(a, b)) { + if ((*r_extent_hooks)->merge == NULL) { return true; } @@ -2284,23 +2128,22 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, if (a_elm_b != NULL) { rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL, - SC_NSIZES, false); + NSIZES, false); } if (b_elm_b != NULL) { rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL, - SC_NSIZES, false); + NSIZES, false); } else { b_elm_b = b_elm_a; } extent_size_set(a, extent_size_get(a) + extent_size_get(b)); - extent_szind_set(a, SC_NSIZES); + extent_szind_set(a, NSIZES); extent_sn_set(a, (extent_sn_get(a) < extent_sn_get(b)) ? extent_sn_get(a) : extent_sn_get(b)); extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b)); - extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, SC_NSIZES, - false); + extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, NSIZES, false); extent_unlock2(tsdn, a, b); @@ -2332,72 +2175,3 @@ extent_boot(void) { return false; } - -void -extent_util_stats_get(tsdn_t *tsdn, const void *ptr, - size_t *nfree, size_t *nregs, size_t *size) { - assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL); - - const extent_t *extent = iealloc(tsdn, ptr); - if (unlikely(extent == NULL)) { - *nfree = *nregs = *size = 0; - return; - } - - *size = extent_size_get(extent); - if (!extent_slab_get(extent)) { - *nfree = 0; - *nregs = 1; - } else { - *nfree = extent_nfree_get(extent); - *nregs = bin_infos[extent_szind_get(extent)].nregs; - assert(*nfree <= *nregs); - assert(*nfree * extent_usize_get(extent) <= *size); - } -} - -void -extent_util_stats_verbose_get(tsdn_t *tsdn, const void *ptr, - size_t *nfree, size_t *nregs, size_t *size, - size_t *bin_nfree, size_t *bin_nregs, void **slabcur_addr) { - assert(ptr != NULL && nfree != NULL && nregs != NULL && size != NULL - && bin_nfree != NULL && bin_nregs != NULL && slabcur_addr != NULL); - - const extent_t *extent = iealloc(tsdn, ptr); - if (unlikely(extent == NULL)) { - *nfree = *nregs = *size = *bin_nfree = *bin_nregs = 0; - *slabcur_addr = NULL; - return; - } - - *size = extent_size_get(extent); - if (!extent_slab_get(extent)) { - *nfree = *bin_nfree = *bin_nregs = 0; - *nregs = 1; - *slabcur_addr = NULL; - return; - } - - *nfree = extent_nfree_get(extent); - const szind_t szind = extent_szind_get(extent); - *nregs = bin_infos[szind].nregs; - assert(*nfree <= *nregs); - assert(*nfree * extent_usize_get(extent) <= *size); - - const arena_t *arena = extent_arena_get(extent); - assert(arena != NULL); - const unsigned binshard = extent_binshard_get(extent); - bin_t *bin = &arena->bins[szind].bin_shards[binshard]; - - malloc_mutex_lock(tsdn, &bin->lock); - if (config_stats) { - *bin_nregs = *nregs * bin->stats.curslabs; - assert(*bin_nregs >= bin->stats.curregs); - *bin_nfree = *bin_nregs - bin->stats.curregs; - } else { - *bin_nfree = *bin_nregs = 0; - } - *slabcur_addr = extent_addr_get(bin->slabcur); - assert(*slabcur_addr != NULL); - malloc_mutex_unlock(tsdn, &bin->lock); -} diff --git a/deps/jemalloc/src/extent_dss.c b/deps/jemalloc/src/extent_dss.c index 8581789..2b1ea9c 100644 --- a/deps/jemalloc/src/extent_dss.c +++ b/deps/jemalloc/src/extent_dss.c @@ -113,7 +113,7 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, cassert(have_dss); assert(size > 0); - assert(alignment == ALIGNMENT_CEILING(alignment, PAGE)); + assert(alignment > 0); /* * sbrk() uses a signed increment argument, so take care not to @@ -154,10 +154,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, (uintptr_t)gap_addr_page; if (gap_size_page != 0) { extent_init(gap, arena, gap_addr_page, - gap_size_page, false, SC_NSIZES, + gap_size_page, false, NSIZES, arena_extent_sn_next(arena), - extent_state_active, false, true, true, - EXTENT_NOT_HEAD); + extent_state_active, false, true, true); } /* * Compute the address just past the end of the desired @@ -199,9 +198,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, extent_t extent; extent_init(&extent, arena, ret, size, - size, false, SC_NSIZES, + size, false, NSIZES, extent_state_active, false, true, - true, EXTENT_NOT_HEAD); + true); if (extent_purge_forced_wrapper(tsdn, arena, &extent_hooks, &extent, 0, size)) { diff --git a/deps/jemalloc/src/extent_mmap.c b/deps/jemalloc/src/extent_mmap.c index 17fd1c8..8d607dc 100644 --- a/deps/jemalloc/src/extent_mmap.c +++ b/deps/jemalloc/src/extent_mmap.c @@ -21,8 +21,8 @@ bool opt_retain = void * extent_alloc_mmap(void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit) { - assert(alignment == ALIGNMENT_CEILING(alignment, PAGE)); - void *ret = pages_map(new_addr, size, alignment, commit); + void *ret = pages_map(new_addr, size, ALIGNMENT_CEILING(alignment, + PAGE), commit); if (ret == NULL) { return NULL; } diff --git a/deps/jemalloc/src/hook.c b/deps/jemalloc/src/hook.c deleted file mode 100644 index 9ac703c..0000000 --- a/deps/jemalloc/src/hook.c +++ /dev/null @@ -1,195 +0,0 @@ -#include "jemalloc/internal/jemalloc_preamble.h" - -#include "jemalloc/internal/hook.h" - -#include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/seq.h" - -typedef struct hooks_internal_s hooks_internal_t; -struct hooks_internal_s { - hooks_t hooks; - bool in_use; -}; - -seq_define(hooks_internal_t, hooks) - -static atomic_u_t nhooks = ATOMIC_INIT(0); -static seq_hooks_t hooks[HOOK_MAX]; -static malloc_mutex_t hooks_mu; - -bool -hook_boot() { - return malloc_mutex_init(&hooks_mu, "hooks", WITNESS_RANK_HOOK, - malloc_mutex_rank_exclusive); -} - -static void * -hook_install_locked(hooks_t *to_install) { - hooks_internal_t hooks_internal; - for (int i = 0; i < HOOK_MAX; i++) { - bool success = seq_try_load_hooks(&hooks_internal, &hooks[i]); - /* We hold mu; no concurrent access. */ - assert(success); - if (!hooks_internal.in_use) { - hooks_internal.hooks = *to_install; - hooks_internal.in_use = true; - seq_store_hooks(&hooks[i], &hooks_internal); - atomic_store_u(&nhooks, - atomic_load_u(&nhooks, ATOMIC_RELAXED) + 1, - ATOMIC_RELAXED); - return &hooks[i]; - } - } - return NULL; -} - -void * -hook_install(tsdn_t *tsdn, hooks_t *to_install) { - malloc_mutex_lock(tsdn, &hooks_mu); - void *ret = hook_install_locked(to_install); - if (ret != NULL) { - tsd_global_slow_inc(tsdn); - } - malloc_mutex_unlock(tsdn, &hooks_mu); - return ret; -} - -static void -hook_remove_locked(seq_hooks_t *to_remove) { - hooks_internal_t hooks_internal; - bool success = seq_try_load_hooks(&hooks_internal, to_remove); - /* We hold mu; no concurrent access. */ - assert(success); - /* Should only remove hooks that were added. */ - assert(hooks_internal.in_use); - hooks_internal.in_use = false; - seq_store_hooks(to_remove, &hooks_internal); - atomic_store_u(&nhooks, atomic_load_u(&nhooks, ATOMIC_RELAXED) - 1, - ATOMIC_RELAXED); -} - -void -hook_remove(tsdn_t *tsdn, void *opaque) { - if (config_debug) { - char *hooks_begin = (char *)&hooks[0]; - char *hooks_end = (char *)&hooks[HOOK_MAX]; - char *hook = (char *)opaque; - assert(hooks_begin <= hook && hook < hooks_end - && (hook - hooks_begin) % sizeof(seq_hooks_t) == 0); - } - malloc_mutex_lock(tsdn, &hooks_mu); - hook_remove_locked((seq_hooks_t *)opaque); - tsd_global_slow_dec(tsdn); - malloc_mutex_unlock(tsdn, &hooks_mu); -} - -#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr) \ -for (int for_each_hook_counter = 0; \ - for_each_hook_counter < HOOK_MAX; \ - for_each_hook_counter++) { \ - bool for_each_hook_success = seq_try_load_hooks( \ - (hooks_internal_ptr), &hooks[for_each_hook_counter]); \ - if (!for_each_hook_success) { \ - continue; \ - } \ - if (!(hooks_internal_ptr)->in_use) { \ - continue; \ - } -#define FOR_EACH_HOOK_END \ -} - -static bool * -hook_reentrantp() { - /* - * We prevent user reentrancy within hooks. This is basically just a - * thread-local bool that triggers an early-exit. - * - * We don't fold in_hook into reentrancy. There are two reasons for - * this: - * - Right now, we turn on reentrancy during things like extent hook - * execution. Allocating during extent hooks is not officially - * supported, but we don't want to break it for the time being. These - * sorts of allocations should probably still be hooked, though. - * - If a hook allocates, we may want it to be relatively fast (after - * all, it executes on every allocator operation). Turning on - * reentrancy is a fairly heavyweight mode (disabling tcache, - * redirecting to arena 0, etc.). It's possible we may one day want - * to turn on reentrant mode here, if it proves too difficult to keep - * this working. But that's fairly easy for us to see; OTOH, people - * not using hooks because they're too slow is easy for us to miss. - * - * The tricky part is - * that this code might get invoked even if we don't have access to tsd. - * This function mimics getting a pointer to thread-local data, except - * that it might secretly return a pointer to some global data if we - * know that the caller will take the early-exit path. - * If we return a bool that indicates that we are reentrant, then the - * caller will go down the early exit path, leaving the global - * untouched. - */ - static bool in_hook_global = true; - tsdn_t *tsdn = tsdn_fetch(); - tcache_t *tcache = tsdn_tcachep_get(tsdn); - if (tcache != NULL) { - return &tcache->in_hook; - } - return &in_hook_global; -} - -#define HOOK_PROLOGUE \ - if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) { \ - return; \ - } \ - bool *in_hook = hook_reentrantp(); \ - if (*in_hook) { \ - return; \ - } \ - *in_hook = true; - -#define HOOK_EPILOGUE \ - *in_hook = false; - -void -hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, - uintptr_t args_raw[3]) { - HOOK_PROLOGUE - - hooks_internal_t hook; - FOR_EACH_HOOK_BEGIN(&hook) - hook_alloc h = hook.hooks.alloc_hook; - if (h != NULL) { - h(hook.hooks.extra, type, result, result_raw, args_raw); - } - FOR_EACH_HOOK_END - - HOOK_EPILOGUE -} - -void -hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) { - HOOK_PROLOGUE - hooks_internal_t hook; - FOR_EACH_HOOK_BEGIN(&hook) - hook_dalloc h = hook.hooks.dalloc_hook; - if (h != NULL) { - h(hook.hooks.extra, type, address, args_raw); - } - FOR_EACH_HOOK_END - HOOK_EPILOGUE -} - -void -hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, - size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]) { - HOOK_PROLOGUE - hooks_internal_t hook; - FOR_EACH_HOOK_BEGIN(&hook) - hook_expand h = hook.hooks.expand_hook; - if (h != NULL) { - h(hook.hooks.extra, type, address, old_usize, new_usize, - result_raw, args_raw); - } - FOR_EACH_HOOK_END - HOOK_EPILOGUE -} diff --git a/deps/jemalloc/src/test_hooks.c b/deps/jemalloc/src/hooks.c similarity index 79% rename from deps/jemalloc/src/test_hooks.c rename to deps/jemalloc/src/hooks.c index ace00d9..6266ecd 100644 --- a/deps/jemalloc/src/test_hooks.c +++ b/deps/jemalloc/src/hooks.c @@ -6,7 +6,7 @@ * from outside the generated library, so that we can use them in test code. */ JEMALLOC_EXPORT -void (*test_hooks_arena_new_hook)() = NULL; +void (*hooks_arena_new_hook)() = NULL; JEMALLOC_EXPORT -void (*test_hooks_libc_hook)() = NULL; +void (*hooks_libc_hook)() = NULL; diff --git a/deps/jemalloc/src/jemalloc.c b/deps/jemalloc/src/jemalloc.c index ed13718..585645a 100644 --- a/deps/jemalloc/src/jemalloc.c +++ b/deps/jemalloc/src/jemalloc.c @@ -7,14 +7,12 @@ #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_mmap.h" -#include "jemalloc/internal/hook.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/log.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/safety_check.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/spin.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" @@ -43,8 +41,6 @@ bool opt_abort_conf = false #endif ; -/* Intentionally default off, even with debug builds. */ -bool opt_confirm_conf = false; const char *opt_junk = #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) "true" @@ -89,10 +85,8 @@ malloc_mutex_t arenas_lock; JEMALLOC_ALIGNED(CACHELINE) atomic_p_t arenas[MALLOCX_ARENA_LIMIT]; static atomic_u_t narenas_total; /* Use narenas_total_*(). */ -/* Below three are read-only after initialization. */ -static arena_t *a0; /* arenas[0]. */ -unsigned narenas_auto; -unsigned manual_arena_base; +static arena_t *a0; /* arenas[0]; read-only after initialization. */ +unsigned narenas_auto; /* Read-only after initialization. */ typedef enum { malloc_init_uninitialized = 3, @@ -332,7 +326,7 @@ arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { */ arena = arena_get(tsdn, ind, false); if (arena != NULL) { - assert(arena_is_auto(arena)); + assert(ind < narenas_auto); return arena; } @@ -347,12 +341,12 @@ arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) { if (ind == 0) { return; } - /* - * Avoid creating a new background thread just for the huge arena, which - * purges eagerly by default. - */ - if (have_background_thread && !arena_is_huge(ind)) { - if (background_thread_create(tsdn_tsd(tsdn), ind)) { + if (have_background_thread) { + bool err; + malloc_mutex_lock(tsdn, &background_thread_lock); + err = background_thread_create(tsdn_tsd(tsdn), ind); + malloc_mutex_unlock(tsdn, &background_thread_lock); + if (err) { malloc_printf(": error in background thread " "creation for arena %u. Abort.\n", ind); abort(); @@ -382,14 +376,6 @@ arena_bind(tsd_t *tsd, unsigned ind, bool internal) { tsd_iarena_set(tsd, arena); } else { tsd_arena_set(tsd, arena); - unsigned shard = atomic_fetch_add_u(&arena->binshard_next, 1, - ATOMIC_RELAXED); - tsd_binshards_t *bins = tsd_binshardsp_get(tsd); - for (unsigned i = 0; i < SC_NBINS; i++) { - assert(bin_infos[i].n_shards > 0 && - bin_infos[i].n_shards <= BIN_SHARDS_MAX); - bins->binshard[i] = shard % bin_infos[i].n_shards; - } } } @@ -775,50 +761,6 @@ init_opt_stats_print_opts(const char *v, size_t vlen) { assert(opts_len == strlen(opt_stats_print_opts)); } -/* Reads the next size pair in a multi-sized option. */ -static bool -malloc_conf_multi_sizes_next(const char **slab_size_segment_cur, - size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) { - const char *cur = *slab_size_segment_cur; - char *end; - uintmax_t um; - - set_errno(0); - - /* First number, then '-' */ - um = malloc_strtoumax(cur, &end, 0); - if (get_errno() != 0 || *end != '-') { - return true; - } - *slab_start = (size_t)um; - cur = end + 1; - - /* Second number, then ':' */ - um = malloc_strtoumax(cur, &end, 0); - if (get_errno() != 0 || *end != ':') { - return true; - } - *slab_end = (size_t)um; - cur = end + 1; - - /* Last number */ - um = malloc_strtoumax(cur, &end, 0); - if (get_errno() != 0) { - return true; - } - *new_size = (size_t)um; - - /* Consume the separator if there is one. */ - if (*end == '|') { - end++; - } - - *vlen_left -= end - *slab_size_segment_cur; - *slab_size_segment_cur = end; - - return false; -} - static bool malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, char const **v_p, size_t *vlen_p) { @@ -908,11 +850,6 @@ malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, malloc_printf(": %s: %.*s:%.*s\n", msg, (int)klen, k, (int)vlen, v); /* If abort_conf is set, error out after processing all options. */ - const char *experimental = "experimental_"; - if (strncmp(k, experimental, strlen(experimental)) == 0) { - /* However, tolerate experimental features. */ - return; - } had_conf_error = true; } @@ -931,141 +868,88 @@ malloc_slow_flag_init(void) { malloc_slow = (malloc_slow_flags != 0); } -/* Number of sources for initializing malloc_conf */ -#define MALLOC_CONF_NSOURCES 4 - -static const char * -obtain_malloc_conf(unsigned which_source, char buf[PATH_MAX + 1]) { - if (config_debug) { - static unsigned read_source = 0; - /* - * Each source should only be read once, to minimize # of - * syscalls on init. - */ - assert(read_source++ == which_source); - } - assert(which_source < MALLOC_CONF_NSOURCES); - - const char *ret; - switch (which_source) { - case 0: - ret = config_malloc_conf; - break; - case 1: - if (je_malloc_conf != NULL) { - /* Use options that were compiled into the program. */ - ret = je_malloc_conf; - } else { - /* No configuration specified. */ - ret = NULL; - } - break; - case 2: { - ssize_t linklen = 0; -#ifndef _WIN32 - int saved_errno = errno; - const char *linkname = -# ifdef JEMALLOC_PREFIX - "/etc/"JEMALLOC_PREFIX"malloc.conf" -# else - "/etc/malloc.conf" -# endif - ; - - /* - * Try to use the contents of the "/etc/malloc.conf" symbolic - * link's name. - */ -#ifndef JEMALLOC_READLINKAT - linklen = readlink(linkname, buf, PATH_MAX); -#else - linklen = readlinkat(AT_FDCWD, linkname, buf, PATH_MAX); -#endif - if (linklen == -1) { - /* No configuration specified. */ - linklen = 0; - /* Restore errno. */ - set_errno(saved_errno); - } -#endif - buf[linklen] = '\0'; - ret = buf; - break; - } case 3: { - const char *envname = -#ifdef JEMALLOC_PREFIX - JEMALLOC_CPREFIX"MALLOC_CONF" -#else - "MALLOC_CONF" -#endif - ; - - if ((ret = jemalloc_secure_getenv(envname)) != NULL) { - /* - * Do nothing; opts is already initialized to the value - * of the MALLOC_CONF environment variable. - */ - } else { - /* No configuration specified. */ - ret = NULL; - } - break; - } default: - not_reached(); - ret = NULL; - } - return ret; -} - static void -malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], - bool initial_call, const char *opts_cache[MALLOC_CONF_NSOURCES], - char buf[PATH_MAX + 1]) { - static const char *opts_explain[MALLOC_CONF_NSOURCES] = { - "string specified via --with-malloc-conf", - "string pointed to by the global variable malloc_conf", - "\"name\" of the file referenced by the symbolic link named " - "/etc/malloc.conf", - "value of the environment variable MALLOC_CONF" - }; +malloc_conf_init(void) { unsigned i; + char buf[PATH_MAX + 1]; const char *opts, *k, *v; size_t klen, vlen; - for (i = 0; i < MALLOC_CONF_NSOURCES; i++) { + for (i = 0; i < 4; i++) { /* Get runtime configuration. */ - if (initial_call) { - opts_cache[i] = obtain_malloc_conf(i, buf); - } - opts = opts_cache[i]; - if (!initial_call && opt_confirm_conf) { - malloc_printf( - ": malloc_conf #%u (%s): \"%s\"\n", - i + 1, opts_explain[i], opts != NULL ? opts : ""); - } - if (opts == NULL) { - continue; + switch (i) { + case 0: + opts = config_malloc_conf; + break; + case 1: + if (je_malloc_conf != NULL) { + /* + * Use options that were compiled into the + * program. + */ + opts = je_malloc_conf; + } else { + /* No configuration specified. */ + buf[0] = '\0'; + opts = buf; + } + break; + case 2: { + ssize_t linklen = 0; +#ifndef _WIN32 + int saved_errno = errno; + const char *linkname = +# ifdef JEMALLOC_PREFIX + "/etc/"JEMALLOC_PREFIX"malloc.conf" +# else + "/etc/malloc.conf" +# endif + ; + + /* + * Try to use the contents of the "/etc/malloc.conf" + * symbolic link's name. + */ + linklen = readlink(linkname, buf, sizeof(buf) - 1); + if (linklen == -1) { + /* No configuration specified. */ + linklen = 0; + /* Restore errno. */ + set_errno(saved_errno); + } +#endif + buf[linklen] = '\0'; + opts = buf; + break; + } case 3: { + const char *envname = +#ifdef JEMALLOC_PREFIX + JEMALLOC_CPREFIX"MALLOC_CONF" +#else + "MALLOC_CONF" +#endif + ; + + if ((opts = jemalloc_secure_getenv(envname)) != NULL) { + /* + * Do nothing; opts is already initialized to + * the value of the MALLOC_CONF environment + * variable. + */ + } else { + /* No configuration specified. */ + buf[0] = '\0'; + opts = buf; + } + break; + } default: + not_reached(); + buf[0] = '\0'; + opts = buf; } while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, &vlen)) { - -#define CONF_ERROR(msg, k, klen, v, vlen) \ - if (!initial_call) { \ - malloc_conf_error( \ - msg, k, klen, v, vlen); \ - cur_opt_valid = false; \ - } -#define CONF_CONTINUE { \ - if (!initial_call && opt_confirm_conf \ - && cur_opt_valid) { \ - malloc_printf(": -- " \ - "Set conf value: %.*s:%.*s" \ - "\n", (int)klen, k, \ - (int)vlen, v); \ - } \ - continue; \ - } #define CONF_MATCH(n) \ (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) #define CONF_MATCH_VALUE(n) \ @@ -1077,23 +961,16 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], } else if (CONF_MATCH_VALUE("false")) { \ o = false; \ } else { \ - CONF_ERROR("Invalid conf value",\ + malloc_conf_error( \ + "Invalid conf value", \ k, klen, v, vlen); \ } \ - CONF_CONTINUE; \ + continue; \ } - /* - * One of the CONF_MIN macros below expands, in one of the use points, - * to "unsigned integer < 0", which is always false, triggering the - * GCC -Wtype-limits warning, which we disable here and re-enable below. - */ - JEMALLOC_DIAGNOSTIC_PUSH - JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS - -#define CONF_DONT_CHECK_MIN(um, min) false -#define CONF_CHECK_MIN(um, min) ((um) < (min)) -#define CONF_DONT_CHECK_MAX(um, max) false -#define CONF_CHECK_MAX(um, max) ((um) > (max)) +#define CONF_MIN_no(um, min) false +#define CONF_MIN_yes(um, min) ((um) < (min)) +#define CONF_MAX_no(um, max) false +#define CONF_MAX_yes(um, max) ((um) > (max)) #define CONF_HANDLE_T_U(t, o, n, min, max, check_min, check_max, clip) \ if (CONF_MATCH(n)) { \ uintmax_t um; \ @@ -1103,21 +980,26 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], um = malloc_strtoumax(v, &end, 0); \ if (get_errno() != 0 || (uintptr_t)end -\ (uintptr_t)v != vlen) { \ - CONF_ERROR("Invalid conf value",\ + malloc_conf_error( \ + "Invalid conf value", \ k, klen, v, vlen); \ } else if (clip) { \ - if (check_min(um, (t)(min))) { \ + if (CONF_MIN_##check_min(um, \ + (t)(min))) { \ o = (t)(min); \ } else if ( \ - check_max(um, (t)(max))) { \ + CONF_MAX_##check_max(um, \ + (t)(max))) { \ o = (t)(max); \ } else { \ o = (t)um; \ } \ } else { \ - if (check_min(um, (t)(min)) || \ - check_max(um, (t)(max))) { \ - CONF_ERROR( \ + if (CONF_MIN_##check_min(um, \ + (t)(min)) || \ + CONF_MAX_##check_max(um, \ + (t)(max))) { \ + malloc_conf_error( \ "Out-of-range " \ "conf value", \ k, klen, v, vlen); \ @@ -1125,7 +1007,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], o = (t)um; \ } \ } \ - CONF_CONTINUE; \ + continue; \ } #define CONF_HANDLE_UNSIGNED(o, n, min, max, check_min, check_max, \ clip) \ @@ -1143,17 +1025,18 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], l = strtol(v, &end, 0); \ if (get_errno() != 0 || (uintptr_t)end -\ (uintptr_t)v != vlen) { \ - CONF_ERROR("Invalid conf value",\ + malloc_conf_error( \ + "Invalid conf value", \ k, klen, v, vlen); \ } else if (l < (ssize_t)(min) || l > \ (ssize_t)(max)) { \ - CONF_ERROR( \ + malloc_conf_error( \ "Out-of-range conf value", \ k, klen, v, vlen); \ } else { \ o = l; \ } \ - CONF_CONTINUE; \ + continue; \ } #define CONF_HANDLE_CHAR_P(o, n, d) \ if (CONF_MATCH(n)) { \ @@ -1162,14 +1045,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], sizeof(o)-1; \ strncpy(o, v, cpylen); \ o[cpylen] = '\0'; \ - CONF_CONTINUE; \ - } - - bool cur_opt_valid = true; - - CONF_HANDLE_BOOL(opt_confirm_conf, "confirm_conf") - if (initial_call) { - continue; + continue; \ } CONF_HANDLE_BOOL(opt_abort, "abort") @@ -1186,10 +1062,10 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], } } if (!match) { - CONF_ERROR("Invalid conf value", + malloc_conf_error("Invalid conf value", k, klen, v, vlen); } - CONF_CONTINUE; + continue; } CONF_HANDLE_BOOL(opt_retain, "retain") if (strncmp("dss", k, klen) == 0) { @@ -1199,7 +1075,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], if (strncmp(dss_prec_names[i], v, vlen) == 0) { if (extent_dss_prec_set(i)) { - CONF_ERROR( + malloc_conf_error( "Error setting dss", k, klen, v, vlen); } else { @@ -1211,36 +1087,13 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], } } if (!match) { - CONF_ERROR("Invalid conf value", + malloc_conf_error("Invalid conf value", k, klen, v, vlen); } - CONF_CONTINUE; + continue; } CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, - UINT_MAX, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, - false) - if (CONF_MATCH("bin_shards")) { - const char *bin_shards_segment_cur = v; - size_t vlen_left = vlen; - do { - size_t size_start; - size_t size_end; - size_t nshards; - bool err = malloc_conf_multi_sizes_next( - &bin_shards_segment_cur, &vlen_left, - &size_start, &size_end, &nshards); - if (err || bin_update_shard_size( - bin_shard_sizes, size_start, - size_end, nshards)) { - CONF_ERROR( - "Invalid settings for " - "bin_shards", k, klen, v, - vlen); - break; - } - } while (vlen_left > 0); - CONF_CONTINUE; - } + UINT_MAX, yes, no, false) CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms, "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) : @@ -1252,7 +1105,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], CONF_HANDLE_BOOL(opt_stats_print, "stats_print") if (CONF_MATCH("stats_print_opts")) { init_opt_stats_print_opts(v, vlen); - CONF_CONTINUE; + continue; } if (config_fill) { if (CONF_MATCH("junk")) { @@ -1273,11 +1126,11 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], opt_junk_alloc = false; opt_junk_free = true; } else { - CONF_ERROR( - "Invalid conf value", - k, klen, v, vlen); + malloc_conf_error( + "Invalid conf value", k, + klen, v, vlen); } - CONF_CONTINUE; + continue; } CONF_HANDLE_BOOL(opt_zero, "zero") } @@ -1288,25 +1141,11 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") } CONF_HANDLE_BOOL(opt_tcache, "tcache") - CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max", - -1, (sizeof(size_t) << 3) - 1) - - /* - * The runtime option of oversize_threshold remains - * undocumented. It may be tweaked in the next major - * release (6.0). The default value 8M is rather - * conservative / safe. Tuning it further down may - * improve fragmentation a bit more, but may also cause - * contention on the huge arena. - */ - CONF_HANDLE_SIZE_T(opt_oversize_threshold, - "oversize_threshold", 0, SC_LARGE_MAXCLASS, - CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, false) CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit, "lg_extent_max_active_fit", 0, - (sizeof(size_t) << 3), CONF_DONT_CHECK_MIN, - CONF_CHECK_MAX, false) - + (sizeof(size_t) << 3), yes, yes, false) + CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max", + -1, (sizeof(size_t) << 3) - 1) if (strncmp("percpu_arena", k, klen) == 0) { bool match = false; for (int i = percpu_arena_mode_names_base; i < @@ -1314,7 +1153,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], if (strncmp(percpu_arena_mode_names[i], v, vlen) == 0) { if (!have_percpu_arena) { - CONF_ERROR( + malloc_conf_error( "No getcpu support", k, klen, v, vlen); } @@ -1324,42 +1163,17 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], } } if (!match) { - CONF_ERROR("Invalid conf value", + malloc_conf_error("Invalid conf value", k, klen, v, vlen); } - CONF_CONTINUE; + continue; } CONF_HANDLE_BOOL(opt_background_thread, "background_thread"); CONF_HANDLE_SIZE_T(opt_max_background_threads, "max_background_threads", 1, - opt_max_background_threads, - CONF_CHECK_MIN, CONF_CHECK_MAX, + opt_max_background_threads, yes, yes, true); - if (CONF_MATCH("slab_sizes")) { - bool err; - const char *slab_size_segment_cur = v; - size_t vlen_left = vlen; - do { - size_t slab_start; - size_t slab_end; - size_t pgs; - err = malloc_conf_multi_sizes_next( - &slab_size_segment_cur, - &vlen_left, &slab_start, &slab_end, - &pgs); - if (!err) { - sc_data_update_slab_size( - sc_data, slab_start, - slab_end, (int)pgs); - } else { - CONF_ERROR("Invalid settings " - "for slab_sizes", - k, klen, v, vlen); - } - } while (!err && vlen_left > 0); - CONF_CONTINUE; - } if (config_prof) { CONF_HANDLE_BOOL(opt_prof, "prof") CONF_HANDLE_CHAR_P(opt_prof_prefix, @@ -1369,8 +1183,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], "prof_thread_active_init") CONF_HANDLE_SIZE_T(opt_lg_prof_sample, "lg_prof_sample", 0, (sizeof(uint64_t) << 3) - - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX, - true) + - 1, no, yes, true) CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, "lg_prof_interval", -1, @@ -1378,7 +1191,6 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") CONF_HANDLE_BOOL(opt_prof_final, "prof_final") CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") - CONF_HANDLE_BOOL(opt_prof_log, "prof_log") } if (config_log) { if (CONF_MATCH("log")) { @@ -1387,7 +1199,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], vlen : sizeof(log_var_names) - 1); strncpy(log_var_names, v, cpylen); log_var_names[cpylen] = '\0'; - CONF_CONTINUE; + continue; } } if (CONF_MATCH("thp")) { @@ -1396,7 +1208,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], if (strncmp(thp_mode_names[i],v, vlen) == 0) { if (!have_madvise_huge) { - CONF_ERROR( + malloc_conf_error( "No THP support", k, klen, v, vlen); } @@ -1406,28 +1218,25 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], } } if (!match) { - CONF_ERROR("Invalid conf value", + malloc_conf_error("Invalid conf value", k, klen, v, vlen); } - CONF_CONTINUE; + continue; } - CONF_ERROR("Invalid conf pair", k, klen, v, vlen); -#undef CONF_ERROR -#undef CONF_CONTINUE + malloc_conf_error("Invalid conf pair", k, klen, v, + vlen); #undef CONF_MATCH #undef CONF_MATCH_VALUE #undef CONF_HANDLE_BOOL -#undef CONF_DONT_CHECK_MIN -#undef CONF_CHECK_MIN -#undef CONF_DONT_CHECK_MAX -#undef CONF_CHECK_MAX +#undef CONF_MIN_no +#undef CONF_MIN_yes +#undef CONF_MAX_no +#undef CONF_MAX_yes #undef CONF_HANDLE_T_U #undef CONF_HANDLE_UNSIGNED #undef CONF_HANDLE_SIZE_T #undef CONF_HANDLE_SSIZE_T #undef CONF_HANDLE_CHAR_P - /* Re-enable diagnostic "-Wtype-limits" */ - JEMALLOC_DIAGNOSTIC_POP } if (opt_abort_conf && had_conf_error) { malloc_abort_invalid_conf(); @@ -1436,19 +1245,6 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], atomic_store_b(&log_init_done, true, ATOMIC_RELEASE); } -static void -malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { - const char *opts_cache[MALLOC_CONF_NSOURCES] = {NULL, NULL, NULL, NULL}; - char buf[PATH_MAX + 1]; - - /* The first call only set the confirm_conf option and opts_cache */ - malloc_conf_init_helper(NULL, NULL, true, opts_cache, buf); - malloc_conf_init_helper(sc_data, bin_shard_sizes, false, opts_cache, - NULL); -} - -#undef MALLOC_CONF_NSOURCES - static bool malloc_init_hard_needed(void) { if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state == @@ -1479,33 +1275,10 @@ static bool malloc_init_hard_a0_locked() { malloc_initializer = INITIALIZER; - JEMALLOC_DIAGNOSTIC_PUSH - JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS - sc_data_t sc_data = {0}; - JEMALLOC_DIAGNOSTIC_POP - - /* - * Ordering here is somewhat tricky; we need sc_boot() first, since that - * determines what the size classes will be, and then - * malloc_conf_init(), since any slab size tweaking will need to be done - * before sz_boot and bin_boot, which assume that the values they read - * out of sc_data_global are final. - */ - sc_boot(&sc_data); - unsigned bin_shard_sizes[SC_NBINS]; - bin_shard_sizes_boot(bin_shard_sizes); - /* - * prof_boot0 only initializes opt_prof_prefix. We need to do it before - * we parse malloc_conf options, in case malloc_conf parsing overwrites - * it. - */ if (config_prof) { prof_boot0(); } - malloc_conf_init(&sc_data, bin_shard_sizes); - sz_boot(&sc_data); - bin_boot(&sc_data, bin_shard_sizes); - + malloc_conf_init(); if (opt_stats_print) { /* Print statistics at exit. */ if (atexit(stats_print_atexit) != 0) { @@ -1530,7 +1303,7 @@ malloc_init_hard_a0_locked() { if (config_prof) { prof_boot1(); } - arena_boot(&sc_data); + arena_boot(); if (tcache_boot(TSDN_NULL)) { return true; } @@ -1538,13 +1311,11 @@ malloc_init_hard_a0_locked() { malloc_mutex_rank_exclusive)) { return true; } - hook_boot(); /* * Create enough scaffolding to allow recursive allocation in * malloc_ncpus(). */ narenas_auto = 1; - manual_arena_base = narenas_auto + 1; memset(arenas, 0, sizeof(arena_t *) * narenas_auto); /* * Initialize one arena here. The rest are lazily created in @@ -1692,10 +1463,6 @@ malloc_init_narenas(void) { narenas_auto); } narenas_total_set(narenas_auto); - if (arena_init_huge()) { - narenas_total_inc(); - } - manual_arena_base = narenas_total_get(); return false; } @@ -1793,7 +1560,11 @@ malloc_init_hard(void) { * sets isthreaded) needs to be called without holding any lock. */ background_thread_ctl_init(tsd_tsdn(tsd)); - if (background_thread_create(tsd, 0)) { + + malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); + bool err = background_thread_create(tsd, 0); + malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); + if (err) { return true; } } @@ -1816,12 +1587,8 @@ typedef struct static_opts_s static_opts_t; struct static_opts_s { /* Whether or not allocation size may overflow. */ bool may_overflow; - - /* - * Whether or not allocations (with alignment) of size 0 should be - * treated as size 1. - */ - bool bump_empty_aligned_alloc; + /* Whether or not allocations of size 0 should be treated as size 1. */ + bool bump_empty_alloc; /* * Whether to assert that allocations are not of size 0 (after any * bumping). @@ -1854,16 +1621,12 @@ struct static_opts_s { * initialization) options. */ bool slow; - /* - * Return size. - */ - bool usize; }; JEMALLOC_ALWAYS_INLINE void static_opts_init(static_opts_t *static_opts) { static_opts->may_overflow = false; - static_opts->bump_empty_aligned_alloc = false; + static_opts->bump_empty_alloc = false; static_opts->assert_nonempty_alloc = false; static_opts->null_out_result_on_error = false; static_opts->set_errno_on_error = false; @@ -1871,7 +1634,6 @@ static_opts_init(static_opts_t *static_opts) { static_opts->oom_string = ""; static_opts->invalid_alignment_string = ""; static_opts->slow = false; - static_opts->usize = false; } /* @@ -1886,7 +1648,6 @@ static_opts_init(static_opts_t *static_opts) { typedef struct dynamic_opts_s dynamic_opts_t; struct dynamic_opts_s { void **result; - size_t usize; size_t num_items; size_t item_size; size_t alignment; @@ -1898,7 +1659,6 @@ struct dynamic_opts_s { JEMALLOC_ALWAYS_INLINE void dynamic_opts_init(dynamic_opts_t *dynamic_opts) { dynamic_opts->result = NULL; - dynamic_opts->usize = 0; dynamic_opts->num_items = 0; dynamic_opts->item_size = 0; dynamic_opts->alignment = 0; @@ -1962,13 +1722,12 @@ imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, szind_t ind_large; size_t bumped_usize = usize; - if (usize <= SC_SMALL_MAXCLASS) { - assert(((dopts->alignment == 0) ? - sz_s2u(SC_LARGE_MINCLASS) : - sz_sa2u(SC_LARGE_MINCLASS, dopts->alignment)) - == SC_LARGE_MINCLASS); - ind_large = sz_size2index(SC_LARGE_MINCLASS); - bumped_usize = sz_s2u(SC_LARGE_MINCLASS); + if (usize <= SMALL_MAXCLASS) { + assert(((dopts->alignment == 0) ? sz_s2u(LARGE_MINCLASS) : + sz_sa2u(LARGE_MINCLASS, dopts->alignment)) + == LARGE_MINCLASS); + ind_large = sz_size2index(LARGE_MINCLASS); + bumped_usize = sz_s2u(LARGE_MINCLASS); ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize, bumped_usize, ind_large); if (unlikely(ret == NULL)) { @@ -2051,6 +1810,17 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { goto label_oom; } + /* Validate the user input. */ + if (sopts->bump_empty_alloc) { + if (unlikely(size == 0)) { + size = 1; + } + } + + if (sopts->assert_nonempty_alloc) { + assert (size != 0); + } + if (unlikely(dopts->alignment < sopts->min_alignment || (dopts->alignment & (dopts->alignment - 1)) != 0)) { goto label_invalid_alignment; @@ -2060,32 +1830,19 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { if (dopts->alignment == 0) { ind = sz_size2index(size); - if (unlikely(ind >= SC_NSIZES)) { + if (unlikely(ind >= NSIZES)) { goto label_oom; } - if (config_stats || (config_prof && opt_prof) || sopts->usize) { + if (config_stats || (config_prof && opt_prof)) { usize = sz_index2size(ind); - dopts->usize = usize; - assert(usize > 0 && usize - <= SC_LARGE_MAXCLASS); + assert(usize > 0 && usize <= LARGE_MAXCLASS); } } else { - if (sopts->bump_empty_aligned_alloc) { - if (unlikely(size == 0)) { - size = 1; - } - } usize = sz_sa2u(size, dopts->alignment); - dopts->usize = usize; - if (unlikely(usize == 0 - || usize > SC_LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { goto label_oom; } } - /* Validate the user input. */ - if (sopts->assert_nonempty_alloc) { - assert (size != 0); - } check_entry_exit_locking(tsd_tsdn(tsd)); @@ -2118,8 +1875,7 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { alloc_ctx_t alloc_ctx; if (likely((uintptr_t)tctx == (uintptr_t)1U)) { - alloc_ctx.slab = (usize - <= SC_SMALL_MAXCLASS); + alloc_ctx.slab = (usize <= SMALL_MAXCLASS); allocation = imalloc_no_sample( sopts, dopts, tsd, usize, usize, ind); } else if ((uintptr_t)tctx > (uintptr_t)1U) { @@ -2224,8 +1980,9 @@ label_invalid_alignment: return EINVAL; } -JEMALLOC_ALWAYS_INLINE bool -imalloc_init_check(static_opts_t *sopts, dynamic_opts_t *dopts) { +/* Returns the errno-style error code of the allocation. */ +JEMALLOC_ALWAYS_INLINE int +imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { if (unlikely(!malloc_initialized()) && unlikely(malloc_init())) { if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(sopts->oom_string); @@ -2235,16 +1992,6 @@ imalloc_init_check(static_opts_t *sopts, dynamic_opts_t *dopts) { set_errno(ENOMEM); *dopts->result = NULL; - return false; - } - - return true; -} - -/* Returns the errno-style error code of the allocation. */ -JEMALLOC_ALWAYS_INLINE int -imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { - if (tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) { return ENOMEM; } @@ -2257,18 +2004,19 @@ imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { sopts->slow = false; return imalloc_body(sopts, dopts, tsd); } else { - if (!tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) { - return ENOMEM; - } - sopts->slow = true; return imalloc_body(sopts, dopts, tsd); } } +/******************************************************************************/ +/* + * Begin malloc(3)-compatible functions. + */ -JEMALLOC_NOINLINE -void * -malloc_default(size_t size) { +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN +void JEMALLOC_NOTHROW * +JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) +je_malloc(size_t size) { void *ret; static_opts_t sopts; dynamic_opts_t dopts; @@ -2278,6 +2026,7 @@ malloc_default(size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); + sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.oom_string = ": Error in malloc(): out of memory\n"; @@ -2287,107 +2036,12 @@ malloc_default(size_t size) { dopts.item_size = size; imalloc(&sopts, &dopts); - /* - * Note that this branch gets optimized away -- it immediately follows - * the check on tsd_fast that sets sopts.slow. - */ - if (sopts.slow) { - uintptr_t args[3] = {size}; - hook_invoke_alloc(hook_alloc_malloc, ret, (uintptr_t)ret, args); - } LOG("core.malloc.exit", "result: %p", ret); return ret; } -/******************************************************************************/ -/* - * Begin malloc(3)-compatible functions. - */ - -/* - * malloc() fastpath. - * - * Fastpath assumes size <= SC_LOOKUP_MAXCLASS, and that we hit - * tcache. If either of these is false, we tail-call to the slowpath, - * malloc_default(). Tail-calling is used to avoid any caller-saved - * registers. - * - * fastpath supports ticker and profiling, both of which will also - * tail-call to the slowpath if they fire. - */ -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * -JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) -je_malloc(size_t size) { - LOG("core.malloc.entry", "size: %zu", size); - - if (tsd_get_allocates() && unlikely(!malloc_initialized())) { - return malloc_default(size); - } - - tsd_t *tsd = tsd_get(false); - if (unlikely(!tsd || !tsd_fast(tsd) || (size > SC_LOOKUP_MAXCLASS))) { - return malloc_default(size); - } - - tcache_t *tcache = tsd_tcachep_get(tsd); - - if (unlikely(ticker_trytick(&tcache->gc_ticker))) { - return malloc_default(size); - } - - szind_t ind = sz_size2index_lookup(size); - size_t usize; - if (config_stats || config_prof) { - usize = sz_index2size(ind); - } - /* Fast path relies on size being a bin. I.e. SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS */ - assert(ind < SC_NBINS); - assert(size <= SC_SMALL_MAXCLASS); - - if (config_prof) { - int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd); - bytes_until_sample -= usize; - tsd_bytes_until_sample_set(tsd, bytes_until_sample); - - if (unlikely(bytes_until_sample < 0)) { - /* - * Avoid a prof_active check on the fastpath. - * If prof_active is false, set bytes_until_sample to - * a large value. If prof_active is set to true, - * bytes_until_sample will be reset. - */ - if (!prof_active) { - tsd_bytes_until_sample_set(tsd, SSIZE_MAX); - } - return malloc_default(size); - } - } - - cache_bin_t *bin = tcache_small_bin_get(tcache, ind); - bool tcache_success; - void* ret = cache_bin_alloc_easy(bin, &tcache_success); - - if (tcache_success) { - if (config_stats) { - *tsd_thread_allocatedp_get(tsd) += usize; - bin->tstats.nrequests++; - } - if (config_prof) { - tcache->prof_accumbytes += usize; - } - - LOG("core.malloc.exit", "result: %p", ret); - - /* Fastpath success */ - return ret; - } - - return malloc_default(size); -} - JEMALLOC_EXPORT int JEMALLOC_NOTHROW JEMALLOC_ATTR(nonnull(1)) je_posix_memalign(void **memptr, size_t alignment, size_t size) { @@ -2401,7 +2055,7 @@ je_posix_memalign(void **memptr, size_t alignment, size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_aligned_alloc = true; + sopts.bump_empty_alloc = true; sopts.min_alignment = sizeof(void *); sopts.oom_string = ": Error allocating aligned memory: out of memory\n"; @@ -2414,12 +2068,6 @@ je_posix_memalign(void **memptr, size_t alignment, size_t size) { dopts.alignment = alignment; ret = imalloc(&sopts, &dopts); - if (sopts.slow) { - uintptr_t args[3] = {(uintptr_t)memptr, (uintptr_t)alignment, - (uintptr_t)size}; - hook_invoke_alloc(hook_alloc_posix_memalign, *memptr, - (uintptr_t)ret, args); - } LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret, *memptr); @@ -2442,7 +2090,7 @@ je_aligned_alloc(size_t alignment, size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_aligned_alloc = true; + sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.min_alignment = 1; @@ -2457,11 +2105,6 @@ je_aligned_alloc(size_t alignment, size_t size) { dopts.alignment = alignment; imalloc(&sopts, &dopts); - if (sopts.slow) { - uintptr_t args[3] = {(uintptr_t)alignment, (uintptr_t)size}; - hook_invoke_alloc(hook_alloc_aligned_alloc, ret, - (uintptr_t)ret, args); - } LOG("core.aligned_alloc.exit", "result: %p", ret); @@ -2482,6 +2125,7 @@ je_calloc(size_t num, size_t size) { dynamic_opts_init(&dopts); sopts.may_overflow = true; + sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.oom_string = ": Error in calloc(): out of memory\n"; @@ -2492,10 +2136,6 @@ je_calloc(size_t num, size_t size) { dopts.zero = true; imalloc(&sopts, &dopts); - if (sopts.slow) { - uintptr_t args[3] = {(uintptr_t)num, (uintptr_t)size}; - hook_invoke_alloc(hook_alloc_calloc, ret, (uintptr_t)ret, args); - } LOG("core.calloc.exit", "result: %p", ret); @@ -2504,22 +2144,20 @@ je_calloc(size_t num, size_t size) { static void * irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, - prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) { + prof_tctx_t *tctx) { void *p; if (tctx == NULL) { return NULL; } - if (usize <= SC_SMALL_MAXCLASS) { - p = iralloc(tsd, old_ptr, old_usize, - SC_LARGE_MINCLASS, 0, false, hook_args); + if (usize <= SMALL_MAXCLASS) { + p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false); if (p == NULL) { return NULL; } arena_prof_promote(tsd_tsdn(tsd), p, usize); } else { - p = iralloc(tsd, old_ptr, old_usize, usize, 0, false, - hook_args); + p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); } return p; @@ -2527,7 +2165,7 @@ irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, JEMALLOC_ALWAYS_INLINE void * irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, - alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) { + alloc_ctx_t *alloc_ctx) { void *p; bool prof_active; prof_tctx_t *old_tctx, *tctx; @@ -2536,11 +2174,9 @@ irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx); tctx = prof_alloc_prep(tsd, usize, prof_active, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { - p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx, - hook_args); + p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx); } else { - p = iralloc(tsd, old_ptr, old_usize, usize, 0, false, - hook_args); + p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, true); @@ -2569,7 +2205,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != SC_NSIZES); + assert(alloc_ctx.szind != NSIZES); size_t usize; if (config_prof && opt_prof) { @@ -2650,12 +2286,11 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * JEMALLOC_ALLOC_SIZE(2) -je_realloc(void *ptr, size_t arg_size) { +je_realloc(void *ptr, size_t size) { void *ret; tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL); size_t usize JEMALLOC_CC_SILENCE_INIT(0); size_t old_usize = 0; - size_t size = arg_size; LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size); @@ -2670,10 +2305,6 @@ je_realloc(void *ptr, size_t arg_size) { } else { tcache = NULL; } - - uintptr_t args[3] = {(uintptr_t)ptr, size}; - hook_invoke_dalloc(hook_dalloc_realloc, ptr, args); - ifree(tsd, ptr, tcache, true); LOG("core.realloc.exit", "result: %p", NULL); @@ -2688,58 +2319,29 @@ je_realloc(void *ptr, size_t arg_size) { check_entry_exit_locking(tsd_tsdn(tsd)); - - hook_ralloc_args_t hook_args = {true, {(uintptr_t)ptr, - (uintptr_t)arg_size, 0, 0}}; - alloc_ctx_t alloc_ctx; rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != SC_NSIZES); + assert(alloc_ctx.szind != NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); if (config_prof && opt_prof) { usize = sz_s2u(size); - if (unlikely(usize == 0 - || usize > SC_LARGE_MAXCLASS)) { - ret = NULL; - } else { - ret = irealloc_prof(tsd, ptr, old_usize, usize, - &alloc_ctx, &hook_args); - } + ret = unlikely(usize == 0 || usize > LARGE_MAXCLASS) ? + NULL : irealloc_prof(tsd, ptr, old_usize, usize, + &alloc_ctx); } else { if (config_stats) { usize = sz_s2u(size); } - ret = iralloc(tsd, ptr, old_usize, size, 0, false, - &hook_args); + ret = iralloc(tsd, ptr, old_usize, size, 0, false); } tsdn = tsd_tsdn(tsd); } else { /* realloc(NULL, size) is equivalent to malloc(size). */ - static_opts_t sopts; - dynamic_opts_t dopts; - - static_opts_init(&sopts); - dynamic_opts_init(&dopts); - - sopts.null_out_result_on_error = true; - sopts.set_errno_on_error = true; - sopts.oom_string = - ": Error in realloc(): out of memory\n"; - - dopts.result = &ret; - dopts.num_items = 1; - dopts.item_size = size; - - imalloc(&sopts, &dopts); - if (sopts.slow) { - uintptr_t args[3] = {(uintptr_t)ptr, arg_size}; - hook_invoke_alloc(hook_alloc_realloc, ret, - (uintptr_t)ret, args); - } - + void *ret = je_malloc(size); + LOG("core.realloc.exit", "result: %p", ret); return ret; } @@ -2766,9 +2368,10 @@ je_realloc(void *ptr, size_t arg_size) { return ret; } -JEMALLOC_NOINLINE -void -free_default(void *ptr) { +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_free(void *ptr) { + LOG("core.free.entry", "ptr: %p", ptr); + UTRACE(ptr, 0, 0); if (likely(ptr != NULL)) { /* @@ -2794,79 +2397,10 @@ free_default(void *ptr) { } else { tcache = NULL; } - uintptr_t args_raw[3] = {(uintptr_t)ptr}; - hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw); ifree(tsd, ptr, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); } -} - -JEMALLOC_ALWAYS_INLINE -bool free_fastpath(void *ptr, size_t size, bool size_hint) { - tsd_t *tsd = tsd_get(false); - if (unlikely(!tsd || !tsd_fast(tsd))) { - return false; - } - - tcache_t *tcache = tsd_tcachep_get(tsd); - - alloc_ctx_t alloc_ctx; - /* - * If !config_cache_oblivious, we can check PAGE alignment to - * detect sampled objects. Otherwise addresses are - * randomized, and we have to look it up in the rtree anyway. - * See also isfree(). - */ - if (!size_hint || config_cache_oblivious) { - rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); - bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), &extents_rtree, - rtree_ctx, (uintptr_t)ptr, - &alloc_ctx.szind, &alloc_ctx.slab); - - /* Note: profiled objects will have alloc_ctx.slab set */ - if (!res || !alloc_ctx.slab) { - return false; - } - assert(alloc_ctx.szind != SC_NSIZES); - } else { - /* - * Check for both sizes that are too large, and for sampled objects. - * Sampled objects are always page-aligned. The sampled object check - * will also check for null ptr. - */ - if (size > SC_LOOKUP_MAXCLASS || (((uintptr_t)ptr & PAGE_MASK) == 0)) { - return false; - } - alloc_ctx.szind = sz_size2index_lookup(size); - } - - if (unlikely(ticker_trytick(&tcache->gc_ticker))) { - return false; - } - - cache_bin_t *bin = tcache_small_bin_get(tcache, alloc_ctx.szind); - cache_bin_info_t *bin_info = &tcache_bin_info[alloc_ctx.szind]; - if (!cache_bin_dalloc_easy(bin, bin_info, ptr)) { - return false; - } - - if (config_stats) { - size_t usize = sz_index2size(alloc_ctx.szind); - *tsd_thread_deallocatedp_get(tsd) += usize; - } - - return true; -} - -JEMALLOC_EXPORT void JEMALLOC_NOTHROW -je_free(void *ptr) { - LOG("core.free.entry", "ptr: %p", ptr); - - if (!free_fastpath(ptr, 0, false)) { - free_default(ptr); - } - LOG("core.free.exit", ""); } @@ -2893,6 +2427,7 @@ je_memalign(size_t alignment, size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); + sopts.bump_empty_alloc = true; sopts.min_alignment = 1; sopts.oom_string = ": Error allocating aligned memory: out of memory\n"; @@ -2906,11 +2441,6 @@ je_memalign(size_t alignment, size_t size) { dopts.alignment = alignment; imalloc(&sopts, &dopts); - if (sopts.slow) { - uintptr_t args[3] = {alignment, size}; - hook_invoke_alloc(hook_alloc_memalign, ret, (uintptr_t)ret, - args); - } LOG("core.memalign.exit", "result: %p", ret); return ret; @@ -2932,6 +2462,7 @@ je_valloc(size_t size) { static_opts_init(&sopts); dynamic_opts_init(&dopts); + sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.min_alignment = PAGE; sopts.oom_string = @@ -2945,10 +2476,6 @@ je_valloc(size_t size) { dopts.alignment = PAGE; imalloc(&sopts, &dopts); - if (sopts.slow) { - uintptr_t args[3] = {size}; - hook_invoke_alloc(hook_alloc_valloc, ret, (uintptr_t)ret, args); - } LOG("core.valloc.exit", "result: %p\n", ret); return ret; @@ -3016,82 +2543,6 @@ int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign); * Begin non-standard functions. */ -#ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API - -#define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y -#define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \ - JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) - -typedef struct { - void *ptr; - size_t size; -} smallocx_return_t; - -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -smallocx_return_t JEMALLOC_NOTHROW -/* - * The attribute JEMALLOC_ATTR(malloc) cannot be used due to: - * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488 - */ -JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT) - (size_t size, int flags) { - /* - * Note: the attribute JEMALLOC_ALLOC_SIZE(1) cannot be - * used here because it makes writing beyond the `size` - * of the `ptr` undefined behavior, but the objective - * of this function is to allow writing beyond `size` - * up to `smallocx_return_t::size`. - */ - smallocx_return_t ret; - static_opts_t sopts; - dynamic_opts_t dopts; - - LOG("core.smallocx.entry", "size: %zu, flags: %d", size, flags); - - static_opts_init(&sopts); - dynamic_opts_init(&dopts); - - sopts.assert_nonempty_alloc = true; - sopts.null_out_result_on_error = true; - sopts.oom_string = ": Error in mallocx(): out of memory\n"; - sopts.usize = true; - - dopts.result = &ret.ptr; - dopts.num_items = 1; - dopts.item_size = size; - if (unlikely(flags != 0)) { - if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) { - dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); - } - - dopts.zero = MALLOCX_ZERO_GET(flags); - - if ((flags & MALLOCX_TCACHE_MASK) != 0) { - if ((flags & MALLOCX_TCACHE_MASK) - == MALLOCX_TCACHE_NONE) { - dopts.tcache_ind = TCACHE_IND_NONE; - } else { - dopts.tcache_ind = MALLOCX_TCACHE_GET(flags); - } - } else { - dopts.tcache_ind = TCACHE_IND_AUTOMATIC; - } - - if ((flags & MALLOCX_ARENA_MASK) != 0) - dopts.arena_ind = MALLOCX_ARENA_GET(flags); - } - - imalloc(&sopts, &dopts); - assert(dopts.usize == je_nallocx(size, flags)); - ret.size = dopts.usize; - - LOG("core.smallocx.exit", "result: %p, size: %zu", ret.ptr, ret.size); - return ret; -} -#undef JEMALLOC_SMALLOCX_CONCAT_HELPER -#undef JEMALLOC_SMALLOCX_CONCAT_HELPER2 -#endif - JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) @@ -3135,11 +2586,6 @@ je_mallocx(size_t size, int flags) { } imalloc(&sopts, &dopts); - if (sopts.slow) { - uintptr_t args[3] = {size, flags}; - hook_invoke_alloc(hook_alloc_mallocx, ret, (uintptr_t)ret, - args); - } LOG("core.mallocx.exit", "result: %p", ret); return ret; @@ -3148,23 +2594,22 @@ je_mallocx(size_t size, int flags) { static void * irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, - prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) { + prof_tctx_t *tctx) { void *p; if (tctx == NULL) { return NULL; } - if (usize <= SC_SMALL_MAXCLASS) { - p = iralloct(tsdn, old_ptr, old_usize, - SC_LARGE_MINCLASS, alignment, zero, tcache, - arena, hook_args); + if (usize <= SMALL_MAXCLASS) { + p = iralloct(tsdn, old_ptr, old_usize, LARGE_MINCLASS, + alignment, zero, tcache, arena); if (p == NULL) { return NULL; } arena_prof_promote(tsdn, p, usize); } else { p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero, - tcache, arena, hook_args); + tcache, arena); } return p; @@ -3173,7 +2618,7 @@ irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, JEMALLOC_ALWAYS_INLINE void * irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, size_t alignment, size_t *usize, bool zero, tcache_t *tcache, - arena_t *arena, alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) { + arena_t *arena, alloc_ctx_t *alloc_ctx) { void *p; bool prof_active; prof_tctx_t *old_tctx, *tctx; @@ -3183,10 +2628,10 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, tctx = prof_alloc_prep(tsd, *usize, prof_active, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize, - *usize, alignment, zero, tcache, arena, tctx, hook_args); + *usize, alignment, zero, tcache, arena, tctx); } else { p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment, - zero, tcache, arena, hook_args); + zero, tcache, arena); } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, false); @@ -3257,27 +2702,23 @@ je_rallocx(void *ptr, size_t size, int flags) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != SC_NSIZES); + assert(alloc_ctx.szind != NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); - - hook_ralloc_args_t hook_args = {false, {(uintptr_t)ptr, size, flags, - 0}}; if (config_prof && opt_prof) { usize = (alignment == 0) ? sz_s2u(size) : sz_sa2u(size, alignment); - if (unlikely(usize == 0 - || usize > SC_LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { goto label_oom; } p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, - zero, tcache, arena, &alloc_ctx, &hook_args); + zero, tcache, arena, &alloc_ctx); if (unlikely(p == NULL)) { goto label_oom; } } else { p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment, - zero, tcache, arena, &hook_args); + zero, tcache, arena); if (unlikely(p == NULL)) { goto label_oom; } @@ -3311,14 +2752,14 @@ label_oom: JEMALLOC_ALWAYS_INLINE size_t ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, size_t extra, size_t alignment, bool zero) { - size_t newsize; + size_t usize; - if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero, - &newsize)) { + if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero)) { return old_usize; } + usize = isalloc(tsdn, ptr); - return newsize; + return usize; } static size_t @@ -3352,19 +2793,17 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, */ if (alignment == 0) { usize_max = sz_s2u(size+extra); - assert(usize_max > 0 - && usize_max <= SC_LARGE_MAXCLASS); + assert(usize_max > 0 && usize_max <= LARGE_MAXCLASS); } else { usize_max = sz_sa2u(size+extra, alignment); - if (unlikely(usize_max == 0 - || usize_max > SC_LARGE_MAXCLASS)) { + if (unlikely(usize_max == 0 || usize_max > LARGE_MAXCLASS)) { /* * usize_max is out of range, and chances are that * allocation will fail, but use the maximum possible * value and carry on with prof_alloc_prep(), just in * case allocation succeeds. */ - usize_max = SC_LARGE_MAXCLASS; + usize_max = LARGE_MAXCLASS; } } tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); @@ -3407,24 +2846,24 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != SC_NSIZES); + assert(alloc_ctx.szind != NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); /* * The API explicitly absolves itself of protecting against (size + * extra) numerical overflow, but we may need to clamp extra to avoid - * exceeding SC_LARGE_MAXCLASS. + * exceeding LARGE_MAXCLASS. * * Ordinarily, size limit checking is handled deeper down, but here we * have to check as part of (size + extra) clamping, since we need the * clamped value in the above helper functions. */ - if (unlikely(size > SC_LARGE_MAXCLASS)) { + if (unlikely(size > LARGE_MAXCLASS)) { usize = old_usize; goto label_not_resized; } - if (unlikely(SC_LARGE_MAXCLASS - size < extra)) { - extra = SC_LARGE_MAXCLASS - size; + if (unlikely(LARGE_MAXCLASS - size < extra)) { + extra = LARGE_MAXCLASS - size; } if (config_prof && opt_prof) { @@ -3443,12 +2882,6 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { *tsd_thread_deallocatedp_get(tsd) += old_usize; } label_not_resized: - if (unlikely(!tsd_fast(tsd))) { - uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags}; - hook_invoke_expand(hook_expand_xallocx, ptr, old_usize, - usize, (uintptr_t)usize, args); - } - UTRACE(ptr, size, ptr); check_entry_exit_locking(tsd_tsdn(tsd)); @@ -3458,7 +2891,7 @@ label_not_resized: JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW JEMALLOC_ATTR(pure) -je_sallocx(const void *ptr, int flags) { +je_sallocx(const void *ptr, UNUSED int flags) { size_t usize; tsdn_t *tsdn; @@ -3521,8 +2954,6 @@ je_dallocx(void *ptr, int flags) { tsd_assert_fast(tsd); ifree(tsd, ptr, tcache, false); } else { - uintptr_t args_raw[3] = {(uintptr_t)ptr, flags}; - hook_invoke_dalloc(hook_dalloc_dallocx, ptr, args_raw); ifree(tsd, ptr, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); @@ -3544,11 +2975,14 @@ inallocx(tsdn_t *tsdn, size_t size, int flags) { return usize; } -JEMALLOC_NOINLINE void -sdallocx_default(void *ptr, size_t size, int flags) { +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_sdallocx(void *ptr, size_t size, int flags) { assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); + LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, + size, flags); + tsd_t *tsd = tsd_fetch(); bool fast = tsd_fast(tsd); size_t usize = inallocx(tsd_tsdn(tsd), size, flags); @@ -3582,35 +3016,10 @@ sdallocx_default(void *ptr, size_t size, int flags) { tsd_assert_fast(tsd); isfree(tsd, ptr, usize, tcache, false); } else { - uintptr_t args_raw[3] = {(uintptr_t)ptr, size, flags}; - hook_invoke_dalloc(hook_dalloc_sdallocx, ptr, args_raw); isfree(tsd, ptr, usize, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); -} - -JEMALLOC_EXPORT void JEMALLOC_NOTHROW -je_sdallocx(void *ptr, size_t size, int flags) { - LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, - size, flags); - - if (flags !=0 || !free_fastpath(ptr, size, true)) { - sdallocx_default(ptr, size, flags); - } - - LOG("core.sdallocx.exit", ""); -} - -void JEMALLOC_NOTHROW -je_sdallocx_noflags(void *ptr, size_t size) { - LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: 0", ptr, - size); - - if (!free_fastpath(ptr, size, true)) { - sdallocx_default(ptr, size, 0); - } - LOG("core.sdallocx.exit", ""); } @@ -3631,7 +3040,7 @@ je_nallocx(size_t size, int flags) { check_entry_exit_locking(tsdn); usize = inallocx(tsdn, size, flags); - if (unlikely(usize > SC_LARGE_MAXCLASS)) { + if (unlikely(usize > LARGE_MAXCLASS)) { LOG("core.nallocx.exit", "result: %zu", ZU(0)); return 0; } @@ -3847,7 +3256,6 @@ _malloc_prefork(void) } } prof_prefork1(tsd_tsdn(tsd)); - tsd_prefork(tsd); } #ifndef JEMALLOC_MUTEX_INIT_CB @@ -3870,8 +3278,6 @@ _malloc_postfork(void) tsd = tsd_fetch(); - tsd_postfork_parent(tsd); - witness_postfork_parent(tsd_witness_tsdp_get(tsd)); /* Release all mutexes, now that fork() has completed. */ for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { @@ -3899,8 +3305,6 @@ jemalloc_postfork_child(void) { tsd = tsd_fetch(); - tsd_postfork_child(tsd); - witness_postfork_child(tsd_witness_tsdp_get(tsd)); /* Release all mutexes, now that fork() has completed. */ for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { @@ -3920,3 +3324,12 @@ jemalloc_postfork_child(void) { } /******************************************************************************/ + +/* Helps the application decide if a pointer is worth re-allocating in order to reduce fragmentation. + * returns 1 if the allocation should be moved, and 0 if the allocation be kept. + * If the application decides to re-allocate it should use MALLOCX_TCACHE_NONE when doing so. */ +JEMALLOC_EXPORT int JEMALLOC_NOTHROW +get_defrag_hint(void* ptr) { + assert(ptr != NULL); + return iget_defrag_hint(TSDN_NULL, ptr); +} diff --git a/deps/jemalloc/src/jemalloc_cpp.cpp b/deps/jemalloc/src/jemalloc_cpp.cpp index da0441a..f0cedda 100644 --- a/deps/jemalloc/src/jemalloc_cpp.cpp +++ b/deps/jemalloc/src/jemalloc_cpp.cpp @@ -128,14 +128,14 @@ operator delete(void *ptr, std::size_t size) noexcept { if (unlikely(ptr == nullptr)) { return; } - je_sdallocx_noflags(ptr, size); + je_sdallocx(ptr, size, /*flags=*/0); } void operator delete[](void *ptr, std::size_t size) noexcept { if (unlikely(ptr == nullptr)) { return; } - je_sdallocx_noflags(ptr, size); + je_sdallocx(ptr, size, /*flags=*/0); } #endif // __cpp_sized_deallocation diff --git a/deps/jemalloc/src/large.c b/deps/jemalloc/src/large.c index 8e7a781..27a2c67 100644 --- a/deps/jemalloc/src/large.c +++ b/deps/jemalloc/src/large.c @@ -28,7 +28,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, assert(!tsdn_null(tsdn) || arena != NULL); ausize = sz_sa2u(usize, alignment); - if (unlikely(ausize == 0 || ausize > SC_LARGE_MAXCLASS)) { + if (unlikely(ausize == 0 || ausize > LARGE_MAXCLASS)) { return NULL; } @@ -42,7 +42,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, */ is_zeroed = zero; if (likely(!tsdn_null(tsdn))) { - arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize); + arena = arena_choose(tsdn_tsd(tsdn), arena); } if (unlikely(arena == NULL) || (extent = arena_extent_alloc_large(tsdn, arena, usize, alignment, &is_zeroed)) == NULL) { @@ -109,7 +109,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) { if (diff != 0) { extent_t *trail = extent_split_wrapper(tsdn, arena, &extent_hooks, extent, usize + sz_large_pad, - sz_size2index(usize), false, diff, SC_NSIZES, false); + sz_size2index(usize), false, diff, NSIZES, false); if (trail == NULL) { return true; } @@ -154,17 +154,17 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, bool new_mapping; if ((trail = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, extent_past_get(extent), trailsize, 0, - CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL + CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL || (trail = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, extent_past_get(extent), trailsize, 0, - CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) { + CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL) { if (config_stats) { new_mapping = false; } } else { if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks, extent_past_get(extent), trailsize, 0, CACHELINE, false, - SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) { + NSIZES, &is_zeroed_trail, &commit)) == NULL) { return true; } if (config_stats) { @@ -221,10 +221,9 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, size_t oldusize = extent_usize_get(extent); /* The following should have been caught by callers. */ - assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS); + assert(usize_min > 0 && usize_max <= LARGE_MAXCLASS); /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= SC_LARGE_MINCLASS - && usize_max >= SC_LARGE_MINCLASS); + assert(oldusize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS); if (usize_max > oldusize) { /* Attempt to expand the allocation in-place. */ @@ -271,23 +270,17 @@ large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, } void * -large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, - size_t alignment, bool zero, tcache_t *tcache, - hook_ralloc_args_t *hook_args) { - extent_t *extent = iealloc(tsdn, ptr); - +large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, + size_t alignment, bool zero, tcache_t *tcache) { size_t oldusize = extent_usize_get(extent); + /* The following should have been caught by callers. */ - assert(usize > 0 && usize <= SC_LARGE_MAXCLASS); + assert(usize > 0 && usize <= LARGE_MAXCLASS); /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= SC_LARGE_MINCLASS - && usize >= SC_LARGE_MINCLASS); + assert(oldusize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS); /* Try to avoid moving the allocation. */ if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero)) { - hook_invoke_expand(hook_args->is_realloc - ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize, - usize, (uintptr_t)ptr, hook_args->args); return extent_addr_get(extent); } @@ -302,12 +295,6 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, return NULL; } - hook_invoke_alloc(hook_args->is_realloc - ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, - hook_args->args); - hook_invoke_dalloc(hook_args->is_realloc - ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); - size_t copysize = (usize < oldusize) ? usize : oldusize; memcpy(ret, extent_addr_get(extent), copysize); isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, NULL, true); @@ -331,9 +318,8 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent, large_dalloc_maybe_junk(extent_addr_get(extent), extent_usize_get(extent)); } else { - /* Only hold the large_mtx if necessary. */ + malloc_mutex_assert_owner(tsdn, &arena->large_mtx); if (!arena_is_auto(arena)) { - malloc_mutex_assert_owner(tsdn, &arena->large_mtx); extent_list_remove(&arena->large, extent); } } @@ -383,13 +369,3 @@ void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent) { large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U); } - -nstime_t -large_prof_alloc_time_get(const extent_t *extent) { - return extent_prof_alloc_time_get(extent); -} - -void -large_prof_alloc_time_set(extent_t *extent, nstime_t t) { - extent_prof_alloc_time_set(extent, t); -} diff --git a/deps/jemalloc/src/malloc_io.c b/deps/jemalloc/src/malloc_io.c index d7cb0f5..7bdc13f 100644 --- a/deps/jemalloc/src/malloc_io.c +++ b/deps/jemalloc/src/malloc_io.c @@ -362,7 +362,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { } \ } while (0) #define GET_ARG_NUMERIC(val, len) do { \ - switch ((unsigned char)len) { \ + switch (len) { \ case '?': \ val = va_arg(ap, int); \ break; \ @@ -632,6 +632,7 @@ malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, */ write_cb = (je_malloc_message != NULL) ? je_malloc_message : wrtmessage; + cbopaque = NULL; } malloc_vsnprintf(buf, sizeof(buf), format, ap); diff --git a/deps/jemalloc/src/mutex.c b/deps/jemalloc/src/mutex.c index 3f920f5..30222b3 100644 --- a/deps/jemalloc/src/mutex.c +++ b/deps/jemalloc/src/mutex.c @@ -46,7 +46,7 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, void malloc_mutex_lock_slow(malloc_mutex_t *mutex) { mutex_prof_data_t *data = &mutex->prof_data; - nstime_t before = NSTIME_ZERO_INITIALIZER; + UNUSED nstime_t before = NSTIME_ZERO_INITIALIZER; if (ncpus == 1) { goto label_spin_done; @@ -55,8 +55,7 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) { int cnt = 0, max_cnt = MALLOC_MUTEX_MAX_SPIN; do { spin_cpu_spinwait(); - if (!atomic_load_b(&mutex->locked, ATOMIC_RELAXED) - && !malloc_mutex_trylock_final(mutex)) { + if (!malloc_mutex_trylock_final(mutex)) { data->n_spin_acquired++; return; } @@ -145,7 +144,9 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, } # endif #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) - mutex->lock = OS_UNFAIR_LOCK_INIT; + mutex->lock = OS_UNFAIR_LOCK_INIT; +#elif (defined(JEMALLOC_OSSPIN)) + mutex->lock = 0; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) if (postpone_init) { mutex->postponed_next = postponed_mutexes; diff --git a/deps/jemalloc/src/pages.c b/deps/jemalloc/src/pages.c index 13de27a..2600269 100644 --- a/deps/jemalloc/src/pages.c +++ b/deps/jemalloc/src/pages.c @@ -180,35 +180,6 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) { assert(alignment >= PAGE); assert(ALIGNMENT_ADDR2BASE(addr, alignment) == addr); -#if defined(__FreeBSD__) && defined(MAP_EXCL) - /* - * FreeBSD has mechanisms both to mmap at specific address without - * touching existing mappings, and to mmap with specific alignment. - */ - { - if (os_overcommits) { - *commit = true; - } - - int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; - int flags = mmap_flags; - - if (addr != NULL) { - flags |= MAP_FIXED | MAP_EXCL; - } else { - unsigned alignment_bits = ffs_zu(alignment); - assert(alignment_bits > 1); - flags |= MAP_ALIGNED(alignment_bits - 1); - } - - void *ret = mmap(addr, size, prot, flags, -1, 0); - if (ret == MAP_FAILED) { - ret = NULL; - } - - return ret; - } -#endif /* * Ideally, there would be a way to specify alignment to mmap() (like * NetBSD has), but in the absence of such a feature, we have to work @@ -290,7 +261,7 @@ pages_decommit(void *addr, size_t size) { bool pages_purge_lazy(void *addr, size_t size) { - assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr); + assert(PAGE_ADDR2BASE(addr) == addr); assert(PAGE_CEILING(size) == size); if (!pages_can_purge_lazy) { @@ -420,10 +391,6 @@ os_page_detect(void) { GetSystemInfo(&si); return si.dwPageSize; #elif defined(__FreeBSD__) - /* - * This returns the value obtained from - * the auxv vector, avoiding a syscall. - */ return getpagesize(); #else long result = sysconf(_SC_PAGESIZE); @@ -577,10 +544,6 @@ init_thp_state(void) { close(fd); #endif - if (nread < 0) { - goto label_error; - } - if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) { init_system_thp_mode = thp_mode_default; } else if (strncmp(buf, sys_state_always, (size_t)nread) == 0) { @@ -625,11 +588,6 @@ pages_boot(void) { init_thp_state(); -#ifdef __FreeBSD__ - /* - * FreeBSD doesn't need the check; madvise(2) is known to work. - */ -#else /* Detect lazy purge runtime support. */ if (pages_can_purge_lazy) { bool committed = false; @@ -643,7 +601,6 @@ pages_boot(void) { } os_pages_unmap(madv_free_page, PAGE); } -#endif return false; } diff --git a/deps/jemalloc/src/prof.c b/deps/jemalloc/src/prof.c index 13334cb..13df641 100644 --- a/deps/jemalloc/src/prof.c +++ b/deps/jemalloc/src/prof.c @@ -7,7 +7,6 @@ #include "jemalloc/internal/hash.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/emitter.h" /******************************************************************************/ @@ -24,7 +23,7 @@ */ #undef _Unwind_Backtrace #include -#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) +#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook) #endif /******************************************************************************/ @@ -39,7 +38,6 @@ bool opt_prof_gdump = false; bool opt_prof_final = false; bool opt_prof_leak = false; bool opt_prof_accum = false; -bool opt_prof_log = false; char opt_prof_prefix[ /* Minimize memory bloat for non-prof builds. */ #ifdef JEMALLOC_PROF @@ -72,100 +70,6 @@ uint64_t prof_interval = 0; size_t lg_prof_sample; -typedef enum prof_logging_state_e prof_logging_state_t; -enum prof_logging_state_e { - prof_logging_state_stopped, - prof_logging_state_started, - prof_logging_state_dumping -}; - -/* - * - stopped: log_start never called, or previous log_stop has completed. - * - started: log_start called, log_stop not called yet. Allocations are logged. - * - dumping: log_stop called but not finished; samples are not logged anymore. - */ -prof_logging_state_t prof_logging_state = prof_logging_state_stopped; - -#ifdef JEMALLOC_JET -static bool prof_log_dummy = false; -#endif - -/* Incremented for every log file that is output. */ -static uint64_t log_seq = 0; -static char log_filename[ - /* Minimize memory bloat for non-prof builds. */ -#ifdef JEMALLOC_PROF - PATH_MAX + -#endif - 1]; - -/* Timestamp for most recent call to log_start(). */ -static nstime_t log_start_timestamp = NSTIME_ZERO_INITIALIZER; - -/* Increment these when adding to the log_bt and log_thr linked lists. */ -static size_t log_bt_index = 0; -static size_t log_thr_index = 0; - -/* Linked list node definitions. These are only used in prof.c. */ -typedef struct prof_bt_node_s prof_bt_node_t; - -struct prof_bt_node_s { - prof_bt_node_t *next; - size_t index; - prof_bt_t bt; - /* Variable size backtrace vector pointed to by bt. */ - void *vec[1]; -}; - -typedef struct prof_thr_node_s prof_thr_node_t; - -struct prof_thr_node_s { - prof_thr_node_t *next; - size_t index; - uint64_t thr_uid; - /* Variable size based on thr_name_sz. */ - char name[1]; -}; - -typedef struct prof_alloc_node_s prof_alloc_node_t; - -/* This is output when logging sampled allocations. */ -struct prof_alloc_node_s { - prof_alloc_node_t *next; - /* Indices into an array of thread data. */ - size_t alloc_thr_ind; - size_t free_thr_ind; - - /* Indices into an array of backtraces. */ - size_t alloc_bt_ind; - size_t free_bt_ind; - - uint64_t alloc_time_ns; - uint64_t free_time_ns; - - size_t usize; -}; - -/* - * Created on the first call to prof_log_start and deleted on prof_log_stop. - * These are the backtraces and threads that have already been logged by an - * allocation. - */ -static bool log_tables_initialized = false; -static ckh_t log_bt_node_set; -static ckh_t log_thr_node_set; - -/* Store linked lists for logged data. */ -static prof_bt_node_t *log_bt_first = NULL; -static prof_bt_node_t *log_bt_last = NULL; -static prof_thr_node_t *log_thr_first = NULL; -static prof_thr_node_t *log_thr_last = NULL; -static prof_alloc_node_t *log_alloc_first = NULL; -static prof_alloc_node_t *log_alloc_last = NULL; - -/* Protects the prof_logging_state and any log_{...} variable. */ -static malloc_mutex_t log_mtx; - /* * Table of mutexes that are shared among gctx's. These are leaf locks, so * there is no problem with using them for more than one gctx at the same time. @@ -241,12 +145,6 @@ static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached); static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name); -/* Hashtable functions for log_bt_node_set and log_thr_node_set. */ -static void prof_thr_node_hash(const void *key, size_t r_hash[2]); -static bool prof_thr_node_keycomp(const void *k1, const void *k2); -static void prof_bt_node_hash(const void *key, size_t r_hash[2]); -static bool prof_bt_node_keycomp(const void *k1, const void *k2); - /******************************************************************************/ /* Red-black trees. */ @@ -344,12 +242,6 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) { prof_tctx_set(tsdn, ptr, usize, NULL, tctx); - /* Get the current time and set this in the extent_t. We'll read this - * when free() is called. */ - nstime_t t = NSTIME_ZERO_INITIALIZER; - nstime_update(&t); - prof_alloc_time_set(tsdn, ptr, NULL, t); - malloc_mutex_lock(tsdn, tctx->tdata->lock); tctx->cnts.curobjs++; tctx->cnts.curbytes += usize; @@ -361,174 +253,14 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, malloc_mutex_unlock(tsdn, tctx->tdata->lock); } -static size_t -prof_log_bt_index(tsd_t *tsd, prof_bt_t *bt) { - assert(prof_logging_state == prof_logging_state_started); - malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx); - - prof_bt_node_t dummy_node; - dummy_node.bt = *bt; - prof_bt_node_t *node; - - /* See if this backtrace is already cached in the table. */ - if (ckh_search(&log_bt_node_set, (void *)(&dummy_node), - (void **)(&node), NULL)) { - size_t sz = offsetof(prof_bt_node_t, vec) + - (bt->len * sizeof(void *)); - prof_bt_node_t *new_node = (prof_bt_node_t *) - iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, - true, arena_get(TSDN_NULL, 0, true), true); - if (log_bt_first == NULL) { - log_bt_first = new_node; - log_bt_last = new_node; - } else { - log_bt_last->next = new_node; - log_bt_last = new_node; - } - - new_node->next = NULL; - new_node->index = log_bt_index; - /* - * Copy the backtrace: bt is inside a tdata or gctx, which - * might die before prof_log_stop is called. - */ - new_node->bt.len = bt->len; - memcpy(new_node->vec, bt->vec, bt->len * sizeof(void *)); - new_node->bt.vec = new_node->vec; - - log_bt_index++; - ckh_insert(tsd, &log_bt_node_set, (void *)new_node, NULL); - return new_node->index; - } else { - return node->index; - } -} -static size_t -prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) { - assert(prof_logging_state == prof_logging_state_started); - malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx); - - prof_thr_node_t dummy_node; - dummy_node.thr_uid = thr_uid; - prof_thr_node_t *node; - - /* See if this thread is already cached in the table. */ - if (ckh_search(&log_thr_node_set, (void *)(&dummy_node), - (void **)(&node), NULL)) { - size_t sz = offsetof(prof_thr_node_t, name) + strlen(name) + 1; - prof_thr_node_t *new_node = (prof_thr_node_t *) - iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, - true, arena_get(TSDN_NULL, 0, true), true); - if (log_thr_first == NULL) { - log_thr_first = new_node; - log_thr_last = new_node; - } else { - log_thr_last->next = new_node; - log_thr_last = new_node; - } - - new_node->next = NULL; - new_node->index = log_thr_index; - new_node->thr_uid = thr_uid; - strcpy(new_node->name, name); - - log_thr_index++; - ckh_insert(tsd, &log_thr_node_set, (void *)new_node, NULL); - return new_node->index; - } else { - return node->index; - } -} - -static void -prof_try_log(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx) { - malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); - - prof_tdata_t *cons_tdata = prof_tdata_get(tsd, false); - if (cons_tdata == NULL) { - /* - * We decide not to log these allocations. cons_tdata will be - * NULL only when the current thread is in a weird state (e.g. - * it's being destroyed). - */ - return; - } - - malloc_mutex_lock(tsd_tsdn(tsd), &log_mtx); - - if (prof_logging_state != prof_logging_state_started) { - goto label_done; - } - - if (!log_tables_initialized) { - bool err1 = ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS, - prof_bt_node_hash, prof_bt_node_keycomp); - bool err2 = ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS, - prof_thr_node_hash, prof_thr_node_keycomp); - if (err1 || err2) { - goto label_done; - } - log_tables_initialized = true; - } - - nstime_t alloc_time = prof_alloc_time_get(tsd_tsdn(tsd), ptr, - (alloc_ctx_t *)NULL); - nstime_t free_time = NSTIME_ZERO_INITIALIZER; - nstime_update(&free_time); - - size_t sz = sizeof(prof_alloc_node_t); - prof_alloc_node_t *new_node = (prof_alloc_node_t *) - iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true, - arena_get(TSDN_NULL, 0, true), true); - - const char *prod_thr_name = (tctx->tdata->thread_name == NULL)? - "" : tctx->tdata->thread_name; - const char *cons_thr_name = prof_thread_name_get(tsd); - - prof_bt_t bt; - /* Initialize the backtrace, using the buffer in tdata to store it. */ - bt_init(&bt, cons_tdata->vec); - prof_backtrace(&bt); - prof_bt_t *cons_bt = &bt; - - /* We haven't destroyed tctx yet, so gctx should be good to read. */ - prof_bt_t *prod_bt = &tctx->gctx->bt; - - new_node->next = NULL; - new_node->alloc_thr_ind = prof_log_thr_index(tsd, tctx->tdata->thr_uid, - prod_thr_name); - new_node->free_thr_ind = prof_log_thr_index(tsd, cons_tdata->thr_uid, - cons_thr_name); - new_node->alloc_bt_ind = prof_log_bt_index(tsd, prod_bt); - new_node->free_bt_ind = prof_log_bt_index(tsd, cons_bt); - new_node->alloc_time_ns = nstime_ns(&alloc_time); - new_node->free_time_ns = nstime_ns(&free_time); - new_node->usize = usize; - - if (log_alloc_first == NULL) { - log_alloc_first = new_node; - log_alloc_last = new_node; - } else { - log_alloc_last->next = new_node; - log_alloc_last = new_node; - } - -label_done: - malloc_mutex_unlock(tsd_tsdn(tsd), &log_mtx); -} - void -prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize, - prof_tctx_t *tctx) { +prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) { malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); - assert(tctx->cnts.curobjs > 0); assert(tctx->cnts.curbytes >= usize); tctx->cnts.curobjs--; tctx->cnts.curbytes -= usize; - prof_try_log(tsd, ptr, usize, tctx); - if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) { prof_tctx_destroy(tsd, tctx); } else { @@ -1139,12 +871,15 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) { void prof_sample_threshold_update(prof_tdata_t *tdata) { #ifdef JEMALLOC_PROF + uint64_t r; + double u; + if (!config_prof) { return; } if (lg_prof_sample == 0) { - tsd_bytes_until_sample_set(tsd_fetch(), 0); + tdata->bytes_until_sample = 0; return; } @@ -1166,16 +901,11 @@ prof_sample_threshold_update(prof_tdata_t *tdata) { * pp 500 * (http://luc.devroye.org/rnbookindex.html) */ - uint64_t r = prng_lg_range_u64(&tdata->prng_state, 53); - double u = (double)r * (1.0/9007199254740992.0L); - uint64_t bytes_until_sample = (uint64_t)(log(u) / + r = prng_lg_range_u64(&tdata->prng_state, 53); + u = (double)r * (1.0/9007199254740992.0L); + tdata->bytes_until_sample = (uint64_t)(log(u) / log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) + (uint64_t)1U; - if (bytes_until_sample > SSIZE_MAX) { - bytes_until_sample = SSIZE_MAX; - } - tsd_bytes_until_sample_set(tsd_fetch(), bytes_until_sample); - #endif } @@ -1292,7 +1022,7 @@ prof_dump_write(bool propagate_err, const char *s) { } } - if (prof_dump_buf_end + slen - i <= PROF_DUMP_BUFSIZE) { + if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { /* Finish writing. */ n = slen - i; } else { @@ -1303,7 +1033,6 @@ prof_dump_write(bool propagate_err, const char *s) { prof_dump_buf_end += n; i += n; } - assert(i == slen); return false; } @@ -2158,33 +1887,6 @@ prof_bt_keycomp(const void *k1, const void *k2) { return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); } -static void -prof_bt_node_hash(const void *key, size_t r_hash[2]) { - const prof_bt_node_t *bt_node = (prof_bt_node_t *)key; - prof_bt_hash((void *)(&bt_node->bt), r_hash); -} - -static bool -prof_bt_node_keycomp(const void *k1, const void *k2) { - const prof_bt_node_t *bt_node1 = (prof_bt_node_t *)k1; - const prof_bt_node_t *bt_node2 = (prof_bt_node_t *)k2; - return prof_bt_keycomp((void *)(&bt_node1->bt), - (void *)(&bt_node2->bt)); -} - -static void -prof_thr_node_hash(const void *key, size_t r_hash[2]) { - const prof_thr_node_t *thr_node = (prof_thr_node_t *)key; - hash(&thr_node->thr_uid, sizeof(uint64_t), 0x94122f35U, r_hash); -} - -static bool -prof_thr_node_keycomp(const void *k1, const void *k2) { - const prof_thr_node_t *thr_node1 = (prof_thr_node_t *)k1; - const prof_thr_node_t *thr_node2 = (prof_thr_node_t *)k2; - return thr_node1->thr_uid == thr_node2->thr_uid; -} - static uint64_t prof_thr_uid_alloc(tsdn_t *tsdn) { uint64_t thr_uid; @@ -2417,368 +2119,6 @@ prof_active_set(tsdn_t *tsdn, bool active) { return prof_active_old; } -#ifdef JEMALLOC_JET -size_t -prof_log_bt_count(void) { - size_t cnt = 0; - prof_bt_node_t *node = log_bt_first; - while (node != NULL) { - cnt++; - node = node->next; - } - return cnt; -} - -size_t -prof_log_alloc_count(void) { - size_t cnt = 0; - prof_alloc_node_t *node = log_alloc_first; - while (node != NULL) { - cnt++; - node = node->next; - } - return cnt; -} - -size_t -prof_log_thr_count(void) { - size_t cnt = 0; - prof_thr_node_t *node = log_thr_first; - while (node != NULL) { - cnt++; - node = node->next; - } - return cnt; -} - -bool -prof_log_is_logging(void) { - return prof_logging_state == prof_logging_state_started; -} - -bool -prof_log_rep_check(void) { - if (prof_logging_state == prof_logging_state_stopped - && log_tables_initialized) { - return true; - } - - if (log_bt_last != NULL && log_bt_last->next != NULL) { - return true; - } - if (log_thr_last != NULL && log_thr_last->next != NULL) { - return true; - } - if (log_alloc_last != NULL && log_alloc_last->next != NULL) { - return true; - } - - size_t bt_count = prof_log_bt_count(); - size_t thr_count = prof_log_thr_count(); - size_t alloc_count = prof_log_alloc_count(); - - - if (prof_logging_state == prof_logging_state_stopped) { - if (bt_count != 0 || thr_count != 0 || alloc_count || 0) { - return true; - } - } - - prof_alloc_node_t *node = log_alloc_first; - while (node != NULL) { - if (node->alloc_bt_ind >= bt_count) { - return true; - } - if (node->free_bt_ind >= bt_count) { - return true; - } - if (node->alloc_thr_ind >= thr_count) { - return true; - } - if (node->free_thr_ind >= thr_count) { - return true; - } - if (node->alloc_time_ns > node->free_time_ns) { - return true; - } - node = node->next; - } - - return false; -} - -void -prof_log_dummy_set(bool new_value) { - prof_log_dummy = new_value; -} -#endif - -bool -prof_log_start(tsdn_t *tsdn, const char *filename) { - if (!opt_prof || !prof_booted) { - return true; - } - - bool ret = false; - size_t buf_size = PATH_MAX + 1; - - malloc_mutex_lock(tsdn, &log_mtx); - - if (prof_logging_state != prof_logging_state_stopped) { - ret = true; - } else if (filename == NULL) { - /* Make default name. */ - malloc_snprintf(log_filename, buf_size, "%s.%d.%"FMTu64".json", - opt_prof_prefix, prof_getpid(), log_seq); - log_seq++; - prof_logging_state = prof_logging_state_started; - } else if (strlen(filename) >= buf_size) { - ret = true; - } else { - strcpy(log_filename, filename); - prof_logging_state = prof_logging_state_started; - } - - if (!ret) { - nstime_update(&log_start_timestamp); - } - - malloc_mutex_unlock(tsdn, &log_mtx); - - return ret; -} - -/* Used as an atexit function to stop logging on exit. */ -static void -prof_log_stop_final(void) { - tsd_t *tsd = tsd_fetch(); - prof_log_stop(tsd_tsdn(tsd)); -} - -struct prof_emitter_cb_arg_s { - int fd; - ssize_t ret; -}; - -static void -prof_emitter_write_cb(void *opaque, const char *to_write) { - struct prof_emitter_cb_arg_s *arg = - (struct prof_emitter_cb_arg_s *)opaque; - size_t bytes = strlen(to_write); -#ifdef JEMALLOC_JET - if (prof_log_dummy) { - return; - } -#endif - arg->ret = write(arg->fd, (void *)to_write, bytes); -} - -/* - * prof_log_emit_{...} goes through the appropriate linked list, emitting each - * node to the json and deallocating it. - */ -static void -prof_log_emit_threads(tsd_t *tsd, emitter_t *emitter) { - emitter_json_array_kv_begin(emitter, "threads"); - prof_thr_node_t *thr_node = log_thr_first; - prof_thr_node_t *thr_old_node; - while (thr_node != NULL) { - emitter_json_object_begin(emitter); - - emitter_json_kv(emitter, "thr_uid", emitter_type_uint64, - &thr_node->thr_uid); - - char *thr_name = thr_node->name; - - emitter_json_kv(emitter, "thr_name", emitter_type_string, - &thr_name); - - emitter_json_object_end(emitter); - thr_old_node = thr_node; - thr_node = thr_node->next; - idalloc(tsd, thr_old_node); - } - emitter_json_array_end(emitter); -} - -static void -prof_log_emit_traces(tsd_t *tsd, emitter_t *emitter) { - emitter_json_array_kv_begin(emitter, "stack_traces"); - prof_bt_node_t *bt_node = log_bt_first; - prof_bt_node_t *bt_old_node; - /* - * Calculate how many hex digits we need: twice number of bytes, two for - * "0x", and then one more for terminating '\0'. - */ - char buf[2 * sizeof(intptr_t) + 3]; - size_t buf_sz = sizeof(buf); - while (bt_node != NULL) { - emitter_json_array_begin(emitter); - size_t i; - for (i = 0; i < bt_node->bt.len; i++) { - malloc_snprintf(buf, buf_sz, "%p", bt_node->bt.vec[i]); - char *trace_str = buf; - emitter_json_value(emitter, emitter_type_string, - &trace_str); - } - emitter_json_array_end(emitter); - - bt_old_node = bt_node; - bt_node = bt_node->next; - idalloc(tsd, bt_old_node); - } - emitter_json_array_end(emitter); -} - -static void -prof_log_emit_allocs(tsd_t *tsd, emitter_t *emitter) { - emitter_json_array_kv_begin(emitter, "allocations"); - prof_alloc_node_t *alloc_node = log_alloc_first; - prof_alloc_node_t *alloc_old_node; - while (alloc_node != NULL) { - emitter_json_object_begin(emitter); - - emitter_json_kv(emitter, "alloc_thread", emitter_type_size, - &alloc_node->alloc_thr_ind); - - emitter_json_kv(emitter, "free_thread", emitter_type_size, - &alloc_node->free_thr_ind); - - emitter_json_kv(emitter, "alloc_trace", emitter_type_size, - &alloc_node->alloc_bt_ind); - - emitter_json_kv(emitter, "free_trace", emitter_type_size, - &alloc_node->free_bt_ind); - - emitter_json_kv(emitter, "alloc_timestamp", - emitter_type_uint64, &alloc_node->alloc_time_ns); - - emitter_json_kv(emitter, "free_timestamp", emitter_type_uint64, - &alloc_node->free_time_ns); - - emitter_json_kv(emitter, "usize", emitter_type_uint64, - &alloc_node->usize); - - emitter_json_object_end(emitter); - - alloc_old_node = alloc_node; - alloc_node = alloc_node->next; - idalloc(tsd, alloc_old_node); - } - emitter_json_array_end(emitter); -} - -static void -prof_log_emit_metadata(emitter_t *emitter) { - emitter_json_object_kv_begin(emitter, "info"); - - nstime_t now = NSTIME_ZERO_INITIALIZER; - - nstime_update(&now); - uint64_t ns = nstime_ns(&now) - nstime_ns(&log_start_timestamp); - emitter_json_kv(emitter, "duration", emitter_type_uint64, &ns); - - char *vers = JEMALLOC_VERSION; - emitter_json_kv(emitter, "version", - emitter_type_string, &vers); - - emitter_json_kv(emitter, "lg_sample_rate", - emitter_type_int, &lg_prof_sample); - - int pid = prof_getpid(); - emitter_json_kv(emitter, "pid", emitter_type_int, &pid); - - emitter_json_object_end(emitter); -} - - -bool -prof_log_stop(tsdn_t *tsdn) { - if (!opt_prof || !prof_booted) { - return true; - } - - tsd_t *tsd = tsdn_tsd(tsdn); - malloc_mutex_lock(tsdn, &log_mtx); - - if (prof_logging_state != prof_logging_state_started) { - malloc_mutex_unlock(tsdn, &log_mtx); - return true; - } - - /* - * Set the state to dumping. We'll set it to stopped when we're done. - * Since other threads won't be able to start/stop/log when the state is - * dumping, we don't have to hold the lock during the whole method. - */ - prof_logging_state = prof_logging_state_dumping; - malloc_mutex_unlock(tsdn, &log_mtx); - - - emitter_t emitter; - - /* Create a file. */ - - int fd; -#ifdef JEMALLOC_JET - if (prof_log_dummy) { - fd = 0; - } else { - fd = creat(log_filename, 0644); - } -#else - fd = creat(log_filename, 0644); -#endif - - if (fd == -1) { - malloc_printf(": creat() for log file \"%s\" " - " failed with %d\n", log_filename, errno); - if (opt_abort) { - abort(); - } - return true; - } - - /* Emit to json. */ - struct prof_emitter_cb_arg_s arg; - arg.fd = fd; - emitter_init(&emitter, emitter_output_json, &prof_emitter_write_cb, - (void *)(&arg)); - - emitter_begin(&emitter); - prof_log_emit_metadata(&emitter); - prof_log_emit_threads(tsd, &emitter); - prof_log_emit_traces(tsd, &emitter); - prof_log_emit_allocs(tsd, &emitter); - emitter_end(&emitter); - - /* Reset global state. */ - if (log_tables_initialized) { - ckh_delete(tsd, &log_bt_node_set); - ckh_delete(tsd, &log_thr_node_set); - } - log_tables_initialized = false; - log_bt_index = 0; - log_thr_index = 0; - log_bt_first = NULL; - log_bt_last = NULL; - log_thr_first = NULL; - log_thr_last = NULL; - log_alloc_first = NULL; - log_alloc_last = NULL; - - malloc_mutex_lock(tsdn, &log_mtx); - prof_logging_state = prof_logging_state_stopped; - malloc_mutex_unlock(tsdn, &log_mtx); - -#ifdef JEMALLOC_JET - if (prof_log_dummy) { - return false; - } -#endif - return close(fd); -} - const char * prof_thread_name_get(tsd_t *tsd) { prof_tdata_t *tdata; @@ -3015,35 +2355,6 @@ prof_boot2(tsd_t *tsd) { } } - if (opt_prof_log) { - prof_log_start(tsd_tsdn(tsd), NULL); - } - - if (atexit(prof_log_stop_final) != 0) { - malloc_write(": Error in atexit() " - "for logging\n"); - if (opt_abort) { - abort(); - } - } - - if (malloc_mutex_init(&log_mtx, "prof_log", - WITNESS_RANK_PROF_LOG, malloc_mutex_rank_exclusive)) { - return true; - } - - if (ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS, - prof_bt_node_hash, prof_bt_node_keycomp)) { - return true; - } - - if (ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS, - prof_thr_node_hash, prof_thr_node_keycomp)) { - return true; - } - - log_tables_initialized = true; - gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), b0get(), PROF_NCTX_LOCKS * sizeof(malloc_mutex_t), CACHELINE); @@ -3071,14 +2382,16 @@ prof_boot2(tsd_t *tsd) { return true; } } -#ifdef JEMALLOC_PROF_LIBGCC - /* - * Cause the backtracing machinery to allocate its internal - * state before enabling profiling. - */ - _Unwind_Backtrace(prof_unwind_init_callback, NULL); -#endif } + +#ifdef JEMALLOC_PROF_LIBGCC + /* + * Cause the backtracing machinery to allocate its internal state + * before enabling profiling. + */ + _Unwind_Backtrace(prof_unwind_init_callback, NULL); +#endif + prof_booted = true; return false; diff --git a/deps/jemalloc/src/rtree.c b/deps/jemalloc/src/rtree.c index 4ae41fe..53702cf 100644 --- a/deps/jemalloc/src/rtree.c +++ b/deps/jemalloc/src/rtree.c @@ -39,7 +39,7 @@ rtree_node_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *node) { /* Nodes are never deleted during normal operation. */ not_reached(); } -rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc = +UNUSED rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc = rtree_node_dalloc_impl; static rtree_leaf_elm_t * @@ -54,7 +54,7 @@ rtree_leaf_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *leaf) { /* Leaves are never deleted during normal operation. */ not_reached(); } -rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc = +UNUSED rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc = rtree_leaf_dalloc_impl; #ifdef JEMALLOC_JET diff --git a/deps/jemalloc/src/safety_check.c b/deps/jemalloc/src/safety_check.c deleted file mode 100644 index 804155d..0000000 --- a/deps/jemalloc/src/safety_check.c +++ /dev/null @@ -1,24 +0,0 @@ -#include "jemalloc/internal/jemalloc_preamble.h" -#include "jemalloc/internal/jemalloc_internal_includes.h" - -static void (*safety_check_abort)(const char *message); - -void safety_check_set_abort(void (*abort_fn)(const char *)) { - safety_check_abort = abort_fn; -} - -void safety_check_fail(const char *format, ...) { - char buf[MALLOC_PRINTF_BUFSIZE]; - - va_list ap; - va_start(ap, format); - malloc_vsnprintf(buf, MALLOC_PRINTF_BUFSIZE, format, ap); - va_end(ap); - - if (safety_check_abort == NULL) { - malloc_write(buf); - abort(); - } else { - safety_check_abort(buf); - } -} diff --git a/deps/jemalloc/src/sc.c b/deps/jemalloc/src/sc.c deleted file mode 100644 index 89ddb6b..0000000 --- a/deps/jemalloc/src/sc.c +++ /dev/null @@ -1,313 +0,0 @@ -#include "jemalloc/internal/jemalloc_preamble.h" - -#include "jemalloc/internal/assert.h" -#include "jemalloc/internal/bit_util.h" -#include "jemalloc/internal/bitmap.h" -#include "jemalloc/internal/pages.h" -#include "jemalloc/internal/sc.h" - -/* - * This module computes the size classes used to satisfy allocations. The logic - * here was ported more or less line-by-line from a shell script, and because of - * that is not the most idiomatic C. Eventually we should fix this, but for now - * at least the damage is compartmentalized to this file. - */ - -sc_data_t sc_data_global; - -static size_t -reg_size_compute(int lg_base, int lg_delta, int ndelta) { - return (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta); -} - -/* Returns the number of pages in the slab. */ -static int -slab_size(int lg_page, int lg_base, int lg_delta, int ndelta) { - size_t page = (ZU(1) << lg_page); - size_t reg_size = reg_size_compute(lg_base, lg_delta, ndelta); - - size_t try_slab_size = page; - size_t try_nregs = try_slab_size / reg_size; - size_t perfect_slab_size = 0; - bool perfect = false; - /* - * This loop continues until we find the least common multiple of the - * page size and size class size. Size classes are all of the form - * base + ndelta * delta == (ndelta + base/ndelta) * delta, which is - * (ndelta + ngroup) * delta. The way we choose slabbing strategies - * means that delta is at most the page size and ndelta < ngroup. So - * the loop executes for at most 2 * ngroup - 1 iterations, which is - * also the bound on the number of pages in a slab chosen by default. - * With the current default settings, this is at most 7. - */ - while (!perfect) { - perfect_slab_size = try_slab_size; - size_t perfect_nregs = try_nregs; - try_slab_size += page; - try_nregs = try_slab_size / reg_size; - if (perfect_slab_size == perfect_nregs * reg_size) { - perfect = true; - } - } - return (int)(perfect_slab_size / page); -} - -static void -size_class( - /* Output. */ - sc_t *sc, - /* Configuration decisions. */ - int lg_max_lookup, int lg_page, int lg_ngroup, - /* Inputs specific to the size class. */ - int index, int lg_base, int lg_delta, int ndelta) { - sc->index = index; - sc->lg_base = lg_base; - sc->lg_delta = lg_delta; - sc->ndelta = ndelta; - sc->psz = (reg_size_compute(lg_base, lg_delta, ndelta) - % (ZU(1) << lg_page) == 0); - size_t size = (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta); - if (index == 0) { - assert(!sc->psz); - } - if (size < (ZU(1) << (lg_page + lg_ngroup))) { - sc->bin = true; - sc->pgs = slab_size(lg_page, lg_base, lg_delta, ndelta); - } else { - sc->bin = false; - sc->pgs = 0; - } - if (size <= (ZU(1) << lg_max_lookup)) { - sc->lg_delta_lookup = lg_delta; - } else { - sc->lg_delta_lookup = 0; - } -} - -static void -size_classes( - /* Output. */ - sc_data_t *sc_data, - /* Determined by the system. */ - size_t lg_ptr_size, int lg_quantum, - /* Configuration decisions. */ - int lg_tiny_min, int lg_max_lookup, int lg_page, int lg_ngroup) { - int ptr_bits = (1 << lg_ptr_size) * 8; - int ngroup = (1 << lg_ngroup); - int ntiny = 0; - int nlbins = 0; - int lg_tiny_maxclass = (unsigned)-1; - int nbins = 0; - int npsizes = 0; - - int index = 0; - - int ndelta = 0; - int lg_base = lg_tiny_min; - int lg_delta = lg_base; - - /* Outputs that we update as we go. */ - size_t lookup_maxclass = 0; - size_t small_maxclass = 0; - int lg_large_minclass = 0; - size_t large_maxclass = 0; - - /* Tiny size classes. */ - while (lg_base < lg_quantum) { - sc_t *sc = &sc_data->sc[index]; - size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, - lg_base, lg_delta, ndelta); - if (sc->lg_delta_lookup != 0) { - nlbins = index + 1; - } - if (sc->psz) { - npsizes++; - } - if (sc->bin) { - nbins++; - } - ntiny++; - /* Final written value is correct. */ - lg_tiny_maxclass = lg_base; - index++; - lg_delta = lg_base; - lg_base++; - } - - /* First non-tiny (pseudo) group. */ - if (ntiny != 0) { - sc_t *sc = &sc_data->sc[index]; - /* - * See the note in sc.h; the first non-tiny size class has an - * unusual encoding. - */ - lg_base--; - ndelta = 1; - size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, - lg_base, lg_delta, ndelta); - index++; - lg_base++; - lg_delta++; - if (sc->psz) { - npsizes++; - } - if (sc->bin) { - nbins++; - } - } - while (ndelta < ngroup) { - sc_t *sc = &sc_data->sc[index]; - size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, - lg_base, lg_delta, ndelta); - index++; - ndelta++; - if (sc->psz) { - npsizes++; - } - if (sc->bin) { - nbins++; - } - } - - /* All remaining groups. */ - lg_base = lg_base + lg_ngroup; - while (lg_base < ptr_bits - 1) { - ndelta = 1; - int ndelta_limit; - if (lg_base == ptr_bits - 2) { - ndelta_limit = ngroup - 1; - } else { - ndelta_limit = ngroup; - } - while (ndelta <= ndelta_limit) { - sc_t *sc = &sc_data->sc[index]; - size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, - lg_base, lg_delta, ndelta); - if (sc->lg_delta_lookup != 0) { - nlbins = index + 1; - /* Final written value is correct. */ - lookup_maxclass = (ZU(1) << lg_base) - + (ZU(ndelta) << lg_delta); - } - if (sc->psz) { - npsizes++; - } - if (sc->bin) { - nbins++; - /* Final written value is correct. */ - small_maxclass = (ZU(1) << lg_base) - + (ZU(ndelta) << lg_delta); - if (lg_ngroup > 0) { - lg_large_minclass = lg_base + 1; - } else { - lg_large_minclass = lg_base + 2; - } - } - large_maxclass = (ZU(1) << lg_base) - + (ZU(ndelta) << lg_delta); - index++; - ndelta++; - } - lg_base++; - lg_delta++; - } - /* Additional outputs. */ - int nsizes = index; - unsigned lg_ceil_nsizes = lg_ceil(nsizes); - - /* Fill in the output data. */ - sc_data->ntiny = ntiny; - sc_data->nlbins = nlbins; - sc_data->nbins = nbins; - sc_data->nsizes = nsizes; - sc_data->lg_ceil_nsizes = lg_ceil_nsizes; - sc_data->npsizes = npsizes; - sc_data->lg_tiny_maxclass = lg_tiny_maxclass; - sc_data->lookup_maxclass = lookup_maxclass; - sc_data->small_maxclass = small_maxclass; - sc_data->lg_large_minclass = lg_large_minclass; - sc_data->large_minclass = (ZU(1) << lg_large_minclass); - sc_data->large_maxclass = large_maxclass; - - /* - * We compute these values in two ways: - * - Incrementally, as above. - * - In macros, in sc.h. - * The computation is easier when done incrementally, but putting it in - * a constant makes it available to the fast paths without having to - * touch the extra global cacheline. We assert, however, that the two - * computations are equivalent. - */ - assert(sc_data->npsizes == SC_NPSIZES); - assert(sc_data->lg_tiny_maxclass == SC_LG_TINY_MAXCLASS); - assert(sc_data->small_maxclass == SC_SMALL_MAXCLASS); - assert(sc_data->large_minclass == SC_LARGE_MINCLASS); - assert(sc_data->lg_large_minclass == SC_LG_LARGE_MINCLASS); - assert(sc_data->large_maxclass == SC_LARGE_MAXCLASS); - - /* - * In the allocation fastpath, we want to assume that we can - * unconditionally subtract the requested allocation size from - * a ssize_t, and detect passing through 0 correctly. This - * results in optimal generated code. For this to work, the - * maximum allocation size must be less than SSIZE_MAX. - */ - assert(SC_LARGE_MAXCLASS < SSIZE_MAX); -} - -void -sc_data_init(sc_data_t *sc_data) { - assert(!sc_data->initialized); - - int lg_max_lookup = 12; - - size_classes(sc_data, LG_SIZEOF_PTR, LG_QUANTUM, SC_LG_TINY_MIN, - lg_max_lookup, LG_PAGE, 2); - - sc_data->initialized = true; -} - -static void -sc_data_update_sc_slab_size(sc_t *sc, size_t reg_size, size_t pgs_guess) { - size_t min_pgs = reg_size / PAGE; - if (reg_size % PAGE != 0) { - min_pgs++; - } - /* - * BITMAP_MAXBITS is actually determined by putting the smallest - * possible size-class on one page, so this can never be 0. - */ - size_t max_pgs = BITMAP_MAXBITS * reg_size / PAGE; - - assert(min_pgs <= max_pgs); - assert(min_pgs > 0); - assert(max_pgs >= 1); - if (pgs_guess < min_pgs) { - sc->pgs = (int)min_pgs; - } else if (pgs_guess > max_pgs) { - sc->pgs = (int)max_pgs; - } else { - sc->pgs = (int)pgs_guess; - } -} - -void -sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, int pgs) { - assert(data->initialized); - for (int i = 0; i < data->nsizes; i++) { - sc_t *sc = &data->sc[i]; - if (!sc->bin) { - break; - } - size_t reg_size = reg_size_compute(sc->lg_base, sc->lg_delta, - sc->ndelta); - if (begin <= reg_size && reg_size <= end) { - sc_data_update_sc_slab_size(sc, reg_size, pgs); - } - } -} - -void -sc_boot(sc_data_t *data) { - sc_data_init(data); -} diff --git a/deps/jemalloc/src/stats.c b/deps/jemalloc/src/stats.c index 118e05d..08b9507 100644 --- a/deps/jemalloc/src/stats.c +++ b/deps/jemalloc/src/stats.c @@ -52,20 +52,6 @@ char opt_stats_print_opts[stats_print_tot_num_options+1] = ""; /******************************************************************************/ -static uint64_t -rate_per_second(uint64_t value, uint64_t uptime_ns) { - uint64_t billion = 1000000000; - if (uptime_ns == 0 || value == 0) { - return 0; - } - if (uptime_ns < billion) { - return value; - } else { - uint64_t uptime_s = uptime_ns / billion; - return value / uptime_s; - } -} - /* Calculate x.yyy and output a string (takes a fixed sized char array). */ static bool get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) { @@ -118,26 +104,24 @@ mutex_stats_init_cols(emitter_row_t *row, const char *table_name, #define WIDTH_uint32_t 12 #define WIDTH_uint64_t 16 -#define OP(counter, counter_type, human, derived, base_counter) \ +#define OP(counter, counter_type, human) \ col = &col_##counter_type[k_##counter_type]; \ ++k_##counter_type; \ emitter_col_init(col, row); \ col->justify = emitter_justify_right; \ - col->width = derived ? 8 : WIDTH_##counter_type; \ + col->width = WIDTH_##counter_type; \ col->type = emitter_type_title; \ col->str_val = human; MUTEX_PROF_COUNTERS #undef OP #undef WIDTH_uint32_t #undef WIDTH_uint64_t - col_uint64_t[mutex_counter_total_wait_time_ps].width = 10; } static void mutex_stats_read_global(const char *name, emitter_col_t *col_name, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], - uint64_t uptime) { + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { char cmd[MUTEX_CTL_STR_MAX_LENGTH]; col_name->str_val = name; @@ -145,17 +129,12 @@ mutex_stats_read_global(const char *name, emitter_col_t *col_name, emitter_col_t *dst; #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human, derived, base_counter) \ +#define OP(counter, counter_type, human) \ dst = &col_##counter_type[mutex_counter_##counter]; \ dst->type = EMITTER_TYPE_##counter_type; \ - if (!derived) { \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "mutexes", name, #counter); \ - CTL_GET(cmd, (counter_type *)&dst->bool_val, counter_type); \ - } else { \ - emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ - dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ - } + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "mutexes", name, #counter); \ + CTL_GET(cmd, (counter_type *)&dst->bool_val, counter_type); MUTEX_PROF_COUNTERS #undef OP #undef EMITTER_TYPE_uint32_t @@ -166,8 +145,7 @@ static void mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind, const char *name, emitter_col_t *col_name, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], - uint64_t uptime) { + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { char cmd[MUTEX_CTL_STR_MAX_LENGTH]; col_name->str_val = name; @@ -175,17 +153,13 @@ mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind, emitter_col_t *dst; #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human, derived, base_counter) \ +#define OP(counter, counter_type, human) \ dst = &col_##counter_type[mutex_counter_##counter]; \ dst->type = EMITTER_TYPE_##counter_type; \ - if (!derived) { \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "arenas.0.mutexes", arena_mutex_names[mutex_ind], #counter);\ - CTL_M2_GET(cmd, arena_ind, (counter_type *)&dst->bool_val, counter_type); \ - } else { \ - emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ - dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ - } + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "arenas.0.mutexes", arena_mutex_names[mutex_ind], #counter);\ + CTL_M2_GET(cmd, arena_ind, \ + (counter_type *)&dst->bool_val, counter_type); MUTEX_PROF_COUNTERS #undef OP #undef EMITTER_TYPE_uint32_t @@ -195,25 +169,19 @@ mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind, static void mutex_stats_read_arena_bin(unsigned arena_ind, unsigned bin_ind, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], - uint64_t uptime) { + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { char cmd[MUTEX_CTL_STR_MAX_LENGTH]; emitter_col_t *dst; #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human, derived, base_counter) \ +#define OP(counter, counter_type, human) \ dst = &col_##counter_type[mutex_counter_##counter]; \ dst->type = EMITTER_TYPE_##counter_type; \ - if (!derived) { \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "arenas.0.bins.0","mutex", #counter); \ - CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \ - (counter_type *)&dst->bool_val, counter_type); \ - } else { \ - emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ - dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ - } + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "arenas.0.bins.0","mutex", #counter); \ + CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \ + (counter_type *)&dst->bool_val, counter_type); MUTEX_PROF_COUNTERS #undef OP #undef EMITTER_TYPE_uint32_t @@ -236,38 +204,19 @@ mutex_stats_emit(emitter_t *emitter, emitter_row_t *row, #define EMITTER_TYPE_uint32_t emitter_type_uint32 #define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, type, human, derived, base_counter) \ - if (!derived) { \ - col = &col_##type[k_##type]; \ - ++k_##type; \ - emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \ - (const void *)&col->bool_val); \ - } +#define OP(counter, type, human) \ + col = &col_##type[k_##type]; \ + ++k_##type; \ + emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \ + (const void *)&col->bool_val); MUTEX_PROF_COUNTERS; #undef OP #undef EMITTER_TYPE_uint32_t #undef EMITTER_TYPE_uint64_t } -#define COL(row_name, column_name, left_or_right, col_width, etype) \ - emitter_col_t col_##column_name; \ - emitter_col_init(&col_##column_name, &row_name); \ - col_##column_name.justify = emitter_justify_##left_or_right; \ - col_##column_name.width = col_width; \ - col_##column_name.type = emitter_type_##etype; - -#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \ - COL(row_name, column_name, left_or_right, col_width, etype) \ - emitter_col_t header_##column_name; \ - emitter_col_init(&header_##column_name, &header_##row_name); \ - header_##column_name.justify = emitter_justify_##left_or_right; \ - header_##column_name.width = col_width; \ - header_##column_name.type = emitter_type_title; \ - header_##column_name.str_val = human ? human : #column_name; - - static void -stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t uptime) { +stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i) { size_t page; bool in_gap, in_gap_prev; unsigned nbins, j; @@ -281,37 +230,43 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t upti emitter_row_t row; emitter_row_init(&row); +#define COL(name, left_or_right, col_width, etype) \ + emitter_col_t col_##name; \ + emitter_col_init(&col_##name, &row); \ + col_##name.justify = emitter_justify_##left_or_right; \ + col_##name.width = col_width; \ + col_##name.type = emitter_type_##etype; \ + emitter_col_t header_col_##name; \ + emitter_col_init(&header_col_##name, &header_row); \ + header_col_##name.justify = emitter_justify_##left_or_right; \ + header_col_##name.width = col_width; \ + header_col_##name.type = emitter_type_title; \ + header_col_##name.str_val = #name; - COL_HDR(row, size, NULL, right, 20, size) - COL_HDR(row, ind, NULL, right, 4, unsigned) - COL_HDR(row, allocated, NULL, right, 13, uint64) - COL_HDR(row, nmalloc, NULL, right, 13, uint64) - COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, ndalloc, NULL, right, 13, uint64) - COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, nrequests, NULL, right, 13, uint64) - COL_HDR(row, nrequests_ps, "(#/sec)", right, 10, uint64) - COL_HDR(row, nshards, NULL, right, 9, unsigned) - COL_HDR(row, curregs, NULL, right, 13, size) - COL_HDR(row, curslabs, NULL, right, 13, size) - COL_HDR(row, nonfull_slabs, NULL, right, 15, size) - COL_HDR(row, regs, NULL, right, 5, unsigned) - COL_HDR(row, pgs, NULL, right, 4, size) + COL(size, right, 20, size) + COL(ind, right, 4, unsigned) + COL(allocated, right, 13, uint64) + COL(nmalloc, right, 13, uint64) + COL(ndalloc, right, 13, uint64) + COL(nrequests, right, 13, uint64) + COL(curregs, right, 13, size) + COL(curslabs, right, 13, size) + COL(regs, right, 5, unsigned) + COL(pgs, right, 4, size) /* To buffer a right- and left-justified column. */ - COL_HDR(row, justify_spacer, NULL, right, 1, title) - COL_HDR(row, util, NULL, right, 6, title) - COL_HDR(row, nfills, NULL, right, 13, uint64) - COL_HDR(row, nfills_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, nflushes, NULL, right, 13, uint64) - COL_HDR(row, nflushes_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, nslabs, NULL, right, 13, uint64) - COL_HDR(row, nreslabs, NULL, right, 13, uint64) - COL_HDR(row, nreslabs_ps, "(#/sec)", right, 8, uint64) + COL(justify_spacer, right, 1, title) + COL(util, right, 6, title) + COL(nfills, right, 13, uint64) + COL(nflushes, right, 13, uint64) + COL(nslabs, right, 13, uint64) + COL(nreslabs, right, 13, uint64) +#undef COL /* Don't want to actually print the name. */ - header_justify_spacer.str_val = " "; + header_col_justify_spacer.str_val = " "; col_justify_spacer.str_val = " "; + emitter_col_t col_mutex64[mutex_prof_num_uint64_t_counters]; emitter_col_t col_mutex32[mutex_prof_num_uint32_t_counters]; @@ -329,17 +284,16 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t upti * We print a "bins:" header as part of the table row; we need to adjust * the header size column to compensate. */ - header_size.width -=5; + header_col_size.width -=5; emitter_table_printf(emitter, "bins:"); emitter_table_row(emitter, &header_row); - emitter_json_array_kv_begin(emitter, "bins"); + emitter_json_arr_begin(emitter, "bins"); for (j = 0, in_gap = false; j < nbins; j++) { uint64_t nslabs; size_t reg_size, slab_size, curregs; size_t curslabs; - size_t nonfull_slabs; - uint32_t nregs, nshards; + uint32_t nregs; uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; uint64_t nreslabs; @@ -356,7 +310,6 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t upti CTL_M2_GET("arenas.bin.0.size", j, ®_size, size_t); CTL_M2_GET("arenas.bin.0.nregs", j, &nregs, uint32_t); CTL_M2_GET("arenas.bin.0.slab_size", j, &slab_size, size_t); - CTL_M2_GET("arenas.bin.0.nshards", j, &nshards, uint32_t); CTL_M2_M4_GET("stats.arenas.0.bins.0.nmalloc", i, j, &nmalloc, uint64_t); @@ -374,15 +327,13 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t upti uint64_t); CTL_M2_M4_GET("stats.arenas.0.bins.0.curslabs", i, j, &curslabs, size_t); - CTL_M2_M4_GET("stats.arenas.0.bins.0.nonfull_slabs", i, j, &nonfull_slabs, - size_t); if (mutex) { mutex_stats_read_arena_bin(i, j, col_mutex64, - col_mutex32, uptime); + col_mutex32); } - emitter_json_object_begin(emitter); + emitter_json_arr_obj_begin(emitter); emitter_json_kv(emitter, "nmalloc", emitter_type_uint64, &nmalloc); emitter_json_kv(emitter, "ndalloc", emitter_type_uint64, @@ -399,15 +350,13 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t upti &nreslabs); emitter_json_kv(emitter, "curslabs", emitter_type_size, &curslabs); - emitter_json_kv(emitter, "nonfull_slabs", emitter_type_size, - &nonfull_slabs); if (mutex) { - emitter_json_object_kv_begin(emitter, "mutex"); + emitter_json_dict_begin(emitter, "mutex"); mutex_stats_emit(emitter, NULL, col_mutex64, col_mutex32); - emitter_json_object_end(emitter); + emitter_json_dict_end(emitter); } - emitter_json_object_end(emitter); + emitter_json_arr_obj_end(emitter); size_t availregs = nregs * curslabs; char util[6]; @@ -432,25 +381,17 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t upti col_ind.unsigned_val = j; col_allocated.size_val = curregs * reg_size; col_nmalloc.uint64_val = nmalloc; - col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); col_ndalloc.uint64_val = ndalloc; - col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); col_nrequests.uint64_val = nrequests; - col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime); - col_nshards.unsigned_val = nshards; col_curregs.size_val = curregs; col_curslabs.size_val = curslabs; - col_nonfull_slabs.size_val = nonfull_slabs; col_regs.unsigned_val = nregs; col_pgs.size_val = slab_size / page; col_util.str_val = util; col_nfills.uint64_val = nfills; - col_nfills_ps.uint64_val = rate_per_second(nfills, uptime); col_nflushes.uint64_val = nflushes; - col_nflushes_ps.uint64_val = rate_per_second(nflushes, uptime); col_nslabs.uint64_val = nslabs; col_nreslabs.uint64_val = nreslabs; - col_nreslabs_ps.uint64_val = rate_per_second(nreslabs, uptime); /* * Note that mutex columns were initialized above, if mutex == @@ -459,7 +400,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t upti emitter_table_row(emitter, &row); } - emitter_json_array_end(emitter); /* Close "bins". */ + emitter_json_arr_end(emitter); /* Close "bins". */ if (in_gap) { emitter_table_printf(emitter, " ---\n"); @@ -467,7 +408,7 @@ stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t upti } static void -stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { +stats_arena_lextents_print(emitter_t *emitter, unsigned i) { unsigned nbins, nlextents, j; bool in_gap, in_gap_prev; @@ -479,22 +420,34 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { emitter_row_t row; emitter_row_init(&row); - COL_HDR(row, size, NULL, right, 20, size) - COL_HDR(row, ind, NULL, right, 4, unsigned) - COL_HDR(row, allocated, NULL, right, 13, size) - COL_HDR(row, nmalloc, NULL, right, 13, uint64) - COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, ndalloc, NULL, right, 13, uint64) - COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, nrequests, NULL, right, 13, uint64) - COL_HDR(row, nrequests_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, curlextents, NULL, right, 13, size) +#define COL(name, left_or_right, col_width, etype) \ + emitter_col_t header_##name; \ + emitter_col_init(&header_##name, &header_row); \ + header_##name.justify = emitter_justify_##left_or_right; \ + header_##name.width = col_width; \ + header_##name.type = emitter_type_title; \ + header_##name.str_val = #name; \ + \ + emitter_col_t col_##name; \ + emitter_col_init(&col_##name, &row); \ + col_##name.justify = emitter_justify_##left_or_right; \ + col_##name.width = col_width; \ + col_##name.type = emitter_type_##etype; + + COL(size, right, 20, size) + COL(ind, right, 4, unsigned) + COL(allocated, right, 13, size) + COL(nmalloc, right, 13, uint64) + COL(ndalloc, right, 13, uint64) + COL(nrequests, right, 13, uint64) + COL(curlextents, right, 13, size) +#undef COL /* As with bins, we label the large extents table. */ header_size.width -= 6; emitter_table_printf(emitter, "large:"); emitter_table_row(emitter, &header_row); - emitter_json_array_kv_begin(emitter, "lextents"); + emitter_json_arr_begin(emitter, "lextents"); for (j = 0, in_gap = false; j < nlextents; j++) { uint64_t nmalloc, ndalloc, nrequests; @@ -518,122 +471,31 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { CTL_M2_M4_GET("stats.arenas.0.lextents.0.curlextents", i, j, &curlextents, size_t); - emitter_json_object_begin(emitter); + emitter_json_arr_obj_begin(emitter); emitter_json_kv(emitter, "curlextents", emitter_type_size, &curlextents); - emitter_json_object_end(emitter); + emitter_json_arr_obj_end(emitter); col_size.size_val = lextent_size; col_ind.unsigned_val = nbins + j; col_allocated.size_val = curlextents * lextent_size; col_nmalloc.uint64_val = nmalloc; - col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); col_ndalloc.uint64_val = ndalloc; - col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); col_nrequests.uint64_val = nrequests; - col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime); col_curlextents.size_val = curlextents; if (!in_gap) { emitter_table_row(emitter, &row); } } - emitter_json_array_end(emitter); /* Close "lextents". */ + emitter_json_arr_end(emitter); /* Close "lextents". */ if (in_gap) { emitter_table_printf(emitter, " ---\n"); } } static void -stats_arena_extents_print(emitter_t *emitter, unsigned i) { - unsigned j; - bool in_gap, in_gap_prev; - emitter_row_t header_row; - emitter_row_init(&header_row); - emitter_row_t row; - emitter_row_init(&row); - - COL_HDR(row, size, NULL, right, 20, size) - COL_HDR(row, ind, NULL, right, 4, unsigned) - COL_HDR(row, ndirty, NULL, right, 13, size) - COL_HDR(row, dirty, NULL, right, 13, size) - COL_HDR(row, nmuzzy, NULL, right, 13, size) - COL_HDR(row, muzzy, NULL, right, 13, size) - COL_HDR(row, nretained, NULL, right, 13, size) - COL_HDR(row, retained, NULL, right, 13, size) - COL_HDR(row, ntotal, NULL, right, 13, size) - COL_HDR(row, total, NULL, right, 13, size) - - /* Label this section. */ - header_size.width -= 8; - emitter_table_printf(emitter, "extents:"); - emitter_table_row(emitter, &header_row); - emitter_json_array_kv_begin(emitter, "extents"); - - in_gap = false; - for (j = 0; j < SC_NPSIZES; j++) { - size_t ndirty, nmuzzy, nretained, total, dirty_bytes, - muzzy_bytes, retained_bytes, total_bytes; - CTL_M2_M4_GET("stats.arenas.0.extents.0.ndirty", i, j, - &ndirty, size_t); - CTL_M2_M4_GET("stats.arenas.0.extents.0.nmuzzy", i, j, - &nmuzzy, size_t); - CTL_M2_M4_GET("stats.arenas.0.extents.0.nretained", i, j, - &nretained, size_t); - CTL_M2_M4_GET("stats.arenas.0.extents.0.dirty_bytes", i, j, - &dirty_bytes, size_t); - CTL_M2_M4_GET("stats.arenas.0.extents.0.muzzy_bytes", i, j, - &muzzy_bytes, size_t); - CTL_M2_M4_GET("stats.arenas.0.extents.0.retained_bytes", i, j, - &retained_bytes, size_t); - total = ndirty + nmuzzy + nretained; - total_bytes = dirty_bytes + muzzy_bytes + retained_bytes; - - in_gap_prev = in_gap; - in_gap = (total == 0); - - if (in_gap_prev && !in_gap) { - emitter_table_printf(emitter, - " ---\n"); - } - - emitter_json_object_begin(emitter); - emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty); - emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &nmuzzy); - emitter_json_kv(emitter, "nretained", emitter_type_size, - &nretained); - - emitter_json_kv(emitter, "dirty_bytes", emitter_type_size, - &dirty_bytes); - emitter_json_kv(emitter, "muzzy_bytes", emitter_type_size, - &muzzy_bytes); - emitter_json_kv(emitter, "retained_bytes", emitter_type_size, - &retained_bytes); - emitter_json_object_end(emitter); - - col_size.size_val = sz_pind2sz(j); - col_ind.size_val = j; - col_ndirty.size_val = ndirty; - col_dirty.size_val = dirty_bytes; - col_nmuzzy.size_val = nmuzzy; - col_muzzy.size_val = muzzy_bytes; - col_nretained.size_val = nretained; - col_retained.size_val = retained_bytes; - col_ntotal.size_val = total; - col_total.size_val = total_bytes; - - if (!in_gap) { - emitter_table_row(emitter, &row); - } - } - emitter_json_array_end(emitter); /* Close "extents". */ - if (in_gap) { - emitter_table_printf(emitter, " ---\n"); - } -} - -static void -stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptime) { +stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind) { emitter_row_t row; emitter_col_t col_name; emitter_col_t col64[mutex_prof_num_uint64_t_counters]; @@ -642,38 +504,36 @@ stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptim emitter_row_init(&row); mutex_stats_init_cols(&row, "", &col_name, col64, col32); - emitter_json_object_kv_begin(emitter, "mutexes"); + emitter_json_dict_begin(emitter, "mutexes"); emitter_table_row(emitter, &row); for (mutex_prof_arena_ind_t i = 0; i < mutex_prof_num_arena_mutexes; i++) { const char *name = arena_mutex_names[i]; - emitter_json_object_kv_begin(emitter, name); + emitter_json_dict_begin(emitter, name); mutex_stats_read_arena(arena_ind, i, name, &col_name, col64, - col32, uptime); + col32); mutex_stats_emit(emitter, &row, col64, col32); - emitter_json_object_end(emitter); /* Close the mutex dict. */ + emitter_json_dict_end(emitter); /* Close the mutex dict. */ } - emitter_json_object_end(emitter); /* End "mutexes". */ + emitter_json_dict_end(emitter); /* End "mutexes". */ } static void stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, - bool mutex, bool extents) { + bool mutex) { unsigned nthreads; const char *dss; ssize_t dirty_decay_ms, muzzy_decay_ms; size_t page, pactive, pdirty, pmuzzy, mapped, retained; - size_t base, internal, resident, metadata_thp, extent_avail; + size_t base, internal, resident, metadata_thp; uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; size_t small_allocated; - uint64_t small_nmalloc, small_ndalloc, small_nrequests, small_nfills, - small_nflushes; + uint64_t small_nmalloc, small_ndalloc, small_nrequests; size_t large_allocated; - uint64_t large_nmalloc, large_ndalloc, large_nrequests, large_nfills, - large_nflushes; - size_t tcache_bytes, abandoned_vm; + uint64_t large_nmalloc, large_ndalloc, large_nrequests; + size_t tcache_bytes; uint64_t uptime; CTL_GET("arenas.page", &page, size_t); @@ -734,74 +594,98 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, &muzzy_purged); /* Table-style emission. */ - COL(decay_row, decay_type, right, 9, title); - col_decay_type.str_val = "decaying:"; + emitter_col_t decay_type; + emitter_col_init(&decay_type, &decay_row); + decay_type.justify = emitter_justify_right; + decay_type.width = 9; + decay_type.type = emitter_type_title; + decay_type.str_val = "decaying:"; - COL(decay_row, decay_time, right, 6, title); - col_decay_time.str_val = "time"; + emitter_col_t decay_time; + emitter_col_init(&decay_time, &decay_row); + decay_time.justify = emitter_justify_right; + decay_time.width = 6; + decay_time.type = emitter_type_title; + decay_time.str_val = "time"; - COL(decay_row, decay_npages, right, 13, title); - col_decay_npages.str_val = "npages"; + emitter_col_t decay_npages; + emitter_col_init(&decay_npages, &decay_row); + decay_npages.justify = emitter_justify_right; + decay_npages.width = 13; + decay_npages.type = emitter_type_title; + decay_npages.str_val = "npages"; - COL(decay_row, decay_sweeps, right, 13, title); - col_decay_sweeps.str_val = "sweeps"; + emitter_col_t decay_sweeps; + emitter_col_init(&decay_sweeps, &decay_row); + decay_sweeps.justify = emitter_justify_right; + decay_sweeps.width = 13; + decay_sweeps.type = emitter_type_title; + decay_sweeps.str_val = "sweeps"; - COL(decay_row, decay_madvises, right, 13, title); - col_decay_madvises.str_val = "madvises"; + emitter_col_t decay_madvises; + emitter_col_init(&decay_madvises, &decay_row); + decay_madvises.justify = emitter_justify_right; + decay_madvises.width = 13; + decay_madvises.type = emitter_type_title; + decay_madvises.str_val = "madvises"; - COL(decay_row, decay_purged, right, 13, title); - col_decay_purged.str_val = "purged"; + emitter_col_t decay_purged; + emitter_col_init(&decay_purged, &decay_row); + decay_purged.justify = emitter_justify_right; + decay_purged.width = 13; + decay_purged.type = emitter_type_title; + decay_purged.str_val = "purged"; /* Title row. */ emitter_table_row(emitter, &decay_row); /* Dirty row. */ - col_decay_type.str_val = "dirty:"; + decay_type.str_val = "dirty:"; if (dirty_decay_ms >= 0) { - col_decay_time.type = emitter_type_ssize; - col_decay_time.ssize_val = dirty_decay_ms; + decay_time.type = emitter_type_ssize; + decay_time.ssize_val = dirty_decay_ms; } else { - col_decay_time.type = emitter_type_title; - col_decay_time.str_val = "N/A"; + decay_time.type = emitter_type_title; + decay_time.str_val = "N/A"; } - col_decay_npages.type = emitter_type_size; - col_decay_npages.size_val = pdirty; + decay_npages.type = emitter_type_size; + decay_npages.size_val = pdirty; - col_decay_sweeps.type = emitter_type_uint64; - col_decay_sweeps.uint64_val = dirty_npurge; + decay_sweeps.type = emitter_type_uint64; + decay_sweeps.uint64_val = dirty_npurge; - col_decay_madvises.type = emitter_type_uint64; - col_decay_madvises.uint64_val = dirty_nmadvise; + decay_madvises.type = emitter_type_uint64; + decay_madvises.uint64_val = dirty_nmadvise; - col_decay_purged.type = emitter_type_uint64; - col_decay_purged.uint64_val = dirty_purged; + decay_purged.type = emitter_type_uint64; + decay_purged.uint64_val = dirty_purged; emitter_table_row(emitter, &decay_row); /* Muzzy row. */ - col_decay_type.str_val = "muzzy:"; + decay_type.str_val = "muzzy:"; if (muzzy_decay_ms >= 0) { - col_decay_time.type = emitter_type_ssize; - col_decay_time.ssize_val = muzzy_decay_ms; + decay_time.type = emitter_type_ssize; + decay_time.ssize_val = muzzy_decay_ms; } else { - col_decay_time.type = emitter_type_title; - col_decay_time.str_val = "N/A"; + decay_time.type = emitter_type_title; + decay_time.str_val = "N/A"; } - col_decay_npages.type = emitter_type_size; - col_decay_npages.size_val = pmuzzy; + decay_npages.type = emitter_type_size; + decay_npages.size_val = pmuzzy; - col_decay_sweeps.type = emitter_type_uint64; - col_decay_sweeps.uint64_val = muzzy_npurge; + decay_sweeps.type = emitter_type_uint64; + decay_sweeps.uint64_val = muzzy_npurge; - col_decay_madvises.type = emitter_type_uint64; - col_decay_madvises.uint64_val = muzzy_nmadvise; + decay_madvises.type = emitter_type_uint64; + decay_madvises.uint64_val = muzzy_nmadvise; - col_decay_purged.type = emitter_type_uint64; - col_decay_purged.uint64_val = muzzy_purged; + decay_purged.type = emitter_type_uint64; + decay_purged.uint64_val = muzzy_purged; emitter_table_row(emitter, &decay_row); @@ -809,119 +693,81 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, emitter_row_t alloc_count_row; emitter_row_init(&alloc_count_row); - COL(alloc_count_row, count_title, left, 21, title); - col_count_title.str_val = ""; + emitter_col_t alloc_count_title; + emitter_col_init(&alloc_count_title, &alloc_count_row); + alloc_count_title.justify = emitter_justify_left; + alloc_count_title.width = 25; + alloc_count_title.type = emitter_type_title; + alloc_count_title.str_val = ""; - COL(alloc_count_row, count_allocated, right, 16, title); - col_count_allocated.str_val = "allocated"; + emitter_col_t alloc_count_allocated; + emitter_col_init(&alloc_count_allocated, &alloc_count_row); + alloc_count_allocated.justify = emitter_justify_right; + alloc_count_allocated.width = 12; + alloc_count_allocated.type = emitter_type_title; + alloc_count_allocated.str_val = "allocated"; - COL(alloc_count_row, count_nmalloc, right, 16, title); - col_count_nmalloc.str_val = "nmalloc"; - COL(alloc_count_row, count_nmalloc_ps, right, 8, title); - col_count_nmalloc_ps.str_val = "(#/sec)"; + emitter_col_t alloc_count_nmalloc; + emitter_col_init(&alloc_count_nmalloc, &alloc_count_row); + alloc_count_nmalloc.justify = emitter_justify_right; + alloc_count_nmalloc.width = 12; + alloc_count_nmalloc.type = emitter_type_title; + alloc_count_nmalloc.str_val = "nmalloc"; - COL(alloc_count_row, count_ndalloc, right, 16, title); - col_count_ndalloc.str_val = "ndalloc"; - COL(alloc_count_row, count_ndalloc_ps, right, 8, title); - col_count_ndalloc_ps.str_val = "(#/sec)"; + emitter_col_t alloc_count_ndalloc; + emitter_col_init(&alloc_count_ndalloc, &alloc_count_row); + alloc_count_ndalloc.justify = emitter_justify_right; + alloc_count_ndalloc.width = 12; + alloc_count_ndalloc.type = emitter_type_title; + alloc_count_ndalloc.str_val = "ndalloc"; - COL(alloc_count_row, count_nrequests, right, 16, title); - col_count_nrequests.str_val = "nrequests"; - COL(alloc_count_row, count_nrequests_ps, right, 10, title); - col_count_nrequests_ps.str_val = "(#/sec)"; - - COL(alloc_count_row, count_nfills, right, 16, title); - col_count_nfills.str_val = "nfill"; - COL(alloc_count_row, count_nfills_ps, right, 10, title); - col_count_nfills_ps.str_val = "(#/sec)"; - - COL(alloc_count_row, count_nflushes, right, 16, title); - col_count_nflushes.str_val = "nflush"; - COL(alloc_count_row, count_nflushes_ps, right, 10, title); - col_count_nflushes_ps.str_val = "(#/sec)"; + emitter_col_t alloc_count_nrequests; + emitter_col_init(&alloc_count_nrequests, &alloc_count_row); + alloc_count_nrequests.justify = emitter_justify_right; + alloc_count_nrequests.width = 12; + alloc_count_nrequests.type = emitter_type_title; + alloc_count_nrequests.str_val = "nrequests"; emitter_table_row(emitter, &alloc_count_row); - col_count_nmalloc_ps.type = emitter_type_uint64; - col_count_ndalloc_ps.type = emitter_type_uint64; - col_count_nrequests_ps.type = emitter_type_uint64; - col_count_nfills_ps.type = emitter_type_uint64; - col_count_nflushes_ps.type = emitter_type_uint64; - #define GET_AND_EMIT_ALLOC_STAT(small_or_large, name, valtype) \ CTL_M2_GET("stats.arenas.0." #small_or_large "." #name, i, \ &small_or_large##_##name, valtype##_t); \ emitter_json_kv(emitter, #name, emitter_type_##valtype, \ &small_or_large##_##name); \ - col_count_##name.type = emitter_type_##valtype; \ - col_count_##name.valtype##_val = small_or_large##_##name; + alloc_count_##name.type = emitter_type_##valtype; \ + alloc_count_##name.valtype##_val = small_or_large##_##name; - emitter_json_object_kv_begin(emitter, "small"); - col_count_title.str_val = "small:"; + emitter_json_dict_begin(emitter, "small"); + alloc_count_title.str_val = "small:"; GET_AND_EMIT_ALLOC_STAT(small, allocated, size) GET_AND_EMIT_ALLOC_STAT(small, nmalloc, uint64) - col_count_nmalloc_ps.uint64_val = - rate_per_second(col_count_nmalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(small, ndalloc, uint64) - col_count_ndalloc_ps.uint64_val = - rate_per_second(col_count_ndalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(small, nrequests, uint64) - col_count_nrequests_ps.uint64_val = - rate_per_second(col_count_nrequests.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(small, nfills, uint64) - col_count_nfills_ps.uint64_val = - rate_per_second(col_count_nfills.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(small, nflushes, uint64) - col_count_nflushes_ps.uint64_val = - rate_per_second(col_count_nflushes.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); - emitter_json_object_end(emitter); /* Close "small". */ + emitter_json_dict_end(emitter); /* Close "small". */ - emitter_json_object_kv_begin(emitter, "large"); - col_count_title.str_val = "large:"; + emitter_json_dict_begin(emitter, "large"); + alloc_count_title.str_val = "large:"; GET_AND_EMIT_ALLOC_STAT(large, allocated, size) GET_AND_EMIT_ALLOC_STAT(large, nmalloc, uint64) - col_count_nmalloc_ps.uint64_val = - rate_per_second(col_count_nmalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(large, ndalloc, uint64) - col_count_ndalloc_ps.uint64_val = - rate_per_second(col_count_ndalloc.uint64_val, uptime); GET_AND_EMIT_ALLOC_STAT(large, nrequests, uint64) - col_count_nrequests_ps.uint64_val = - rate_per_second(col_count_nrequests.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(large, nfills, uint64) - col_count_nfills_ps.uint64_val = - rate_per_second(col_count_nfills.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(large, nflushes, uint64) - col_count_nflushes_ps.uint64_val = - rate_per_second(col_count_nflushes.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); - emitter_json_object_end(emitter); /* Close "large". */ + emitter_json_dict_end(emitter); /* Close "large". */ #undef GET_AND_EMIT_ALLOC_STAT /* Aggregated small + large stats are emitter only in table mode. */ - col_count_title.str_val = "total:"; - col_count_allocated.size_val = small_allocated + large_allocated; - col_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc; - col_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc; - col_count_nrequests.uint64_val = small_nrequests + large_nrequests; - col_count_nfills.uint64_val = small_nfills + large_nfills; - col_count_nflushes.uint64_val = small_nflushes + large_nflushes; - col_count_nmalloc_ps.uint64_val = - rate_per_second(col_count_nmalloc.uint64_val, uptime); - col_count_ndalloc_ps.uint64_val = - rate_per_second(col_count_ndalloc.uint64_val, uptime); - col_count_nrequests_ps.uint64_val = - rate_per_second(col_count_nrequests.uint64_val, uptime); - col_count_nfills_ps.uint64_val = - rate_per_second(col_count_nfills.uint64_val, uptime); - col_count_nflushes_ps.uint64_val = - rate_per_second(col_count_nflushes.uint64_val, uptime); + alloc_count_title.str_val = "total:"; + alloc_count_allocated.size_val = small_allocated + large_allocated; + alloc_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc; + alloc_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc; + alloc_count_nrequests.uint64_val = small_nrequests + large_nrequests; emitter_table_row(emitter, &alloc_count_row); emitter_row_t mem_count_row; @@ -930,14 +776,14 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, emitter_col_t mem_count_title; emitter_col_init(&mem_count_title, &mem_count_row); mem_count_title.justify = emitter_justify_left; - mem_count_title.width = 21; + mem_count_title.width = 25; mem_count_title.type = emitter_type_title; mem_count_title.str_val = ""; emitter_col_t mem_count_val; emitter_col_init(&mem_count_val, &mem_count_row); mem_count_val.justify = emitter_justify_right; - mem_count_val.width = 16; + mem_count_val.width = 12; mem_count_val.type = emitter_type_title; mem_count_val.str_val = ""; @@ -963,21 +809,16 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, GET_AND_EMIT_MEM_STAT(metadata_thp) GET_AND_EMIT_MEM_STAT(tcache_bytes) GET_AND_EMIT_MEM_STAT(resident) - GET_AND_EMIT_MEM_STAT(abandoned_vm) - GET_AND_EMIT_MEM_STAT(extent_avail) #undef GET_AND_EMIT_MEM_STAT if (mutex) { - stats_arena_mutexes_print(emitter, i, uptime); + stats_arena_mutexes_print(emitter, i); } if (bins) { - stats_arena_bins_print(emitter, mutex, i, uptime); + stats_arena_bins_print(emitter, mutex, i); } if (large) { - stats_arena_lextents_print(emitter, i, uptime); - } - if (extents) { - stats_arena_extents_print(emitter, i); + stats_arena_lextents_print(emitter, i); } } @@ -1016,7 +857,6 @@ stats_general_print(emitter_t *emitter) { emitter_kv(emitter, "malloc_conf", "config.malloc_conf", emitter_type_string, &config_malloc_conf); - CONFIG_WRITE_BOOL(opt_safety_checks); CONFIG_WRITE_BOOL(prof); CONFIG_WRITE_BOOL(prof_libgcc); CONFIG_WRITE_BOOL(prof_libunwind); @@ -1051,8 +891,6 @@ stats_general_print(emitter_t *emitter) { #define OPT_WRITE_UNSIGNED(name) \ OPT_WRITE(name, uv, usz, emitter_type_unsigned) -#define OPT_WRITE_SIZE_T(name) \ - OPT_WRITE(name, sv, ssz, emitter_type_size) #define OPT_WRITE_SSIZE_T(name) \ OPT_WRITE(name, ssv, sssz, emitter_type_ssize) #define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \ @@ -1066,17 +904,15 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_BOOL("abort") OPT_WRITE_BOOL("abort_conf") - OPT_WRITE_BOOL("confirm_conf") OPT_WRITE_BOOL("retain") OPT_WRITE_CHAR_P("dss") OPT_WRITE_UNSIGNED("narenas") OPT_WRITE_CHAR_P("percpu_arena") - OPT_WRITE_SIZE_T("oversize_threshold") OPT_WRITE_CHAR_P("metadata_thp") OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread") OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms") OPT_WRITE_SSIZE_T_MUTABLE("muzzy_decay_ms", "arenas.muzzy_decay_ms") - OPT_WRITE_SIZE_T("lg_extent_max_active_fit") + OPT_WRITE_UNSIGNED("lg_extent_max_active_fit") OPT_WRITE_CHAR_P("junk") OPT_WRITE_BOOL("zero") OPT_WRITE_BOOL("utrace") @@ -1141,7 +977,7 @@ stats_general_print(emitter_t *emitter) { * The json output sticks arena info into an "arenas" dict; the table * output puts them at the top-level. */ - emitter_json_object_kv_begin(emitter, "arenas"); + emitter_json_dict_begin(emitter, "arenas"); CTL_GET("arenas.narenas", &uv, unsigned); emitter_kv(emitter, "narenas", "Arenas", emitter_type_unsigned, &uv); @@ -1182,9 +1018,9 @@ stats_general_print(emitter_t *emitter) { * (not just omit the printing). */ if (emitter->output == emitter_output_json) { - emitter_json_array_kv_begin(emitter, "bin"); + emitter_json_arr_begin(emitter, "bin"); for (unsigned i = 0; i < nbins; i++) { - emitter_json_object_begin(emitter); + emitter_json_arr_obj_begin(emitter); CTL_M2_GET("arenas.bin.0.size", i, &sv, size_t); emitter_json_kv(emitter, "size", emitter_type_size, @@ -1198,13 +1034,9 @@ stats_general_print(emitter_t *emitter) { emitter_json_kv(emitter, "slab_size", emitter_type_size, &sv); - CTL_M2_GET("arenas.bin.0.nshards", i, &u32v, uint32_t); - emitter_json_kv(emitter, "nshards", emitter_type_uint32, - &u32v); - - emitter_json_object_end(emitter); + emitter_json_arr_obj_end(emitter); } - emitter_json_array_end(emitter); /* Close "bin". */ + emitter_json_arr_end(emitter); /* Close "bin". */ } unsigned nlextents; @@ -1213,25 +1045,25 @@ stats_general_print(emitter_t *emitter) { emitter_type_unsigned, &nlextents); if (emitter->output == emitter_output_json) { - emitter_json_array_kv_begin(emitter, "lextent"); + emitter_json_arr_begin(emitter, "lextent"); for (unsigned i = 0; i < nlextents; i++) { - emitter_json_object_begin(emitter); + emitter_json_arr_obj_begin(emitter); CTL_M2_GET("arenas.lextent.0.size", i, &sv, size_t); emitter_json_kv(emitter, "size", emitter_type_size, &sv); - emitter_json_object_end(emitter); + emitter_json_arr_obj_end(emitter); } - emitter_json_array_end(emitter); /* Close "lextent". */ + emitter_json_arr_end(emitter); /* Close "lextent". */ } - emitter_json_object_end(emitter); /* Close "arenas" */ + emitter_json_dict_end(emitter); /* Close "arenas" */ } static void stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, - bool unmerged, bool bins, bool large, bool mutex, bool extents) { + bool unmerged, bool bins, bool large, bool mutex) { /* * These should be deleted. We keep them around for a while, to aid in * the transition to the emitter code. @@ -1263,7 +1095,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, } /* Generic global stats. */ - emitter_json_object_kv_begin(emitter, "stats"); + emitter_json_dict_begin(emitter, "stats"); emitter_json_kv(emitter, "allocated", emitter_type_size, &allocated); emitter_json_kv(emitter, "active", emitter_type_size, &active); emitter_json_kv(emitter, "metadata", emitter_type_size, &metadata); @@ -1279,14 +1111,14 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, resident, mapped, retained); /* Background thread stats. */ - emitter_json_object_kv_begin(emitter, "background_thread"); + emitter_json_dict_begin(emitter, "background_thread"); emitter_json_kv(emitter, "num_threads", emitter_type_size, &num_background_threads); emitter_json_kv(emitter, "num_runs", emitter_type_uint64, &background_thread_num_runs); emitter_json_kv(emitter, "run_interval", emitter_type_uint64, &background_thread_run_interval); - emitter_json_object_end(emitter); /* Close "background_thread". */ + emitter_json_dict_end(emitter); /* Close "background_thread". */ emitter_table_printf(emitter, "Background threads: %zu, " "num_runs: %"FMTu64", run_interval: %"FMTu64" ns\n", @@ -1298,33 +1130,30 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, emitter_col_t name; emitter_col_t col64[mutex_prof_num_uint64_t_counters]; emitter_col_t col32[mutex_prof_num_uint32_t_counters]; - uint64_t uptime; emitter_row_init(&row); mutex_stats_init_cols(&row, "", &name, col64, col32); emitter_table_row(emitter, &row); - emitter_json_object_kv_begin(emitter, "mutexes"); - - CTL_M2_GET("stats.arenas.0.uptime", 0, &uptime, uint64_t); + emitter_json_dict_begin(emitter, "mutexes"); for (int i = 0; i < mutex_prof_num_global_mutexes; i++) { mutex_stats_read_global(global_mutex_names[i], &name, - col64, col32, uptime); - emitter_json_object_kv_begin(emitter, global_mutex_names[i]); + col64, col32); + emitter_json_dict_begin(emitter, global_mutex_names[i]); mutex_stats_emit(emitter, &row, col64, col32); - emitter_json_object_end(emitter); + emitter_json_dict_end(emitter); } - emitter_json_object_end(emitter); /* Close "mutexes". */ + emitter_json_dict_end(emitter); /* Close "mutexes". */ } - emitter_json_object_end(emitter); /* Close "stats". */ + emitter_json_dict_end(emitter); /* Close "stats". */ if (merged || destroyed || unmerged) { unsigned narenas; - emitter_json_object_kv_begin(emitter, "stats.arenas"); + emitter_json_dict_begin(emitter, "stats.arenas"); CTL_GET("arenas.narenas", &narenas, unsigned); size_t mib[3]; @@ -1353,10 +1182,10 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, if (merged && (ninitialized > 1 || !unmerged)) { /* Print merged arena stats. */ emitter_table_printf(emitter, "Merged arenas stats:\n"); - emitter_json_object_kv_begin(emitter, "merged"); + emitter_json_dict_begin(emitter, "merged"); stats_arena_print(emitter, MALLCTL_ARENAS_ALL, bins, - large, mutex, extents); - emitter_json_object_end(emitter); /* Close "merged". */ + large, mutex); + emitter_json_dict_end(emitter); /* Close "merged". */ } /* Destroyed stats. */ @@ -1364,10 +1193,10 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, /* Print destroyed arena stats. */ emitter_table_printf(emitter, "Destroyed arenas stats:\n"); - emitter_json_object_kv_begin(emitter, "destroyed"); + emitter_json_dict_begin(emitter, "destroyed"); stats_arena_print(emitter, MALLCTL_ARENAS_DESTROYED, - bins, large, mutex, extents); - emitter_json_object_end(emitter); /* Close "destroyed". */ + bins, large, mutex); + emitter_json_dict_end(emitter); /* Close "destroyed". */ } /* Unmerged stats. */ @@ -1377,18 +1206,18 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, char arena_ind_str[20]; malloc_snprintf(arena_ind_str, sizeof(arena_ind_str), "%u", i); - emitter_json_object_kv_begin(emitter, + emitter_json_dict_begin(emitter, arena_ind_str); emitter_table_printf(emitter, "arenas[%s]:\n", arena_ind_str); stats_arena_print(emitter, i, bins, - large, mutex, extents); + large, mutex); /* Close "". */ - emitter_json_object_end(emitter); + emitter_json_dict_end(emitter); } } } - emitter_json_object_end(emitter); /* Close "stats.arenas". */ + emitter_json_dict_end(emitter); /* Close "stats.arenas". */ } } @@ -1441,17 +1270,17 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, cbopaque); emitter_begin(&emitter); emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n"); - emitter_json_object_kv_begin(&emitter, "jemalloc"); + emitter_json_dict_begin(&emitter, "jemalloc"); if (general) { stats_general_print(&emitter); } if (config_stats) { stats_print_helper(&emitter, merged, destroyed, unmerged, - bins, large, mutex, extents); + bins, large, mutex); } - emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */ + emitter_json_dict_end(&emitter); /* Closes the "jemalloc" dict. */ emitter_table_printf(&emitter, "--- End jemalloc statistics ---\n"); emitter_end(&emitter); } diff --git a/deps/jemalloc/src/sz.c b/deps/jemalloc/src/sz.c index 8633fb0..9de77e4 100644 --- a/deps/jemalloc/src/sz.c +++ b/deps/jemalloc/src/sz.c @@ -2,63 +2,106 @@ #include "jemalloc/internal/sz.h" JEMALLOC_ALIGNED(CACHELINE) -size_t sz_pind2sz_tab[SC_NPSIZES+1]; - -static void -sz_boot_pind2sz_tab(const sc_data_t *sc_data) { - int pind = 0; - for (unsigned i = 0; i < SC_NSIZES; i++) { - const sc_t *sc = &sc_data->sc[i]; - if (sc->psz) { - sz_pind2sz_tab[pind] = (ZU(1) << sc->lg_base) - + (ZU(sc->ndelta) << sc->lg_delta); - pind++; - } - } - for (int i = pind; i <= (int)SC_NPSIZES; i++) { - sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE; - } -} +const size_t sz_pind2sz_tab[NPSIZES+1] = { +#define PSZ_yes(lg_grp, ndelta, lg_delta) \ + (((ZU(1)<sc[i]; - sz_index2size_tab[i] = (ZU(1) << sc->lg_base) - + (ZU(sc->ndelta) << (sc->lg_delta)); - } -} - -/* - * To keep this table small, we divide sizes by the tiny min size, which gives - * the smallest interval for which the result can change. - */ JEMALLOC_ALIGNED(CACHELINE) -uint8_t sz_size2index_tab[(SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1]; - -static void -sz_boot_size2index_tab(const sc_data_t *sc_data) { - size_t dst_max = (SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1; - size_t dst_ind = 0; - for (unsigned sc_ind = 0; sc_ind < SC_NSIZES && dst_ind < dst_max; - sc_ind++) { - const sc_t *sc = &sc_data->sc[sc_ind]; - size_t sz = (ZU(1) << sc->lg_base) - + (ZU(sc->ndelta) << sc->lg_delta); - size_t max_ind = ((sz + (ZU(1) << SC_LG_TINY_MIN) - 1) - >> SC_LG_TINY_MIN); - for (; dst_ind <= max_ind && dst_ind < dst_max; dst_ind++) { - sz_size2index_tab[dst_ind] = sc_ind; - } - } -} - -void -sz_boot(const sc_data_t *sc_data) { - sz_boot_pind2sz_tab(sc_data); - sz_boot_index2size_tab(sc_data); - sz_boot_size2index_tab(sc_data); -} +const uint8_t sz_size2index_tab[] = { +#if LG_TINY_MIN == 0 +/* The div module doesn't support division by 1. */ +#error "Unsupported LG_TINY_MIN" +#define S2B_0(i) i, +#elif LG_TINY_MIN == 1 +#warning "Dangerous LG_TINY_MIN" +#define S2B_1(i) i, +#elif LG_TINY_MIN == 2 +#warning "Dangerous LG_TINY_MIN" +#define S2B_2(i) i, +#elif LG_TINY_MIN == 3 +#define S2B_3(i) i, +#elif LG_TINY_MIN == 4 +#define S2B_4(i) i, +#elif LG_TINY_MIN == 5 +#define S2B_5(i) i, +#elif LG_TINY_MIN == 6 +#define S2B_6(i) i, +#elif LG_TINY_MIN == 7 +#define S2B_7(i) i, +#elif LG_TINY_MIN == 8 +#define S2B_8(i) i, +#elif LG_TINY_MIN == 9 +#define S2B_9(i) i, +#elif LG_TINY_MIN == 10 +#define S2B_10(i) i, +#elif LG_TINY_MIN == 11 +#define S2B_11(i) i, +#else +#error "Unsupported LG_TINY_MIN" +#endif +#if LG_TINY_MIN < 1 +#define S2B_1(i) S2B_0(i) S2B_0(i) +#endif +#if LG_TINY_MIN < 2 +#define S2B_2(i) S2B_1(i) S2B_1(i) +#endif +#if LG_TINY_MIN < 3 +#define S2B_3(i) S2B_2(i) S2B_2(i) +#endif +#if LG_TINY_MIN < 4 +#define S2B_4(i) S2B_3(i) S2B_3(i) +#endif +#if LG_TINY_MIN < 5 +#define S2B_5(i) S2B_4(i) S2B_4(i) +#endif +#if LG_TINY_MIN < 6 +#define S2B_6(i) S2B_5(i) S2B_5(i) +#endif +#if LG_TINY_MIN < 7 +#define S2B_7(i) S2B_6(i) S2B_6(i) +#endif +#if LG_TINY_MIN < 8 +#define S2B_8(i) S2B_7(i) S2B_7(i) +#endif +#if LG_TINY_MIN < 9 +#define S2B_9(i) S2B_8(i) S2B_8(i) +#endif +#if LG_TINY_MIN < 10 +#define S2B_10(i) S2B_9(i) S2B_9(i) +#endif +#if LG_TINY_MIN < 11 +#define S2B_11(i) S2B_10(i) S2B_10(i) +#endif +#define S2B_no(i) +#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \ + S2B_##lg_delta_lookup(index) + SIZE_CLASSES +#undef S2B_3 +#undef S2B_4 +#undef S2B_5 +#undef S2B_6 +#undef S2B_7 +#undef S2B_8 +#undef S2B_9 +#undef S2B_10 +#undef S2B_11 +#undef S2B_no +#undef SC +}; diff --git a/deps/jemalloc/src/tcache.c b/deps/jemalloc/src/tcache.c index 50099a9..a769a6b 100644 --- a/deps/jemalloc/src/tcache.c +++ b/deps/jemalloc/src/tcache.c @@ -4,8 +4,7 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/safety_check.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/size_classes.h" /******************************************************************************/ /* Data. */ @@ -42,7 +41,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { szind_t binind = tcache->next_gc_bin; cache_bin_t *tbin; - if (binind < SC_NBINS) { + if (binind < NBINS) { tbin = tcache_small_bin_get(tcache, binind); } else { tbin = tcache_large_bin_get(tcache, binind); @@ -51,7 +50,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { /* * Flush (ceiling) 3/4 of the objects below the low water mark. */ - if (binind < SC_NBINS) { + if (binind < NBINS) { tcache_bin_flush_small(tsd, tcache, tbin, binind, tbin->ncached - tbin->low_water + (tbin->low_water >> 2)); @@ -73,7 +72,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { * Increase fill count by 2X for small bins. Make sure * lg_fill_div stays greater than 0. */ - if (binind < SC_NBINS && tcache->lg_fill_div[binind] > 1) { + if (binind < NBINS && tcache->lg_fill_div[binind] > 1) { tcache->lg_fill_div[binind]--; } } @@ -101,67 +100,28 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, return ret; } -/* Enabled with --enable-extra-size-check. */ -static void -tbin_extents_lookup_size_check(tsdn_t *tsdn, cache_bin_t *tbin, szind_t binind, - size_t nflush, extent_t **extents){ - rtree_ctx_t rtree_ctx_fallback; - rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - - /* - * Verify that the items in the tcache all have the correct size; this - * is useful for catching sized deallocation bugs, also to fail early - * instead of corrupting metadata. Since this can be turned on for opt - * builds, avoid the branch in the loop. - */ - szind_t szind; - size_t sz_sum = binind * nflush; - for (unsigned i = 0 ; i < nflush; i++) { - rtree_extent_szind_read(tsdn, &extents_rtree, - rtree_ctx, (uintptr_t)*(tbin->avail - 1 - i), true, - &extents[i], &szind); - sz_sum -= szind; - } - if (sz_sum != 0) { - safety_check_fail(": size mismatch in thread cache " - "detected, likely caused by sized deallocation bugs by " - "application. Abort.\n"); - abort(); - } -} - void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, szind_t binind, unsigned rem) { bool merged_stats = false; - assert(binind < SC_NBINS); + assert(binind < NBINS); assert((cache_bin_sz_t)rem <= tbin->ncached); arena_t *arena = tcache->arena; assert(arena != NULL); unsigned nflush = tbin->ncached - rem; VARIABLE_ARRAY(extent_t *, item_extent, nflush); - /* Look up extent once per item. */ - if (config_opt_safety_checks) { - tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, - nflush, item_extent); - } else { - for (unsigned i = 0 ; i < nflush; i++) { - item_extent[i] = iealloc(tsd_tsdn(tsd), - *(tbin->avail - 1 - i)); - } + for (unsigned i = 0 ; i < nflush; i++) { + item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); } + while (nflush > 0) { /* Lock the arena bin associated with the first object. */ extent_t *extent = item_extent[0]; - unsigned bin_arena_ind = extent_arena_ind_get(extent); - arena_t *bin_arena = arena_get(tsd_tsdn(tsd), bin_arena_ind, - false); - unsigned binshard = extent_binshard_get(extent); - assert(binshard < bin_infos[binind].n_shards); - bin_t *bin = &bin_arena->bins[binind].bin_shards[binshard]; + arena_t *bin_arena = extent_arena_get(extent); + bin_t *bin = &bin_arena->bins[binind]; if (config_prof && bin_arena == arena) { if (arena_prof_accum(tsd_tsdn(tsd), arena, @@ -172,7 +132,8 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, } malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - if (config_stats && bin_arena == arena && !merged_stats) { + if (config_stats && bin_arena == arena) { + assert(!merged_stats); merged_stats = true; bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; @@ -184,10 +145,9 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, extent = item_extent[i]; assert(ptr != NULL && extent != NULL); - if (extent_arena_ind_get(extent) == bin_arena_ind - && extent_binshard_get(extent) == binshard) { + if (extent_arena_get(extent) == bin_arena) { arena_dalloc_bin_junked_locked(tsd_tsdn(tsd), - bin_arena, bin, binind, extent, ptr); + bin_arena, extent, ptr); } else { /* * This object was allocated via a different @@ -209,9 +169,8 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. */ - unsigned binshard; - bin_t *bin = arena_bin_choose_lock(tsd_tsdn(tsd), arena, binind, - &binshard); + bin_t *bin = &arena->bins[binind]; + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; tbin->tstats.nrequests = 0; @@ -234,63 +193,50 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, assert(binind < nhbins); assert((cache_bin_sz_t)rem <= tbin->ncached); - arena_t *tcache_arena = tcache->arena; - assert(tcache_arena != NULL); + arena_t *arena = tcache->arena; + assert(arena != NULL); unsigned nflush = tbin->ncached - rem; VARIABLE_ARRAY(extent_t *, item_extent, nflush); - -#ifndef JEMALLOC_EXTRA_SIZE_CHECK /* Look up extent once per item. */ for (unsigned i = 0 ; i < nflush; i++) { item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); } -#else - tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, nflush, - item_extent); -#endif + while (nflush > 0) { /* Lock the arena associated with the first object. */ extent_t *extent = item_extent[0]; - unsigned locked_arena_ind = extent_arena_ind_get(extent); - arena_t *locked_arena = arena_get(tsd_tsdn(tsd), - locked_arena_ind, false); - bool idump; + arena_t *locked_arena = extent_arena_get(extent); + UNUSED bool idump; if (config_prof) { idump = false; } - bool lock_large = !arena_is_auto(locked_arena); - if (lock_large) { - malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx); - } + malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx); for (unsigned i = 0; i < nflush; i++) { void *ptr = *(tbin->avail - 1 - i); assert(ptr != NULL); extent = item_extent[i]; - if (extent_arena_ind_get(extent) == locked_arena_ind) { + if (extent_arena_get(extent) == locked_arena) { large_dalloc_prep_junked_locked(tsd_tsdn(tsd), extent); } } - if ((config_prof || config_stats) && - (locked_arena == tcache_arena)) { + if ((config_prof || config_stats) && locked_arena == arena) { if (config_prof) { - idump = arena_prof_accum(tsd_tsdn(tsd), - tcache_arena, tcache->prof_accumbytes); + idump = arena_prof_accum(tsd_tsdn(tsd), arena, + tcache->prof_accumbytes); tcache->prof_accumbytes = 0; } if (config_stats) { merged_stats = true; - arena_stats_large_flush_nrequests_add( - tsd_tsdn(tsd), &tcache_arena->stats, binind, + arena_stats_large_nrequests_add(tsd_tsdn(tsd), + &arena->stats, binind, tbin->tstats.nrequests); tbin->tstats.nrequests = 0; } } - if (lock_large) { - malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx); - } + malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx); unsigned ndeferred = 0; for (unsigned i = 0; i < nflush; i++) { @@ -298,7 +244,7 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, extent = item_extent[i]; assert(ptr != NULL && extent != NULL); - if (extent_arena_ind_get(extent) == locked_arena_ind) { + if (extent_arena_get(extent) == locked_arena) { large_dalloc_finish(tsd_tsdn(tsd), extent); } else { /* @@ -324,8 +270,8 @@ tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. */ - arena_stats_large_flush_nrequests_add(tsd_tsdn(tsd), - &tcache_arena->stats, binind, tbin->tstats.nrequests); + arena_stats_large_nrequests_add(tsd_tsdn(tsd), &arena->stats, + binind, tbin->tstats.nrequests); tbin->tstats.nrequests = 0; } @@ -417,10 +363,10 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) { size_t stack_offset = 0; assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); - memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS); - memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS)); + memset(tcache->bins_small, 0, sizeof(cache_bin_t) * NBINS); + memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - NBINS)); unsigned i = 0; - for (; i < SC_NBINS; i++) { + for (; i < NBINS; i++) { tcache->lg_fill_div[i] = 1; stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); /* @@ -512,7 +458,7 @@ static void tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { assert(tcache->arena != NULL); - for (unsigned i = 0; i < SC_NBINS; i++) { + for (unsigned i = 0; i < NBINS; i++) { cache_bin_t *tbin = tcache_small_bin_get(tcache, i); tcache_bin_flush_small(tsd, tcache, tbin, i, 0); @@ -520,7 +466,7 @@ tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { assert(tbin->tstats.nrequests == 0); } } - for (unsigned i = SC_NBINS; i < nhbins; i++) { + for (unsigned i = NBINS; i < nhbins; i++) { cache_bin_t *tbin = tcache_large_bin_get(tcache, i); tcache_bin_flush_large(tsd, tbin, i, 0, tcache); @@ -545,7 +491,6 @@ tcache_flush(tsd_t *tsd) { static void tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { tcache_flush_cache(tsd, tcache); - arena_t *arena = tcache->arena; tcache_arena_dissociate(tsd_tsdn(tsd), tcache); if (tsd_tcache) { @@ -558,23 +503,6 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { /* Release both the tcache struct and avail array. */ idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true); } - - /* - * The deallocation and tcache flush above may not trigger decay since - * we are on the tcache shutdown path (potentially with non-nominal - * tsd). Manually trigger decay to avoid pathological cases. Also - * include arena 0 because the tcache array is allocated from it. - */ - arena_decay(tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false), - false, false); - - if (arena_nthreads_get(arena, false) == 0 && - !background_thread_enabled()) { - /* Force purging when no threads assigned to the arena anymore. */ - arena_decay(tsd_tsdn(tsd), arena, false, true); - } else { - arena_decay(tsd_tsdn(tsd), arena, false, false); - } } /* For auto tcache (embedded in TSD) only. */ @@ -604,10 +532,10 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { cassert(config_stats); /* Merge and reset tcache stats. */ - for (i = 0; i < SC_NBINS; i++) { + for (i = 0; i < NBINS; i++) { + bin_t *bin = &arena->bins[i]; cache_bin_t *tbin = tcache_small_bin_get(tcache, i); - unsigned binshard; - bin_t *bin = arena_bin_choose_lock(tsdn, arena, i, &binshard); + malloc_mutex_lock(tsdn, &bin->lock); bin->stats.nrequests += tbin->tstats.nrequests; malloc_mutex_unlock(tsdn, &bin->lock); tbin->tstats.nrequests = 0; @@ -615,7 +543,7 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { for (; i < nhbins; i++) { cache_bin_t *tbin = tcache_large_bin_get(tcache, i); - arena_stats_large_flush_nrequests_add(tsdn, &arena->stats, i, + arena_stats_large_nrequests_add(tsdn, &arena->stats, i, tbin->tstats.nrequests); tbin->tstats.nrequests = 0; } @@ -686,32 +614,23 @@ label_return: } static tcache_t * -tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm, bool allow_reinit) { +tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm) { malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx); if (elm->tcache == NULL) { return NULL; } tcache_t *tcache = elm->tcache; - if (allow_reinit) { - elm->tcache = TCACHES_ELM_NEED_REINIT; - } else { - elm->tcache = NULL; - } - - if (tcache == TCACHES_ELM_NEED_REINIT) { - return NULL; - } + elm->tcache = NULL; return tcache; } void tcaches_flush(tsd_t *tsd, unsigned ind) { malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); - tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind], true); + tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]); malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); if (tcache != NULL) { - /* Destroy the tcache; recreate in tcaches_get() if needed. */ tcache_destroy(tsd, tcache, false); } } @@ -720,7 +639,7 @@ void tcaches_destroy(tsd_t *tsd, unsigned ind) { malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); tcaches_t *elm = &tcaches[ind]; - tcache_t *tcache = tcaches_elm_remove(tsd, elm, false); + tcache_t *tcache = tcaches_elm_remove(tsd, elm); elm->next = tcaches_avail; tcaches_avail = elm; malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); @@ -733,8 +652,8 @@ bool tcache_boot(tsdn_t *tsdn) { /* If necessary, clamp opt_lg_tcache_max. */ if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) < - SC_SMALL_MAXCLASS) { - tcache_maxclass = SC_SMALL_MAXCLASS; + SMALL_MAXCLASS) { + tcache_maxclass = SMALL_MAXCLASS; } else { tcache_maxclass = (ZU(1) << opt_lg_tcache_max); } @@ -754,7 +673,7 @@ tcache_boot(tsdn_t *tsdn) { } stack_nelms = 0; unsigned i; - for (i = 0; i < SC_NBINS; i++) { + for (i = 0; i < NBINS; i++) { if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) { tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_SMALL_MIN; diff --git a/deps/jemalloc/src/tsd.c b/deps/jemalloc/src/tsd.c index a31f6b9..c143068 100644 --- a/deps/jemalloc/src/tsd.c +++ b/deps/jemalloc/src/tsd.c @@ -12,16 +12,12 @@ static unsigned ncleanups; static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX]; -/* TSD_INITIALIZER triggers "-Wmissing-field-initializer" */ -JEMALLOC_DIAGNOSTIC_PUSH -JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS - #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP -JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER; -JEMALLOC_TSD_TYPE_ATTR(bool) JEMALLOC_TLS_MODEL tsd_initialized = false; +__thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER; +__thread bool JEMALLOC_TLS_MODEL tsd_initialized = false; bool tsd_booted = false; #elif (defined(JEMALLOC_TLS)) -JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls = TSD_INITIALIZER; +__thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER; pthread_key_t tsd_tsd; bool tsd_booted = false; #elif (defined(_WIN32)) @@ -45,7 +41,6 @@ tsd_init_head_t tsd_init_head = { ql_head_initializer(blocks), MALLOC_MUTEX_INITIALIZER }; - tsd_wrapper_t tsd_boot_wrapper = { false, TSD_INITIALIZER @@ -53,164 +48,17 @@ tsd_wrapper_t tsd_boot_wrapper = { bool tsd_booted = false; #endif -JEMALLOC_DIAGNOSTIC_POP /******************************************************************************/ -/* A list of all the tsds in the nominal state. */ -typedef ql_head(tsd_t) tsd_list_t; -static tsd_list_t tsd_nominal_tsds = ql_head_initializer(tsd_nominal_tsds); -static malloc_mutex_t tsd_nominal_tsds_lock; - -/* How many slow-path-enabling features are turned on. */ -static atomic_u32_t tsd_global_slow_count = ATOMIC_INIT(0); - -static bool -tsd_in_nominal_list(tsd_t *tsd) { - tsd_t *tsd_list; - bool found = false; - /* - * We don't know that tsd is nominal; it might not be safe to get data - * out of it here. - */ - malloc_mutex_lock(TSDN_NULL, &tsd_nominal_tsds_lock); - ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { - if (tsd == tsd_list) { - found = true; - break; - } - } - malloc_mutex_unlock(TSDN_NULL, &tsd_nominal_tsds_lock); - return found; -} - -static void -tsd_add_nominal(tsd_t *tsd) { - assert(!tsd_in_nominal_list(tsd)); - assert(tsd_state_get(tsd) <= tsd_state_nominal_max); - ql_elm_new(tsd, TSD_MANGLE(tcache).tsd_link); - malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); - ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); - malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); -} - -static void -tsd_remove_nominal(tsd_t *tsd) { - assert(tsd_in_nominal_list(tsd)); - assert(tsd_state_get(tsd) <= tsd_state_nominal_max); - malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); - ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); - malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); -} - -static void -tsd_force_recompute(tsdn_t *tsdn) { - /* - * The stores to tsd->state here need to synchronize with the exchange - * in tsd_slow_update. - */ - atomic_fence(ATOMIC_RELEASE); - malloc_mutex_lock(tsdn, &tsd_nominal_tsds_lock); - tsd_t *remote_tsd; - ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { - assert(tsd_atomic_load(&remote_tsd->state, ATOMIC_RELAXED) - <= tsd_state_nominal_max); - tsd_atomic_store(&remote_tsd->state, tsd_state_nominal_recompute, - ATOMIC_RELAXED); - } - malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock); -} - -void -tsd_global_slow_inc(tsdn_t *tsdn) { - atomic_fetch_add_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED); - /* - * We unconditionally force a recompute, even if the global slow count - * was already positive. If we didn't, then it would be possible for us - * to return to the user, have the user synchronize externally with some - * other thread, and then have that other thread not have picked up the - * update yet (since the original incrementing thread might still be - * making its way through the tsd list). - */ - tsd_force_recompute(tsdn); -} - -void tsd_global_slow_dec(tsdn_t *tsdn) { - atomic_fetch_sub_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED); - /* See the note in ..._inc(). */ - tsd_force_recompute(tsdn); -} - -static bool -tsd_local_slow(tsd_t *tsd) { - return !tsd_tcache_enabled_get(tsd) - || tsd_reentrancy_level_get(tsd) > 0; -} - -bool -tsd_global_slow() { - return atomic_load_u32(&tsd_global_slow_count, ATOMIC_RELAXED) > 0; -} - -/******************************************************************************/ - -static uint8_t -tsd_state_compute(tsd_t *tsd) { - if (!tsd_nominal(tsd)) { - return tsd_state_get(tsd); - } - /* We're in *a* nominal state; but which one? */ - if (malloc_slow || tsd_local_slow(tsd) || tsd_global_slow()) { - return tsd_state_nominal_slow; - } else { - return tsd_state_nominal; - } -} - void tsd_slow_update(tsd_t *tsd) { - uint8_t old_state; - do { - uint8_t new_state = tsd_state_compute(tsd); - old_state = tsd_atomic_exchange(&tsd->state, new_state, - ATOMIC_ACQUIRE); - } while (old_state == tsd_state_nominal_recompute); -} - -void -tsd_state_set(tsd_t *tsd, uint8_t new_state) { - /* Only the tsd module can change the state *to* recompute. */ - assert(new_state != tsd_state_nominal_recompute); - uint8_t old_state = tsd_atomic_load(&tsd->state, ATOMIC_RELAXED); - if (old_state > tsd_state_nominal_max) { - /* - * Not currently in the nominal list, but it might need to be - * inserted there. - */ - assert(!tsd_in_nominal_list(tsd)); - tsd_atomic_store(&tsd->state, new_state, ATOMIC_RELAXED); - if (new_state <= tsd_state_nominal_max) { - tsd_add_nominal(tsd); - } - } else { - /* - * We're currently nominal. If the new state is non-nominal, - * great; we take ourselves off the list and just enter the new - * state. - */ - assert(tsd_in_nominal_list(tsd)); - if (new_state > tsd_state_nominal_max) { - tsd_remove_nominal(tsd); - tsd_atomic_store(&tsd->state, new_state, - ATOMIC_RELAXED); + if (tsd_nominal(tsd)) { + if (malloc_slow || !tsd_tcache_enabled_get(tsd) || + tsd_reentrancy_level_get(tsd) > 0) { + tsd->state = tsd_state_nominal_slow; } else { - /* - * This is the tricky case. We're transitioning from - * one nominal state to another. The caller can't know - * about any races that are occuring at the same time, - * so we always have to recompute no matter what. - */ - tsd_slow_update(tsd); + tsd->state = tsd_state_nominal; } } } @@ -239,7 +87,6 @@ tsd_data_init(tsd_t *tsd) { static void assert_tsd_data_cleanup_done(tsd_t *tsd) { assert(!tsd_nominal(tsd)); - assert(!tsd_in_nominal_list(tsd)); assert(*tsd_arenap_get_unsafe(tsd) == NULL); assert(*tsd_iarenap_get_unsafe(tsd) == NULL); assert(*tsd_arenas_tdata_bypassp_get_unsafe(tsd) == true); @@ -250,8 +97,8 @@ assert_tsd_data_cleanup_done(tsd_t *tsd) { static bool tsd_data_init_nocleanup(tsd_t *tsd) { - assert(tsd_state_get(tsd) == tsd_state_reincarnated || - tsd_state_get(tsd) == tsd_state_minimal_initialized); + assert(tsd->state == tsd_state_reincarnated || + tsd->state == tsd_state_minimal_initialized); /* * During reincarnation, there is no guarantee that the cleanup function * will be called (deallocation may happen after all tsd destructors). @@ -270,33 +117,27 @@ tsd_t * tsd_fetch_slow(tsd_t *tsd, bool minimal) { assert(!tsd_fast(tsd)); - if (tsd_state_get(tsd) == tsd_state_nominal_slow) { - /* - * On slow path but no work needed. Note that we can't - * necessarily *assert* that we're slow, because we might be - * slow because of an asynchronous modification to global state, - * which might be asynchronously modified *back*. - */ - } else if (tsd_state_get(tsd) == tsd_state_nominal_recompute) { - tsd_slow_update(tsd); - } else if (tsd_state_get(tsd) == tsd_state_uninitialized) { + if (tsd->state == tsd_state_nominal_slow) { + /* On slow path but no work needed. */ + assert(malloc_slow || !tsd_tcache_enabled_get(tsd) || + tsd_reentrancy_level_get(tsd) > 0 || + *tsd_arenas_tdata_bypassp_get(tsd)); + } else if (tsd->state == tsd_state_uninitialized) { if (!minimal) { - if (tsd_booted) { - tsd_state_set(tsd, tsd_state_nominal); - tsd_slow_update(tsd); - /* Trigger cleanup handler registration. */ - tsd_set(tsd); - tsd_data_init(tsd); - } + tsd->state = tsd_state_nominal; + tsd_slow_update(tsd); + /* Trigger cleanup handler registration. */ + tsd_set(tsd); + tsd_data_init(tsd); } else { - tsd_state_set(tsd, tsd_state_minimal_initialized); + tsd->state = tsd_state_minimal_initialized; tsd_set(tsd); tsd_data_init_nocleanup(tsd); } - } else if (tsd_state_get(tsd) == tsd_state_minimal_initialized) { + } else if (tsd->state == tsd_state_minimal_initialized) { if (!minimal) { /* Switch to fully initialized. */ - tsd_state_set(tsd, tsd_state_nominal); + tsd->state = tsd_state_nominal; assert(*tsd_reentrancy_levelp_get(tsd) >= 1); (*tsd_reentrancy_levelp_get(tsd))--; tsd_slow_update(tsd); @@ -304,12 +145,12 @@ tsd_fetch_slow(tsd_t *tsd, bool minimal) { } else { assert_tsd_data_cleanup_done(tsd); } - } else if (tsd_state_get(tsd) == tsd_state_purgatory) { - tsd_state_set(tsd, tsd_state_reincarnated); + } else if (tsd->state == tsd_state_purgatory) { + tsd->state = tsd_state_reincarnated; tsd_set(tsd); tsd_data_init_nocleanup(tsd); } else { - assert(tsd_state_get(tsd) == tsd_state_reincarnated); + assert(tsd->state == tsd_state_reincarnated); } return tsd; @@ -373,7 +214,7 @@ void tsd_cleanup(void *arg) { tsd_t *tsd = (tsd_t *)arg; - switch (tsd_state_get(tsd)) { + switch (tsd->state) { case tsd_state_uninitialized: /* Do nothing. */ break; @@ -391,7 +232,7 @@ tsd_cleanup(void *arg) { case tsd_state_nominal: case tsd_state_nominal_slow: tsd_do_data_cleanup(tsd); - tsd_state_set(tsd, tsd_state_purgatory); + tsd->state = tsd_state_purgatory; tsd_set(tsd); break; case tsd_state_purgatory: @@ -419,10 +260,6 @@ malloc_tsd_boot0(void) { tsd_t *tsd; ncleanups = 0; - if (malloc_mutex_init(&tsd_nominal_tsds_lock, "tsd_nominal_tsds_lock", - WITNESS_RANK_OMIT, malloc_mutex_rank_exclusive)) { - return NULL; - } if (tsd_boot0()) { return NULL; } @@ -473,7 +310,7 @@ _tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { # pragma comment(linker, "/INCLUDE:_tls_callback") # else # pragma comment(linker, "/INCLUDE:_tls_used") -# pragma comment(linker, "/INCLUDE:" STRINGIFY(tls_callback) ) +# pragma comment(linker, "/INCLUDE:tls_callback") # endif # pragma section(".CRT$XLY",long,read) #endif @@ -512,23 +349,3 @@ tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) { malloc_mutex_unlock(TSDN_NULL, &head->lock); } #endif - -void -tsd_prefork(tsd_t *tsd) { - malloc_mutex_prefork(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); -} - -void -tsd_postfork_parent(tsd_t *tsd) { - malloc_mutex_postfork_parent(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); -} - -void -tsd_postfork_child(tsd_t *tsd) { - malloc_mutex_postfork_child(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); - ql_new(&tsd_nominal_tsds); - - if (tsd_state_get(tsd) <= tsd_state_nominal_max) { - tsd_add_nominal(tsd); - } -} diff --git a/deps/jemalloc/test/include/test/jemalloc_test.h.in b/deps/jemalloc/test/include/test/jemalloc_test.h.in index c46af5d..67caa86 100644 --- a/deps/jemalloc/test/include/test/jemalloc_test.h.in +++ b/deps/jemalloc/test/include/test/jemalloc_test.h.in @@ -25,7 +25,7 @@ extern "C" { #include "test/jemalloc_test_defs.h" -#if defined(JEMALLOC_OSATOMIC) +#ifdef JEMALLOC_OSSPIN # include #endif @@ -69,7 +69,7 @@ static const bool config_debug = # define JEMALLOC_N(n) @private_namespace@##n # include "jemalloc/internal/private_namespace.h" -# include "jemalloc/internal/test_hooks.h" +# include "jemalloc/internal/hooks.h" /* Hermetic headers. */ # include "jemalloc/internal/assert.h" diff --git a/deps/jemalloc/test/include/test/mtx.h b/deps/jemalloc/test/include/test/mtx.h index 066a213..58afbc3 100644 --- a/deps/jemalloc/test/include/test/mtx.h +++ b/deps/jemalloc/test/include/test/mtx.h @@ -10,6 +10,8 @@ typedef struct { CRITICAL_SECTION lock; #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock lock; +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLock lock; #else pthread_mutex_t lock; #endif diff --git a/deps/jemalloc/test/integration/aligned_alloc.c b/deps/jemalloc/test/integration/aligned_alloc.c index 4375b17..536b67e 100644 --- a/deps/jemalloc/test/integration/aligned_alloc.c +++ b/deps/jemalloc/test/integration/aligned_alloc.c @@ -34,17 +34,6 @@ TEST_BEGIN(test_alignment_errors) { } TEST_END - -/* - * GCC "-Walloc-size-larger-than" warning detects when one of the memory - * allocation functions is called with a size larger than the maximum size that - * they support. Here we want to explicitly test that the allocation functions - * do indeed fail properly when this is the case, which triggers the warning. - * Therefore we disable the warning for these tests. - */ -JEMALLOC_DIAGNOSTIC_PUSH -JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN - TEST_BEGIN(test_oom_errors) { size_t alignment, size; void *p; @@ -89,9 +78,6 @@ TEST_BEGIN(test_oom_errors) { } TEST_END -/* Re-enable the "-Walloc-size-larger-than=" warning */ -JEMALLOC_DIAGNOSTIC_POP - TEST_BEGIN(test_alignment_and_size) { #define NITER 4 size_t alignment, size, total; @@ -138,20 +124,10 @@ TEST_BEGIN(test_alignment_and_size) { } TEST_END -TEST_BEGIN(test_zero_alloc) { - void *res = aligned_alloc(8, 0); - assert(res); - size_t usable = malloc_usable_size(res); - assert(usable > 0); - free(res); -} -TEST_END - int main(void) { return test( test_alignment_errors, test_oom_errors, - test_alignment_and_size, - test_zero_alloc); + test_alignment_and_size); } diff --git a/deps/jemalloc/test/integration/malloc.c b/deps/jemalloc/test/integration/malloc.c deleted file mode 100644 index 8b33bc8..0000000 --- a/deps/jemalloc/test/integration/malloc.c +++ /dev/null @@ -1,16 +0,0 @@ -#include "test/jemalloc_test.h" - -TEST_BEGIN(test_zero_alloc) { - void *res = malloc(0); - assert(res); - size_t usable = malloc_usable_size(res); - assert(usable > 0); - free(res); -} -TEST_END - -int -main(void) { - return test( - test_zero_alloc); -} diff --git a/deps/jemalloc/test/integration/mallocx.c b/deps/jemalloc/test/integration/mallocx.c index 645d4db..fd960f3 100644 --- a/deps/jemalloc/test/integration/mallocx.c +++ b/deps/jemalloc/test/integration/mallocx.c @@ -51,16 +51,6 @@ purge(void) { "Unexpected mallctl error"); } -/* - * GCC "-Walloc-size-larger-than" warning detects when one of the memory - * allocation functions is called with a size larger than the maximum size that - * they support. Here we want to explicitly test that the allocation functions - * do indeed fail properly when this is the case, which triggers the warning. - * Therefore we disable the warning for these tests. - */ -JEMALLOC_DIAGNOSTIC_PUSH -JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN - TEST_BEGIN(test_overflow) { size_t largemax; @@ -81,38 +71,6 @@ TEST_BEGIN(test_overflow) { } TEST_END -static void * -remote_alloc(void *arg) { - unsigned arena; - size_t sz = sizeof(unsigned); - assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0, - "Unexpected mallctl() failure"); - size_t large_sz; - sz = sizeof(size_t); - assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large_sz, &sz, - NULL, 0), 0, "Unexpected mallctl failure"); - - void *ptr = mallocx(large_sz, MALLOCX_ARENA(arena) - | MALLOCX_TCACHE_NONE); - void **ret = (void **)arg; - *ret = ptr; - - return NULL; -} - -TEST_BEGIN(test_remote_free) { - thd_t thd; - void *ret; - thd_create(&thd, remote_alloc, (void *)&ret); - thd_join(thd, NULL); - assert_ptr_not_null(ret, "Unexpected mallocx failure"); - - /* Avoid TCACHE_NONE to explicitly test tcache_flush(). */ - dallocx(ret, 0); - mallctl("thread.tcache.flush", NULL, NULL, NULL, 0); -} -TEST_END - TEST_BEGIN(test_oom) { size_t largemax; bool oom; @@ -126,7 +84,7 @@ TEST_BEGIN(test_oom) { largemax = get_large_size(get_nlarge()-1); oom = false; for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) { - ptrs[i] = mallocx(largemax, MALLOCX_ARENA(0)); + ptrs[i] = mallocx(largemax, 0); if (ptrs[i] == NULL) { oom = true; } @@ -155,9 +113,6 @@ TEST_BEGIN(test_oom) { } TEST_END -/* Re-enable the "-Walloc-size-larger-than=" warning */ -JEMALLOC_DIAGNOSTIC_POP - TEST_BEGIN(test_basic) { #define MAXSZ (((size_t)1) << 23) size_t sz; @@ -223,12 +178,12 @@ TEST_BEGIN(test_alignment_and_size) { sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { for (i = 0; i < NITER; i++) { nsz = nallocx(sz, MALLOCX_ALIGN(alignment) | - MALLOCX_ZERO | MALLOCX_ARENA(0)); + MALLOCX_ZERO); assert_zu_ne(nsz, 0, "nallocx() error for alignment=%zu, " "size=%zu (%#zx)", alignment, sz, sz); ps[i] = mallocx(sz, MALLOCX_ALIGN(alignment) | - MALLOCX_ZERO | MALLOCX_ARENA(0)); + MALLOCX_ZERO); assert_ptr_not_null(ps[i], "mallocx() error for alignment=%zu, " "size=%zu (%#zx)", alignment, sz, sz); @@ -268,7 +223,6 @@ main(void) { return test( test_overflow, test_oom, - test_remote_free, test_basic, test_alignment_and_size); } diff --git a/deps/jemalloc/test/integration/overflow.c b/deps/jemalloc/test/integration/overflow.c index 748ebb6..6a9785b 100644 --- a/deps/jemalloc/test/integration/overflow.c +++ b/deps/jemalloc/test/integration/overflow.c @@ -1,15 +1,5 @@ #include "test/jemalloc_test.h" -/* - * GCC "-Walloc-size-larger-than" warning detects when one of the memory - * allocation functions is called with a size larger than the maximum size that - * they support. Here we want to explicitly test that the allocation functions - * do indeed fail properly when this is the case, which triggers the warning. - * Therefore we disable the warning for these tests. - */ -JEMALLOC_DIAGNOSTIC_PUSH -JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN - TEST_BEGIN(test_overflow) { unsigned nlextents; size_t mib[4]; @@ -49,9 +39,6 @@ TEST_BEGIN(test_overflow) { } TEST_END -/* Re-enable the "-Walloc-size-larger-than=" warning */ -JEMALLOC_DIAGNOSTIC_POP - int main(void) { return test( diff --git a/deps/jemalloc/test/integration/posix_memalign.c b/deps/jemalloc/test/integration/posix_memalign.c index d992260..2c2726d 100644 --- a/deps/jemalloc/test/integration/posix_memalign.c +++ b/deps/jemalloc/test/integration/posix_memalign.c @@ -85,10 +85,9 @@ TEST_BEGIN(test_alignment_and_size) { alignment <= MAXALIGN; alignment <<= 1) { total = 0; - for (size = 0; + for (size = 1; size < 3 * alignment && size < (1U << 31); - size += ((size == 0) ? 1 : - (alignment >> (LG_SIZEOF_PTR-1)) - 1)) { + size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { for (i = 0; i < NITER; i++) { err = posix_memalign(&ps[i], alignment, size); diff --git a/deps/jemalloc/test/integration/rallocx.c b/deps/jemalloc/test/integration/rallocx.c index 08ed08d..7821ca5 100644 --- a/deps/jemalloc/test/integration/rallocx.c +++ b/deps/jemalloc/test/integration/rallocx.c @@ -208,16 +208,6 @@ TEST_BEGIN(test_lg_align_and_zero) { } TEST_END -/* - * GCC "-Walloc-size-larger-than" warning detects when one of the memory - * allocation functions is called with a size larger than the maximum size that - * they support. Here we want to explicitly test that the allocation functions - * do indeed fail properly when this is the case, which triggers the warning. - * Therefore we disable the warning for these tests. - */ -JEMALLOC_DIAGNOSTIC_PUSH -JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN - TEST_BEGIN(test_overflow) { size_t largemax; void *p; @@ -244,9 +234,6 @@ TEST_BEGIN(test_overflow) { } TEST_END -/* Re-enable the "-Walloc-size-larger-than=" warning */ -JEMALLOC_DIAGNOSTIC_POP - int main(void) { return test( diff --git a/deps/jemalloc/test/integration/slab_sizes.c b/deps/jemalloc/test/integration/slab_sizes.c deleted file mode 100644 index af250c3..0000000 --- a/deps/jemalloc/test/integration/slab_sizes.c +++ /dev/null @@ -1,80 +0,0 @@ -#include "test/jemalloc_test.h" - -/* Note that this test relies on the unusual slab sizes set in slab_sizes.sh. */ - -TEST_BEGIN(test_slab_sizes) { - unsigned nbins; - size_t page; - size_t sizemib[4]; - size_t slabmib[4]; - size_t len; - - len = sizeof(nbins); - assert_d_eq(mallctl("arenas.nbins", &nbins, &len, NULL, 0), 0, - "nbins mallctl failure"); - - len = sizeof(page); - assert_d_eq(mallctl("arenas.page", &page, &len, NULL, 0), 0, - "page mallctl failure"); - - len = 4; - assert_d_eq(mallctlnametomib("arenas.bin.0.size", sizemib, &len), 0, - "bin size mallctlnametomib failure"); - - len = 4; - assert_d_eq(mallctlnametomib("arenas.bin.0.slab_size", slabmib, &len), - 0, "slab size mallctlnametomib failure"); - - size_t biggest_slab_seen = 0; - - for (unsigned i = 0; i < nbins; i++) { - size_t bin_size; - size_t slab_size; - len = sizeof(size_t); - sizemib[2] = i; - slabmib[2] = i; - assert_d_eq(mallctlbymib(sizemib, 4, (void *)&bin_size, &len, - NULL, 0), 0, "bin size mallctlbymib failure"); - - len = sizeof(size_t); - assert_d_eq(mallctlbymib(slabmib, 4, (void *)&slab_size, &len, - NULL, 0), 0, "slab size mallctlbymib failure"); - - if (bin_size < 100) { - /* - * Then we should be as close to 17 as possible. Since - * not all page sizes are valid (because of bitmap - * limitations on the number of items in a slab), we - * should at least make sure that the number of pages - * goes up. - */ - assert_zu_ge(slab_size, biggest_slab_seen, - "Slab sizes should go up"); - biggest_slab_seen = slab_size; - } else if ( - (100 <= bin_size && bin_size < 128) - || (128 < bin_size && bin_size <= 200)) { - assert_zu_eq(slab_size, page, - "Forced-small slabs should be small"); - } else if (bin_size == 128) { - assert_zu_eq(slab_size, 2 * page, - "Forced-2-page slab should be 2 pages"); - } else if (200 < bin_size && bin_size <= 4096) { - assert_zu_ge(slab_size, biggest_slab_seen, - "Slab sizes should go up"); - biggest_slab_seen = slab_size; - } - } - /* - * For any reasonable configuration, 17 pages should be a valid slab - * size for 4096-byte items. - */ - assert_zu_eq(biggest_slab_seen, 17 * page, "Didn't hit page target"); -} -TEST_END - -int -main(void) { - return test( - test_slab_sizes); -} diff --git a/deps/jemalloc/test/integration/slab_sizes.sh b/deps/jemalloc/test/integration/slab_sizes.sh deleted file mode 100644 index 07e3db8..0000000 --- a/deps/jemalloc/test/integration/slab_sizes.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -# Some screwy-looking slab sizes. -export MALLOC_CONF="slab_sizes:1-4096:17|100-200:1|128-128:2" diff --git a/deps/jemalloc/test/integration/smallocx.c b/deps/jemalloc/test/integration/smallocx.c deleted file mode 100644 index 2486752..0000000 --- a/deps/jemalloc/test/integration/smallocx.c +++ /dev/null @@ -1,312 +0,0 @@ -#include "test/jemalloc_test.h" -#include "jemalloc/jemalloc_macros.h" - -#define STR_HELPER(x) #x -#define STR(x) STR_HELPER(x) - -#ifndef JEMALLOC_VERSION_GID_IDENT - #error "JEMALLOC_VERSION_GID_IDENT not defined" -#endif - -#define JOIN(x, y) x ## y -#define JOIN2(x, y) JOIN(x, y) -#define smallocx JOIN2(smallocx_, JEMALLOC_VERSION_GID_IDENT) - -typedef struct { - void *ptr; - size_t size; -} smallocx_return_t; - -extern smallocx_return_t -smallocx(size_t size, int flags); - -static unsigned -get_nsizes_impl(const char *cmd) { - unsigned ret; - size_t z; - - z = sizeof(unsigned); - assert_d_eq(mallctl(cmd, (void *)&ret, &z, NULL, 0), 0, - "Unexpected mallctl(\"%s\", ...) failure", cmd); - - return ret; -} - -static unsigned -get_nlarge(void) { - return get_nsizes_impl("arenas.nlextents"); -} - -static size_t -get_size_impl(const char *cmd, size_t ind) { - size_t ret; - size_t z; - size_t mib[4]; - size_t miblen = 4; - - z = sizeof(size_t); - assert_d_eq(mallctlnametomib(cmd, mib, &miblen), - 0, "Unexpected mallctlnametomib(\"%s\", ...) failure", cmd); - mib[2] = ind; - z = sizeof(size_t); - assert_d_eq(mallctlbymib(mib, miblen, (void *)&ret, &z, NULL, 0), - 0, "Unexpected mallctlbymib([\"%s\", %zu], ...) failure", cmd, ind); - - return ret; -} - -static size_t -get_large_size(size_t ind) { - return get_size_impl("arenas.lextent.0.size", ind); -} - -/* - * On systems which can't merge extents, tests that call this function generate - * a lot of dirty memory very quickly. Purging between cycles mitigates - * potential OOM on e.g. 32-bit Windows. - */ -static void -purge(void) { - assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, - "Unexpected mallctl error"); -} - -/* - * GCC "-Walloc-size-larger-than" warning detects when one of the memory - * allocation functions is called with a size larger than the maximum size that - * they support. Here we want to explicitly test that the allocation functions - * do indeed fail properly when this is the case, which triggers the warning. - * Therefore we disable the warning for these tests. - */ -JEMALLOC_DIAGNOSTIC_PUSH -JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN - -TEST_BEGIN(test_overflow) { - size_t largemax; - - largemax = get_large_size(get_nlarge()-1); - - assert_ptr_null(smallocx(largemax+1, 0).ptr, - "Expected OOM for smallocx(size=%#zx, 0)", largemax+1); - - assert_ptr_null(smallocx(ZU(PTRDIFF_MAX)+1, 0).ptr, - "Expected OOM for smallocx(size=%#zx, 0)", ZU(PTRDIFF_MAX)+1); - - assert_ptr_null(smallocx(SIZE_T_MAX, 0).ptr, - "Expected OOM for smallocx(size=%#zx, 0)", SIZE_T_MAX); - - assert_ptr_null(smallocx(1, MALLOCX_ALIGN(ZU(PTRDIFF_MAX)+1)).ptr, - "Expected OOM for smallocx(size=1, MALLOCX_ALIGN(%#zx))", - ZU(PTRDIFF_MAX)+1); -} -TEST_END - -static void * -remote_alloc(void *arg) { - unsigned arena; - size_t sz = sizeof(unsigned); - assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0, - "Unexpected mallctl() failure"); - size_t large_sz; - sz = sizeof(size_t); - assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large_sz, &sz, - NULL, 0), 0, "Unexpected mallctl failure"); - - smallocx_return_t r - = smallocx(large_sz, MALLOCX_ARENA(arena) | MALLOCX_TCACHE_NONE); - void *ptr = r.ptr; - assert_zu_eq(r.size, - nallocx(large_sz, MALLOCX_ARENA(arena) | MALLOCX_TCACHE_NONE), - "Expected smalloc(size,flags).size == nallocx(size,flags)"); - void **ret = (void **)arg; - *ret = ptr; - - return NULL; -} - -TEST_BEGIN(test_remote_free) { - thd_t thd; - void *ret; - thd_create(&thd, remote_alloc, (void *)&ret); - thd_join(thd, NULL); - assert_ptr_not_null(ret, "Unexpected smallocx failure"); - - /* Avoid TCACHE_NONE to explicitly test tcache_flush(). */ - dallocx(ret, 0); - mallctl("thread.tcache.flush", NULL, NULL, NULL, 0); -} -TEST_END - -TEST_BEGIN(test_oom) { - size_t largemax; - bool oom; - void *ptrs[3]; - unsigned i; - - /* - * It should be impossible to allocate three objects that each consume - * nearly half the virtual address space. - */ - largemax = get_large_size(get_nlarge()-1); - oom = false; - for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) { - ptrs[i] = smallocx(largemax, 0).ptr; - if (ptrs[i] == NULL) { - oom = true; - } - } - assert_true(oom, - "Expected OOM during series of calls to smallocx(size=%zu, 0)", - largemax); - for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) { - if (ptrs[i] != NULL) { - dallocx(ptrs[i], 0); - } - } - purge(); - -#if LG_SIZEOF_PTR == 3 - assert_ptr_null(smallocx(0x8000000000000000ULL, - MALLOCX_ALIGN(0x8000000000000000ULL)).ptr, - "Expected OOM for smallocx()"); - assert_ptr_null(smallocx(0x8000000000000000ULL, - MALLOCX_ALIGN(0x80000000)).ptr, - "Expected OOM for smallocx()"); -#else - assert_ptr_null(smallocx(0x80000000UL, MALLOCX_ALIGN(0x80000000UL)).ptr, - "Expected OOM for smallocx()"); -#endif -} -TEST_END - -/* Re-enable the "-Walloc-size-larger-than=" warning */ -JEMALLOC_DIAGNOSTIC_POP - -TEST_BEGIN(test_basic) { -#define MAXSZ (((size_t)1) << 23) - size_t sz; - - for (sz = 1; sz < MAXSZ; sz = nallocx(sz, 0) + 1) { - smallocx_return_t ret; - size_t nsz, rsz, smz; - void *p; - nsz = nallocx(sz, 0); - assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); - ret = smallocx(sz, 0); - p = ret.ptr; - smz = ret.size; - assert_ptr_not_null(p, - "Unexpected smallocx(size=%zx, flags=0) error", sz); - rsz = sallocx(p, 0); - assert_zu_ge(rsz, sz, "Real size smaller than expected"); - assert_zu_eq(nsz, rsz, "nallocx()/sallocx() size mismatch"); - assert_zu_eq(nsz, smz, "nallocx()/smallocx() size mismatch"); - dallocx(p, 0); - - ret = smallocx(sz, 0); - p = ret.ptr; - smz = ret.size; - assert_ptr_not_null(p, - "Unexpected smallocx(size=%zx, flags=0) error", sz); - dallocx(p, 0); - - nsz = nallocx(sz, MALLOCX_ZERO); - assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); - assert_zu_ne(smz, 0, "Unexpected smallocx() error"); - ret = smallocx(sz, MALLOCX_ZERO); - p = ret.ptr; - assert_ptr_not_null(p, - "Unexpected smallocx(size=%zx, flags=MALLOCX_ZERO) error", - nsz); - rsz = sallocx(p, 0); - assert_zu_eq(nsz, rsz, "nallocx()/sallocx() rsize mismatch"); - assert_zu_eq(nsz, smz, "nallocx()/smallocx() size mismatch"); - dallocx(p, 0); - purge(); - } -#undef MAXSZ -} -TEST_END - -TEST_BEGIN(test_alignment_and_size) { - const char *percpu_arena; - size_t sz = sizeof(percpu_arena); - - if(mallctl("opt.percpu_arena", (void *)&percpu_arena, &sz, NULL, 0) || - strcmp(percpu_arena, "disabled") != 0) { - test_skip("test_alignment_and_size skipped: " - "not working with percpu arena."); - }; -#define MAXALIGN (((size_t)1) << 23) -#define NITER 4 - size_t nsz, rsz, smz, alignment, total; - unsigned i; - void *ps[NITER]; - - for (i = 0; i < NITER; i++) { - ps[i] = NULL; - } - - for (alignment = 8; - alignment <= MAXALIGN; - alignment <<= 1) { - total = 0; - for (sz = 1; - sz < 3 * alignment && sz < (1U << 31); - sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { - for (i = 0; i < NITER; i++) { - nsz = nallocx(sz, MALLOCX_ALIGN(alignment) | - MALLOCX_ZERO); - assert_zu_ne(nsz, 0, - "nallocx() error for alignment=%zu, " - "size=%zu (%#zx)", alignment, sz, sz); - smallocx_return_t ret - = smallocx(sz, MALLOCX_ALIGN(alignment) | MALLOCX_ZERO); - ps[i] = ret.ptr; - assert_ptr_not_null(ps[i], - "smallocx() error for alignment=%zu, " - "size=%zu (%#zx)", alignment, sz, sz); - rsz = sallocx(ps[i], 0); - smz = ret.size; - assert_zu_ge(rsz, sz, - "Real size smaller than expected for " - "alignment=%zu, size=%zu", alignment, sz); - assert_zu_eq(nsz, rsz, - "nallocx()/sallocx() size mismatch for " - "alignment=%zu, size=%zu", alignment, sz); - assert_zu_eq(nsz, smz, - "nallocx()/smallocx() size mismatch for " - "alignment=%zu, size=%zu", alignment, sz); - assert_ptr_null( - (void *)((uintptr_t)ps[i] & (alignment-1)), - "%p inadequately aligned for" - " alignment=%zu, size=%zu", ps[i], - alignment, sz); - total += rsz; - if (total >= (MAXALIGN << 1)) { - break; - } - } - for (i = 0; i < NITER; i++) { - if (ps[i] != NULL) { - dallocx(ps[i], 0); - ps[i] = NULL; - } - } - } - purge(); - } -#undef MAXALIGN -#undef NITER -} -TEST_END - -int -main(void) { - return test( - test_overflow, - test_oom, - test_remote_free, - test_basic, - test_alignment_and_size); -} diff --git a/deps/jemalloc/test/integration/smallocx.sh b/deps/jemalloc/test/integration/smallocx.sh deleted file mode 100644 index d07f10f..0000000 --- a/deps/jemalloc/test/integration/smallocx.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -if [ "x${enable_fill}" = "x1" ] ; then - export MALLOC_CONF="junk:false" -fi diff --git a/deps/jemalloc/test/src/mtx.c b/deps/jemalloc/test/src/mtx.c index d9ce375..a393c01 100644 --- a/deps/jemalloc/test/src/mtx.c +++ b/deps/jemalloc/test/src/mtx.c @@ -13,6 +13,8 @@ mtx_init(mtx_t *mtx) { } #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) mtx->lock = OS_UNFAIR_LOCK_INIT; +#elif (defined(JEMALLOC_OSSPIN)) + mtx->lock = 0; #else pthread_mutexattr_t attr; @@ -33,6 +35,7 @@ void mtx_fini(mtx_t *mtx) { #ifdef _WIN32 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) +#elif (defined(JEMALLOC_OSSPIN)) #else pthread_mutex_destroy(&mtx->lock); #endif @@ -44,6 +47,8 @@ mtx_lock(mtx_t *mtx) { EnterCriticalSection(&mtx->lock); #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock_lock(&mtx->lock); +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLockLock(&mtx->lock); #else pthread_mutex_lock(&mtx->lock); #endif @@ -55,6 +60,8 @@ mtx_unlock(mtx_t *mtx) { LeaveCriticalSection(&mtx->lock); #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock_unlock(&mtx->lock); +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLockUnlock(&mtx->lock); #else pthread_mutex_unlock(&mtx->lock); #endif diff --git a/deps/jemalloc/test/src/test.c b/deps/jemalloc/test/src/test.c index f97ce4d..01a4d73 100644 --- a/deps/jemalloc/test/src/test.c +++ b/deps/jemalloc/test/src/test.c @@ -48,12 +48,12 @@ do_hook(bool *hook_ran, void (**hook)()) { static void libc_reentrancy_hook() { - do_hook(&libc_hook_ran, &test_hooks_libc_hook); + do_hook(&libc_hook_ran, &hooks_libc_hook); } static void arena_new_reentrancy_hook() { - do_hook(&arena_new_hook_ran, &test_hooks_arena_new_hook); + do_hook(&arena_new_hook_ran, &hooks_arena_new_hook); } /* Actual test infrastructure. */ @@ -110,20 +110,6 @@ p_test_fini(void) { test_status_string(test_status)); } -static void -check_global_slow(test_status_t *status) { -#ifdef JEMALLOC_UNIT_TEST - /* - * This check needs to peek into tsd internals, which is why it's only - * exposed in unit tests. - */ - if (tsd_global_slow()) { - malloc_printf("Testing increased global slow count\n"); - *status = test_status_fail; - } -#endif -} - static test_status_t p_test_impl(bool do_malloc_init, bool do_reentrant, test_t *t, va_list ap) { test_status_t ret; @@ -145,31 +131,28 @@ p_test_impl(bool do_malloc_init, bool do_reentrant, test_t *t, va_list ap) { for (; t != NULL; t = va_arg(ap, test_t *)) { /* Non-reentrant run. */ reentrancy = non_reentrant; - test_hooks_arena_new_hook = test_hooks_libc_hook = NULL; + hooks_arena_new_hook = hooks_libc_hook = NULL; t(); if (test_status > ret) { ret = test_status; } - check_global_slow(&ret); /* Reentrant run. */ if (do_reentrant) { reentrancy = libc_reentrant; - test_hooks_arena_new_hook = NULL; - test_hooks_libc_hook = &libc_reentrancy_hook; + hooks_arena_new_hook = NULL; + hooks_libc_hook = &libc_reentrancy_hook; t(); if (test_status > ret) { ret = test_status; } - check_global_slow(&ret); reentrancy = arena_new_reentrant; - test_hooks_libc_hook = NULL; - test_hooks_arena_new_hook = &arena_new_reentrancy_hook; + hooks_libc_hook = NULL; + hooks_arena_new_hook = &arena_new_reentrancy_hook; t(); if (test_status > ret) { ret = test_status; } - check_global_slow(&ret); } } diff --git a/deps/jemalloc/test/stress/hookbench.c b/deps/jemalloc/test/stress/hookbench.c deleted file mode 100644 index 97e90b0..0000000 --- a/deps/jemalloc/test/stress/hookbench.c +++ /dev/null @@ -1,73 +0,0 @@ -#include "test/jemalloc_test.h" - -static void -noop_alloc_hook(void *extra, hook_alloc_t type, void *result, - uintptr_t result_raw, uintptr_t args_raw[3]) { -} - -static void -noop_dalloc_hook(void *extra, hook_dalloc_t type, void *address, - uintptr_t args_raw[3]) { -} - -static void -noop_expand_hook(void *extra, hook_expand_t type, void *address, - size_t old_usize, size_t new_usize, uintptr_t result_raw, - uintptr_t args_raw[4]) { -} - -static void -malloc_free_loop(int iters) { - for (int i = 0; i < iters; i++) { - void *p = mallocx(1, 0); - free(p); - } -} - -static void -test_hooked(int iters) { - hooks_t hooks = {&noop_alloc_hook, &noop_dalloc_hook, &noop_expand_hook, - NULL}; - - int err; - void *handles[HOOK_MAX]; - size_t sz = sizeof(handles[0]); - - for (int i = 0; i < HOOK_MAX; i++) { - err = mallctl("experimental.hooks.install", &handles[i], - &sz, &hooks, sizeof(hooks)); - assert(err == 0); - - timedelta_t timer; - timer_start(&timer); - malloc_free_loop(iters); - timer_stop(&timer); - malloc_printf("With %d hook%s: %"FMTu64"us\n", i + 1, - i + 1 == 1 ? "" : "s", timer_usec(&timer)); - } - for (int i = 0; i < HOOK_MAX; i++) { - err = mallctl("experimental.hooks.remove", NULL, NULL, - &handles[i], sizeof(handles[i])); - assert(err == 0); - } -} - -static void -test_unhooked(int iters) { - timedelta_t timer; - timer_start(&timer); - malloc_free_loop(iters); - timer_stop(&timer); - - malloc_printf("Without hooks: %"FMTu64"us\n", timer_usec(&timer)); -} - -int -main(void) { - /* Initialize */ - free(mallocx(1, 0)); - int iters = 10 * 1000 * 1000; - malloc_printf("Benchmarking hooks with %d iterations:\n", iters); - test_hooked(iters); - test_unhooked(iters); -} diff --git a/deps/jemalloc/test/unit/arena_reset.c b/deps/jemalloc/test/unit/arena_reset.c index b182f31..f5fb24d 100644 --- a/deps/jemalloc/test/unit/arena_reset.c +++ b/deps/jemalloc/test/unit/arena_reset.c @@ -77,7 +77,7 @@ vsalloc(tsdn_t *tsdn, const void *ptr) { return 0; } - if (szind == SC_NSIZES) { + if (szind == NSIZES) { return 0; } @@ -142,7 +142,7 @@ do_arena_reset_post(void **ptrs, unsigned nptrs, unsigned arena_ind) { if (have_background_thread) { malloc_mutex_lock(tsdn, - &background_thread_info_get(arena_ind)->mtx); + &background_thread_info[arena_ind % ncpus].mtx); } /* Verify allocations no longer exist. */ for (i = 0; i < nptrs; i++) { @@ -151,7 +151,7 @@ do_arena_reset_post(void **ptrs, unsigned nptrs, unsigned arena_ind) { } if (have_background_thread) { malloc_mutex_unlock(tsdn, - &background_thread_info_get(arena_ind)->mtx); + &background_thread_info[arena_ind % ncpus].mtx); } free(ptrs); @@ -279,11 +279,8 @@ extent_dalloc_unmap(extent_hooks_t *extent_hooks, void *addr, size_t size, if (!try_dalloc) { return true; } - did_dalloc = true; - if (!maps_coalesce && opt_retain) { - return true; - } pages_unmap(addr, size); + did_dalloc = true; return false; } @@ -307,9 +304,7 @@ TEST_BEGIN(test_arena_destroy_hooks_unmap) { unsigned nptrs; extent_hooks_prep(); - if (maps_coalesce) { - try_decommit = false; - } + try_decommit = false; memcpy(&hooks_orig, &hooks, sizeof(extent_hooks_t)); memcpy(&hooks, &hooks_unmap, sizeof(extent_hooks_t)); diff --git a/deps/jemalloc/test/unit/background_thread_enable.c b/deps/jemalloc/test/unit/background_thread_enable.c index d894e93..ff95e67 100644 --- a/deps/jemalloc/test/unit/background_thread_enable.c +++ b/deps/jemalloc/test/unit/background_thread_enable.c @@ -33,19 +33,20 @@ TEST_END TEST_BEGIN(test_max_background_threads) { test_skip_if(!have_background_thread); - size_t max_n_thds; - size_t opt_max_n_thds; - size_t sz_m = sizeof(max_n_thds); + size_t maxt; + size_t opt_maxt; + size_t sz_m = sizeof(maxt); assert_d_eq(mallctl("opt.max_background_threads", - &opt_max_n_thds, &sz_m, NULL, 0), 0, - "Failed to get opt.max_background_threads"); - assert_d_eq(mallctl("max_background_threads", &max_n_thds, &sz_m, NULL, - 0), 0, "Failed to get max background threads"); - assert_zu_eq(opt_max_n_thds, max_n_thds, - "max_background_threads and " - "opt.max_background_threads should match"); - assert_d_eq(mallctl("max_background_threads", NULL, NULL, &max_n_thds, - sz_m), 0, "Failed to set max background threads"); + &opt_maxt, &sz_m, NULL, 0), 0, + "Failed to get opt.max_background_threads"); + assert_d_eq(mallctl("max_background_threads", &maxt, &sz_m, NULL, 0), 0, + "Failed to get max background threads"); + assert_zu_eq(20, maxt, "should be ncpus"); + assert_zu_eq(opt_maxt, maxt, + "max_background_threads and " + "opt.max_background_threads should match"); + assert_d_eq(mallctl("max_background_threads", NULL, NULL, &maxt, sz_m), + 0, "Failed to set max background threads"); unsigned id; size_t sz_u = sizeof(unsigned); @@ -59,21 +60,18 @@ TEST_BEGIN(test_max_background_threads) { size_t sz_b = sizeof(bool); assert_d_eq(mallctl("background_thread", NULL, NULL, &enable, sz_b), 0, "Failed to enable background threads"); - assert_zu_eq(n_background_threads, max_n_thds, - "Number of background threads should not change.\n"); - size_t new_max_thds = max_n_thds - 1; - if (new_max_thds > 0) { - assert_d_eq(mallctl("max_background_threads", NULL, NULL, - &new_max_thds, sz_m), 0, - "Failed to set max background threads"); - assert_zu_eq(n_background_threads, new_max_thds, - "Number of background threads should decrease by 1.\n"); - } - new_max_thds = 1; - assert_d_eq(mallctl("max_background_threads", NULL, NULL, &new_max_thds, - sz_m), 0, "Failed to set max background threads"); - assert_zu_eq(n_background_threads, new_max_thds, - "Number of background threads should be 1.\n"); + assert_zu_eq(n_background_threads, maxt, + "Number of background threads should be 3.\n"); + maxt = 10; + assert_d_eq(mallctl("max_background_threads", NULL, NULL, &maxt, sz_m), + 0, "Failed to set max background threads"); + assert_zu_eq(n_background_threads, maxt, + "Number of background threads should be 10.\n"); + maxt = 3; + assert_d_eq(mallctl("max_background_threads", NULL, NULL, &maxt, sz_m), + 0, "Failed to set max background threads"); + assert_zu_eq(n_background_threads, maxt, + "Number of background threads should be 3.\n"); } TEST_END diff --git a/deps/jemalloc/test/unit/binshard.c b/deps/jemalloc/test/unit/binshard.c deleted file mode 100644 index d7a8df8..0000000 --- a/deps/jemalloc/test/unit/binshard.c +++ /dev/null @@ -1,154 +0,0 @@ -#include "test/jemalloc_test.h" - -/* Config -- "narenas:1,bin_shards:1-160:16|129-512:4|256-256:8" */ - -#define NTHREADS 16 -#define REMOTE_NALLOC 256 - -static void * -thd_producer(void *varg) { - void **mem = varg; - unsigned arena, i; - size_t sz; - - sz = sizeof(arena); - /* Remote arena. */ - assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0, - "Unexpected mallctl() failure"); - for (i = 0; i < REMOTE_NALLOC / 2; i++) { - mem[i] = mallocx(1, MALLOCX_TCACHE_NONE | MALLOCX_ARENA(arena)); - } - - /* Remote bin. */ - for (; i < REMOTE_NALLOC; i++) { - mem[i] = mallocx(1, MALLOCX_TCACHE_NONE | MALLOCX_ARENA(0)); - } - - return NULL; -} - -TEST_BEGIN(test_producer_consumer) { - thd_t thds[NTHREADS]; - void *mem[NTHREADS][REMOTE_NALLOC]; - unsigned i; - - /* Create producer threads to allocate. */ - for (i = 0; i < NTHREADS; i++) { - thd_create(&thds[i], thd_producer, mem[i]); - } - for (i = 0; i < NTHREADS; i++) { - thd_join(thds[i], NULL); - } - /* Remote deallocation by the current thread. */ - for (i = 0; i < NTHREADS; i++) { - for (unsigned j = 0; j < REMOTE_NALLOC; j++) { - assert_ptr_not_null(mem[i][j], - "Unexpected remote allocation failure"); - dallocx(mem[i][j], 0); - } - } -} -TEST_END - -static void * -thd_start(void *varg) { - void *ptr, *ptr2; - extent_t *extent; - unsigned shard1, shard2; - - tsdn_t *tsdn = tsdn_fetch(); - /* Try triggering allocations from sharded bins. */ - for (unsigned i = 0; i < 1024; i++) { - ptr = mallocx(1, MALLOCX_TCACHE_NONE); - ptr2 = mallocx(129, MALLOCX_TCACHE_NONE); - - extent = iealloc(tsdn, ptr); - shard1 = extent_binshard_get(extent); - dallocx(ptr, 0); - assert_u_lt(shard1, 16, "Unexpected bin shard used"); - - extent = iealloc(tsdn, ptr2); - shard2 = extent_binshard_get(extent); - dallocx(ptr2, 0); - assert_u_lt(shard2, 4, "Unexpected bin shard used"); - - if (shard1 > 0 || shard2 > 0) { - /* Triggered sharded bin usage. */ - return (void *)(uintptr_t)shard1; - } - } - - return NULL; -} - -TEST_BEGIN(test_bin_shard_mt) { - test_skip_if(have_percpu_arena && - PERCPU_ARENA_ENABLED(opt_percpu_arena)); - - thd_t thds[NTHREADS]; - unsigned i; - for (i = 0; i < NTHREADS; i++) { - thd_create(&thds[i], thd_start, NULL); - } - bool sharded = false; - for (i = 0; i < NTHREADS; i++) { - void *ret; - thd_join(thds[i], &ret); - if (ret != NULL) { - sharded = true; - } - } - assert_b_eq(sharded, true, "Did not find sharded bins"); -} -TEST_END - -TEST_BEGIN(test_bin_shard) { - unsigned nbins, i; - size_t mib[4], mib2[4]; - size_t miblen, miblen2, len; - - len = sizeof(nbins); - assert_d_eq(mallctl("arenas.nbins", (void *)&nbins, &len, NULL, 0), 0, - "Unexpected mallctl() failure"); - - miblen = 4; - assert_d_eq(mallctlnametomib("arenas.bin.0.nshards", mib, &miblen), 0, - "Unexpected mallctlnametomib() failure"); - miblen2 = 4; - assert_d_eq(mallctlnametomib("arenas.bin.0.size", mib2, &miblen2), 0, - "Unexpected mallctlnametomib() failure"); - - for (i = 0; i < nbins; i++) { - uint32_t nshards; - size_t size, sz1, sz2; - - mib[2] = i; - sz1 = sizeof(nshards); - assert_d_eq(mallctlbymib(mib, miblen, (void *)&nshards, &sz1, - NULL, 0), 0, "Unexpected mallctlbymib() failure"); - - mib2[2] = i; - sz2 = sizeof(size); - assert_d_eq(mallctlbymib(mib2, miblen2, (void *)&size, &sz2, - NULL, 0), 0, "Unexpected mallctlbymib() failure"); - - if (size >= 1 && size <= 128) { - assert_u_eq(nshards, 16, "Unexpected nshards"); - } else if (size == 256) { - assert_u_eq(nshards, 8, "Unexpected nshards"); - } else if (size > 128 && size <= 512) { - assert_u_eq(nshards, 4, "Unexpected nshards"); - } else { - assert_u_eq(nshards, 1, "Unexpected nshards"); - } - } -} -TEST_END - -int -main(void) { - return test_no_reentrancy( - test_bin_shard, - test_bin_shard_mt, - test_producer_consumer); -} diff --git a/deps/jemalloc/test/unit/binshard.sh b/deps/jemalloc/test/unit/binshard.sh deleted file mode 100644 index c1d58c8..0000000 --- a/deps/jemalloc/test/unit/binshard.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -export MALLOC_CONF="narenas:1,bin_shards:1-160:16|129-512:4|256-256:8" diff --git a/deps/jemalloc/test/unit/bit_util.c b/deps/jemalloc/test/unit/bit_util.c index b747deb..42a9701 100644 --- a/deps/jemalloc/test/unit/bit_util.c +++ b/deps/jemalloc/test/unit/bit_util.c @@ -48,64 +48,10 @@ TEST_BEGIN(test_pow2_ceil_zu) { } TEST_END -void -assert_lg_ceil_range(size_t input, unsigned answer) { - if (input == 1) { - assert_u_eq(0, answer, "Got %u as lg_ceil of 1", answer); - return; - } - assert_zu_le(input, (ZU(1) << answer), - "Got %u as lg_ceil of %zu", answer, input); - assert_zu_gt(input, (ZU(1) << (answer - 1)), - "Got %u as lg_ceil of %zu", answer, input); -} - -void -assert_lg_floor_range(size_t input, unsigned answer) { - if (input == 1) { - assert_u_eq(0, answer, "Got %u as lg_floor of 1", answer); - return; - } - assert_zu_ge(input, (ZU(1) << answer), - "Got %u as lg_floor of %zu", answer, input); - assert_zu_lt(input, (ZU(1) << (answer + 1)), - "Got %u as lg_floor of %zu", answer, input); -} - -TEST_BEGIN(test_lg_ceil_floor) { - for (size_t i = 1; i < 10 * 1000 * 1000; i++) { - assert_lg_ceil_range(i, lg_ceil(i)); - assert_lg_ceil_range(i, LG_CEIL(i)); - assert_lg_floor_range(i, lg_floor(i)); - assert_lg_floor_range(i, LG_FLOOR(i)); - } - for (int i = 10; i < 8 * (1 << LG_SIZEOF_PTR) - 5; i++) { - for (size_t j = 0; j < (1 << 4); j++) { - size_t num1 = ((size_t)1 << i) - - j * ((size_t)1 << (i - 4)); - size_t num2 = ((size_t)1 << i) - + j * ((size_t)1 << (i - 4)); - assert_zu_ne(num1, 0, "Invalid lg argument"); - assert_zu_ne(num2, 0, "Invalid lg argument"); - assert_lg_ceil_range(num1, lg_ceil(num1)); - assert_lg_ceil_range(num1, LG_CEIL(num1)); - assert_lg_ceil_range(num2, lg_ceil(num2)); - assert_lg_ceil_range(num2, LG_CEIL(num2)); - - assert_lg_floor_range(num1, lg_floor(num1)); - assert_lg_floor_range(num1, LG_FLOOR(num1)); - assert_lg_floor_range(num2, lg_floor(num2)); - assert_lg_floor_range(num2, LG_FLOOR(num2)); - } - } -} -TEST_END - int main(void) { return test( test_pow2_ceil_u64, test_pow2_ceil_u32, - test_pow2_ceil_zu, - test_lg_ceil_floor); + test_pow2_ceil_zu); } diff --git a/deps/jemalloc/test/unit/decay.c b/deps/jemalloc/test/unit/decay.c index cf3c079..f727bf9 100644 --- a/deps/jemalloc/test/unit/decay.c +++ b/deps/jemalloc/test/unit/decay.c @@ -121,12 +121,6 @@ get_arena_dirty_npurge(unsigned arena_ind) { return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind); } -static uint64_t -get_arena_dirty_purged(unsigned arena_ind) { - do_epoch(); - return get_arena_npurge_impl("stats.arenas.0.dirty_purged", arena_ind); -} - static uint64_t get_arena_muzzy_npurge(unsigned arena_ind) { do_epoch(); @@ -565,7 +559,7 @@ TEST_BEGIN(test_decay_now) { TEST_END TEST_BEGIN(test_decay_never) { - test_skip_if(check_background_thread_enabled() || !config_stats); + test_skip_if(check_background_thread_enabled()); unsigned arena_ind = do_arena_create(-1, -1); int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; @@ -585,8 +579,8 @@ TEST_BEGIN(test_decay_never) { dallocx(ptrs[i], flags); size_t pdirty = get_arena_pdirty(arena_ind); size_t pmuzzy = get_arena_pmuzzy(arena_ind); - assert_zu_gt(pdirty + (size_t)get_arena_dirty_purged(arena_ind), - pdirty_prev, "Expected dirty pages to increase."); + assert_zu_gt(pdirty, pdirty_prev, + "Expected dirty pages to increase."); assert_zu_eq(pmuzzy, 0, "Unexpected muzzy pages"); pdirty_prev = pdirty; } diff --git a/deps/jemalloc/test/unit/emitter.c b/deps/jemalloc/test/unit/emitter.c index b4a693f..535c7cf 100644 --- a/deps/jemalloc/test/unit/emitter.c +++ b/deps/jemalloc/test/unit/emitter.c @@ -169,7 +169,7 @@ static void emit_nested_dict(emitter_t *emitter) { emitter_end(emitter); } -static const char *nested_object_json = +static const char *nested_dict_json = "{\n" "\t\"json1\": {\n" "\t\t\"json2\": {\n" @@ -183,7 +183,7 @@ static const char *nested_object_json = "\t}\n" "}\n"; -static const char *nested_object_table = +static const char *nested_dict_table = "Dict 1\n" " Dict 2\n" " A primitive: 123\n" @@ -192,8 +192,8 @@ static const char *nested_object_table = " Another primitive: 123\n"; TEST_BEGIN(test_nested_dict) { - assert_emit_output(&emit_nested_dict, nested_object_json, - nested_object_table); + assert_emit_output(&emit_nested_dict, nested_dict_json, + nested_dict_table); } TEST_END @@ -256,14 +256,13 @@ emit_modal(emitter_t *emitter) { int val = 123; emitter_begin(emitter); emitter_dict_begin(emitter, "j0", "T0"); - emitter_json_key(emitter, "j1"); - emitter_json_object_begin(emitter); + emitter_json_dict_begin(emitter, "j1"); emitter_kv(emitter, "i1", "I1", emitter_type_int, &val); emitter_json_kv(emitter, "i2", emitter_type_int, &val); emitter_table_kv(emitter, "I3", emitter_type_int, &val); emitter_table_dict_begin(emitter, "T1"); emitter_kv(emitter, "i4", "I4", emitter_type_int, &val); - emitter_json_object_end(emitter); /* Close j1 */ + emitter_json_dict_end(emitter); /* Close j1 */ emitter_kv(emitter, "i5", "I5", emitter_type_int, &val); emitter_table_dict_end(emitter); /* Close T1 */ emitter_kv(emitter, "i6", "I6", emitter_type_int, &val); @@ -303,26 +302,24 @@ emit_json_arr(emitter_t *emitter) { int ival = 123; emitter_begin(emitter); - emitter_json_key(emitter, "dict"); - emitter_json_object_begin(emitter); - emitter_json_key(emitter, "arr"); - emitter_json_array_begin(emitter); - emitter_json_object_begin(emitter); + emitter_json_dict_begin(emitter, "dict"); + emitter_json_arr_begin(emitter, "arr"); + emitter_json_arr_obj_begin(emitter); emitter_json_kv(emitter, "foo", emitter_type_int, &ival); - emitter_json_object_end(emitter); /* Close arr[0] */ + emitter_json_arr_obj_end(emitter); /* Close arr[0] */ /* arr[1] and arr[2] are primitives. */ - emitter_json_value(emitter, emitter_type_int, &ival); - emitter_json_value(emitter, emitter_type_int, &ival); - emitter_json_object_begin(emitter); + emitter_json_arr_value(emitter, emitter_type_int, &ival); + emitter_json_arr_value(emitter, emitter_type_int, &ival); + emitter_json_arr_obj_begin(emitter); emitter_json_kv(emitter, "bar", emitter_type_int, &ival); emitter_json_kv(emitter, "baz", emitter_type_int, &ival); - emitter_json_object_end(emitter); /* Close arr[3]. */ - emitter_json_array_end(emitter); /* Close arr. */ - emitter_json_object_end(emitter); /* Close dict. */ + emitter_json_arr_obj_end(emitter); /* Close arr[3]. */ + emitter_json_arr_end(emitter); /* Close arr. */ + emitter_json_dict_end(emitter); /* Close dict. */ emitter_end(emitter); } -static const char *json_array_json = +static const char *json_arr_json = "{\n" "\t\"dict\": {\n" "\t\t\"arr\": [\n" @@ -339,62 +336,10 @@ static const char *json_array_json = "\t}\n" "}\n"; -static const char *json_array_table = ""; +static const char *json_arr_table = ""; TEST_BEGIN(test_json_arr) { - assert_emit_output(&emit_json_arr, json_array_json, json_array_table); -} -TEST_END - -static void -emit_json_nested_array(emitter_t *emitter) { - int ival = 123; - char *sval = "foo"; - emitter_begin(emitter); - emitter_json_array_begin(emitter); - emitter_json_array_begin(emitter); - emitter_json_value(emitter, emitter_type_int, &ival); - emitter_json_value(emitter, emitter_type_string, &sval); - emitter_json_value(emitter, emitter_type_int, &ival); - emitter_json_value(emitter, emitter_type_string, &sval); - emitter_json_array_end(emitter); - emitter_json_array_begin(emitter); - emitter_json_value(emitter, emitter_type_int, &ival); - emitter_json_array_end(emitter); - emitter_json_array_begin(emitter); - emitter_json_value(emitter, emitter_type_string, &sval); - emitter_json_value(emitter, emitter_type_int, &ival); - emitter_json_array_end(emitter); - emitter_json_array_begin(emitter); - emitter_json_array_end(emitter); - emitter_json_array_end(emitter); - emitter_end(emitter); -} - -static const char *json_nested_array_json = -"{\n" -"\t[\n" -"\t\t[\n" -"\t\t\t123,\n" -"\t\t\t\"foo\",\n" -"\t\t\t123,\n" -"\t\t\t\"foo\"\n" -"\t\t],\n" -"\t\t[\n" -"\t\t\t123\n" -"\t\t],\n" -"\t\t[\n" -"\t\t\t\"foo\",\n" -"\t\t\t123\n" -"\t\t],\n" -"\t\t[\n" -"\t\t]\n" -"\t]\n" -"}\n"; - -TEST_BEGIN(test_json_nested_arr) { - assert_emit_output(&emit_json_nested_array, json_nested_array_json, - json_array_table); + assert_emit_output(&emit_json_arr, json_arr_json, json_arr_table); } TEST_END @@ -402,11 +347,11 @@ static void emit_table_row(emitter_t *emitter) { emitter_begin(emitter); emitter_row_t row; - emitter_col_t abc = {emitter_justify_left, 10, emitter_type_title, {0}, {0, 0}}; + emitter_col_t abc = {emitter_justify_left, 10, emitter_type_title}; abc.str_val = "ABC title"; - emitter_col_t def = {emitter_justify_right, 15, emitter_type_title, {0}, {0, 0}}; + emitter_col_t def = {emitter_justify_right, 15, emitter_type_title}; def.str_val = "DEF title"; - emitter_col_t ghi = {emitter_justify_right, 5, emitter_type_title, {0}, {0, 0}}; + emitter_col_t ghi = {emitter_justify_right, 5, emitter_type_title}; ghi.str_val = "GHI"; emitter_row_init(&row); @@ -464,6 +409,5 @@ main(void) { test_types, test_modal, test_json_arr, - test_json_nested_arr, test_table_row); } diff --git a/deps/jemalloc/test/unit/extent_util.c b/deps/jemalloc/test/unit/extent_util.c deleted file mode 100644 index 97e55f0..0000000 --- a/deps/jemalloc/test/unit/extent_util.c +++ /dev/null @@ -1,269 +0,0 @@ -#include "test/jemalloc_test.h" - -#define TEST_UTIL_EINVAL(node, a, b, c, d, why_inval) do { \ - assert_d_eq(mallctl("experimental.utilization." node, \ - a, b, c, d), EINVAL, "Should fail when " why_inval); \ - assert_zu_eq(out_sz, out_sz_ref, \ - "Output size touched when given invalid arguments"); \ - assert_d_eq(memcmp(out, out_ref, out_sz_ref), 0, \ - "Output content touched when given invalid arguments"); \ -} while (0) - -#define TEST_UTIL_QUERY_EINVAL(a, b, c, d, why_inval) \ - TEST_UTIL_EINVAL("query", a, b, c, d, why_inval) -#define TEST_UTIL_BATCH_EINVAL(a, b, c, d, why_inval) \ - TEST_UTIL_EINVAL("batch_query", a, b, c, d, why_inval) - -#define TEST_UTIL_VALID(node) do { \ - assert_d_eq(mallctl("experimental.utilization." node, \ - out, &out_sz, in, in_sz), 0, \ - "Should return 0 on correct arguments"); \ - assert_zu_eq(out_sz, out_sz_ref, "incorrect output size"); \ - assert_d_ne(memcmp(out, out_ref, out_sz_ref), 0, \ - "Output content should be changed"); \ -} while (0) - -#define TEST_UTIL_BATCH_VALID TEST_UTIL_VALID("batch_query") - -#define TEST_MAX_SIZE (1 << 20) - -TEST_BEGIN(test_query) { - size_t sz; - /* - * Select some sizes that can span both small and large sizes, and are - * numerically unrelated to any size boundaries. - */ - for (sz = 7; sz <= TEST_MAX_SIZE && sz <= SC_LARGE_MAXCLASS; - sz += (sz <= SC_SMALL_MAXCLASS ? 1009 : 99989)) { - void *p = mallocx(sz, 0); - void **in = &p; - size_t in_sz = sizeof(const void *); - size_t out_sz = sizeof(void *) + sizeof(size_t) * 5; - void *out = mallocx(out_sz, 0); - void *out_ref = mallocx(out_sz, 0); - size_t out_sz_ref = out_sz; - - assert_ptr_not_null(p, - "test pointer allocation failed"); - assert_ptr_not_null(out, - "test output allocation failed"); - assert_ptr_not_null(out_ref, - "test reference output allocation failed"); - -#define SLABCUR_READ(out) (*(void **)out) -#define COUNTS(out) ((size_t *)((void **)out + 1)) -#define NFREE_READ(out) COUNTS(out)[0] -#define NREGS_READ(out) COUNTS(out)[1] -#define SIZE_READ(out) COUNTS(out)[2] -#define BIN_NFREE_READ(out) COUNTS(out)[3] -#define BIN_NREGS_READ(out) COUNTS(out)[4] - - SLABCUR_READ(out) = NULL; - NFREE_READ(out) = NREGS_READ(out) = SIZE_READ(out) = -1; - BIN_NFREE_READ(out) = BIN_NREGS_READ(out) = -1; - memcpy(out_ref, out, out_sz); - - /* Test invalid argument(s) errors */ - TEST_UTIL_QUERY_EINVAL(NULL, &out_sz, in, in_sz, - "old is NULL"); - TEST_UTIL_QUERY_EINVAL(out, NULL, in, in_sz, - "oldlenp is NULL"); - TEST_UTIL_QUERY_EINVAL(out, &out_sz, NULL, in_sz, - "newp is NULL"); - TEST_UTIL_QUERY_EINVAL(out, &out_sz, in, 0, - "newlen is zero"); - in_sz -= 1; - TEST_UTIL_QUERY_EINVAL(out, &out_sz, in, in_sz, - "invalid newlen"); - in_sz += 1; - out_sz_ref = out_sz -= 2 * sizeof(size_t); - TEST_UTIL_QUERY_EINVAL(out, &out_sz, in, in_sz, - "invalid *oldlenp"); - out_sz_ref = out_sz += 2 * sizeof(size_t); - - /* Examine output for valid call */ - TEST_UTIL_VALID("query"); - assert_zu_le(sz, SIZE_READ(out), - "Extent size should be at least allocation size"); - assert_zu_eq(SIZE_READ(out) & (PAGE - 1), 0, - "Extent size should be a multiple of page size"); - if (sz <= SC_SMALL_MAXCLASS) { - assert_zu_le(NFREE_READ(out), NREGS_READ(out), - "Extent free count exceeded region count"); - assert_zu_le(NREGS_READ(out), SIZE_READ(out), - "Extent region count exceeded size"); - assert_zu_ne(NREGS_READ(out), 0, - "Extent region count must be positive"); - assert_ptr_not_null(SLABCUR_READ(out), - "Current slab is null"); - assert_true(NFREE_READ(out) == 0 - || SLABCUR_READ(out) <= p, - "Allocation should follow first fit principle"); - if (config_stats) { - assert_zu_le(BIN_NFREE_READ(out), - BIN_NREGS_READ(out), - "Bin free count exceeded region count"); - assert_zu_ne(BIN_NREGS_READ(out), 0, - "Bin region count must be positive"); - assert_zu_le(NFREE_READ(out), - BIN_NFREE_READ(out), - "Extent free count exceeded bin free count"); - assert_zu_le(NREGS_READ(out), - BIN_NREGS_READ(out), - "Extent region count exceeded " - "bin region count"); - assert_zu_eq(BIN_NREGS_READ(out) - % NREGS_READ(out), 0, - "Bin region count isn't a multiple of " - "extent region count"); - assert_zu_le( - BIN_NFREE_READ(out) - NFREE_READ(out), - BIN_NREGS_READ(out) - NREGS_READ(out), - "Free count in other extents in the bin " - "exceeded region count in other extents " - "in the bin"); - assert_zu_le(NREGS_READ(out) - NFREE_READ(out), - BIN_NREGS_READ(out) - BIN_NFREE_READ(out), - "Extent utilized count exceeded " - "bin utilized count"); - } - } else { - assert_zu_eq(NFREE_READ(out), 0, - "Extent free count should be zero"); - assert_zu_eq(NREGS_READ(out), 1, - "Extent region count should be one"); - assert_ptr_null(SLABCUR_READ(out), - "Current slab must be null for large size classes"); - if (config_stats) { - assert_zu_eq(BIN_NFREE_READ(out), 0, - "Bin free count must be zero for " - "large sizes"); - assert_zu_eq(BIN_NREGS_READ(out), 0, - "Bin region count must be zero for " - "large sizes"); - } - } - -#undef BIN_NREGS_READ -#undef BIN_NFREE_READ -#undef SIZE_READ -#undef NREGS_READ -#undef NFREE_READ -#undef COUNTS -#undef SLABCUR_READ - - free(out_ref); - free(out); - free(p); - } -} -TEST_END - -TEST_BEGIN(test_batch) { - size_t sz; - /* - * Select some sizes that can span both small and large sizes, and are - * numerically unrelated to any size boundaries. - */ - for (sz = 17; sz <= TEST_MAX_SIZE && sz <= SC_LARGE_MAXCLASS; - sz += (sz <= SC_SMALL_MAXCLASS ? 1019 : 99991)) { - void *p = mallocx(sz, 0); - void *q = mallocx(sz, 0); - void *in[] = {p, q}; - size_t in_sz = sizeof(const void *) * 2; - size_t out[] = {-1, -1, -1, -1, -1, -1}; - size_t out_sz = sizeof(size_t) * 6; - size_t out_ref[] = {-1, -1, -1, -1, -1, -1}; - size_t out_sz_ref = out_sz; - - assert_ptr_not_null(p, "test pointer allocation failed"); - assert_ptr_not_null(q, "test pointer allocation failed"); - - /* Test invalid argument(s) errors */ - TEST_UTIL_BATCH_EINVAL(NULL, &out_sz, in, in_sz, - "old is NULL"); - TEST_UTIL_BATCH_EINVAL(out, NULL, in, in_sz, - "oldlenp is NULL"); - TEST_UTIL_BATCH_EINVAL(out, &out_sz, NULL, in_sz, - "newp is NULL"); - TEST_UTIL_BATCH_EINVAL(out, &out_sz, in, 0, - "newlen is zero"); - in_sz -= 1; - TEST_UTIL_BATCH_EINVAL(out, &out_sz, in, in_sz, - "newlen is not an exact multiple"); - in_sz += 1; - out_sz_ref = out_sz -= 2 * sizeof(size_t); - TEST_UTIL_BATCH_EINVAL(out, &out_sz, in, in_sz, - "*oldlenp is not an exact multiple"); - out_sz_ref = out_sz += 2 * sizeof(size_t); - in_sz -= sizeof(const void *); - TEST_UTIL_BATCH_EINVAL(out, &out_sz, in, in_sz, - "*oldlenp and newlen do not match"); - in_sz += sizeof(const void *); - - /* Examine output for valid calls */ -#define TEST_EQUAL_REF(i, message) \ - assert_d_eq(memcmp(out + (i) * 3, out_ref + (i) * 3, 3), 0, message) - -#define NFREE_READ(out, i) out[(i) * 3] -#define NREGS_READ(out, i) out[(i) * 3 + 1] -#define SIZE_READ(out, i) out[(i) * 3 + 2] - - out_sz_ref = out_sz /= 2; - in_sz /= 2; - TEST_UTIL_BATCH_VALID; - assert_zu_le(sz, SIZE_READ(out, 0), - "Extent size should be at least allocation size"); - assert_zu_eq(SIZE_READ(out, 0) & (PAGE - 1), 0, - "Extent size should be a multiple of page size"); - if (sz <= SC_SMALL_MAXCLASS) { - assert_zu_le(NFREE_READ(out, 0), NREGS_READ(out, 0), - "Extent free count exceeded region count"); - assert_zu_le(NREGS_READ(out, 0), SIZE_READ(out, 0), - "Extent region count exceeded size"); - assert_zu_ne(NREGS_READ(out, 0), 0, - "Extent region count must be positive"); - } else { - assert_zu_eq(NFREE_READ(out, 0), 0, - "Extent free count should be zero"); - assert_zu_eq(NREGS_READ(out, 0), 1, - "Extent region count should be one"); - } - TEST_EQUAL_REF(1, - "Should not overwrite content beyond what's needed"); - in_sz *= 2; - out_sz_ref = out_sz *= 2; - - memcpy(out_ref, out, 3 * sizeof(size_t)); - TEST_UTIL_BATCH_VALID; - TEST_EQUAL_REF(0, "Statistics should be stable across calls"); - if (sz <= SC_SMALL_MAXCLASS) { - assert_zu_le(NFREE_READ(out, 1), NREGS_READ(out, 1), - "Extent free count exceeded region count"); - } else { - assert_zu_eq(NFREE_READ(out, 0), 0, - "Extent free count should be zero"); - } - assert_zu_eq(NREGS_READ(out, 0), NREGS_READ(out, 1), - "Extent region count should be same for same region size"); - assert_zu_eq(SIZE_READ(out, 0), SIZE_READ(out, 1), - "Extent size should be same for same region size"); - -#undef SIZE_READ -#undef NREGS_READ -#undef NFREE_READ - -#undef TEST_EQUAL_REF - - free(q); - free(p); - } -} -TEST_END - -int -main(void) { - assert_zu_lt(SC_SMALL_MAXCLASS, TEST_MAX_SIZE, - "Test case cannot cover large classes"); - return test(test_query, test_batch); -} diff --git a/deps/jemalloc/test/unit/hook.c b/deps/jemalloc/test/unit/hook.c deleted file mode 100644 index 72fcc43..0000000 --- a/deps/jemalloc/test/unit/hook.c +++ /dev/null @@ -1,580 +0,0 @@ -#include "test/jemalloc_test.h" - -#include "jemalloc/internal/hook.h" - -static void *arg_extra; -static int arg_type; -static void *arg_result; -static void *arg_address; -static size_t arg_old_usize; -static size_t arg_new_usize; -static uintptr_t arg_result_raw; -static uintptr_t arg_args_raw[4]; - -static int call_count = 0; - -static void -reset_args() { - arg_extra = NULL; - arg_type = 12345; - arg_result = NULL; - arg_address = NULL; - arg_old_usize = 0; - arg_new_usize = 0; - arg_result_raw = 0; - memset(arg_args_raw, 77, sizeof(arg_args_raw)); -} - -static void -alloc_free_size(size_t sz) { - void *ptr = mallocx(1, 0); - free(ptr); - ptr = mallocx(1, 0); - free(ptr); - ptr = mallocx(1, MALLOCX_TCACHE_NONE); - dallocx(ptr, MALLOCX_TCACHE_NONE); -} - -/* - * We want to support a degree of user reentrancy. This tests a variety of - * allocation scenarios. - */ -static void -be_reentrant() { - /* Let's make sure the tcache is non-empty if enabled. */ - alloc_free_size(1); - alloc_free_size(1024); - alloc_free_size(64 * 1024); - alloc_free_size(256 * 1024); - alloc_free_size(1024 * 1024); - - /* Some reallocation. */ - void *ptr = mallocx(129, 0); - ptr = rallocx(ptr, 130, 0); - free(ptr); - - ptr = mallocx(2 * 1024 * 1024, 0); - free(ptr); - ptr = mallocx(1 * 1024 * 1024, 0); - ptr = rallocx(ptr, 2 * 1024 * 1024, 0); - free(ptr); - - ptr = mallocx(1, 0); - ptr = rallocx(ptr, 1000, 0); - free(ptr); -} - -static void -set_args_raw(uintptr_t *args_raw, int nargs) { - memcpy(arg_args_raw, args_raw, sizeof(uintptr_t) * nargs); -} - -static void -assert_args_raw(uintptr_t *args_raw_expected, int nargs) { - int cmp = memcmp(args_raw_expected, arg_args_raw, - sizeof(uintptr_t) * nargs); - assert_d_eq(cmp, 0, "Raw args mismatch"); -} - -static void -reset() { - call_count = 0; - reset_args(); -} - -static void -test_alloc_hook(void *extra, hook_alloc_t type, void *result, - uintptr_t result_raw, uintptr_t args_raw[3]) { - call_count++; - arg_extra = extra; - arg_type = (int)type; - arg_result = result; - arg_result_raw = result_raw; - set_args_raw(args_raw, 3); - be_reentrant(); -} - -static void -test_dalloc_hook(void *extra, hook_dalloc_t type, void *address, - uintptr_t args_raw[3]) { - call_count++; - arg_extra = extra; - arg_type = (int)type; - arg_address = address; - set_args_raw(args_raw, 3); - be_reentrant(); -} - -static void -test_expand_hook(void *extra, hook_expand_t type, void *address, - size_t old_usize, size_t new_usize, uintptr_t result_raw, - uintptr_t args_raw[4]) { - call_count++; - arg_extra = extra; - arg_type = (int)type; - arg_address = address; - arg_old_usize = old_usize; - arg_new_usize = new_usize; - arg_result_raw = result_raw; - set_args_raw(args_raw, 4); - be_reentrant(); -} - -TEST_BEGIN(test_hooks_basic) { - /* Just verify that the record their arguments correctly. */ - hooks_t hooks = { - &test_alloc_hook, &test_dalloc_hook, &test_expand_hook, - (void *)111}; - void *handle = hook_install(TSDN_NULL, &hooks); - uintptr_t args_raw[4] = {10, 20, 30, 40}; - - /* Alloc */ - reset_args(); - hook_invoke_alloc(hook_alloc_posix_memalign, (void *)222, 333, - args_raw); - assert_ptr_eq(arg_extra, (void *)111, "Passed wrong user pointer"); - assert_d_eq((int)hook_alloc_posix_memalign, arg_type, - "Passed wrong alloc type"); - assert_ptr_eq((void *)222, arg_result, "Passed wrong result address"); - assert_u64_eq(333, arg_result_raw, "Passed wrong result"); - assert_args_raw(args_raw, 3); - - /* Dalloc */ - reset_args(); - hook_invoke_dalloc(hook_dalloc_sdallocx, (void *)222, args_raw); - assert_d_eq((int)hook_dalloc_sdallocx, arg_type, - "Passed wrong dalloc type"); - assert_ptr_eq((void *)111, arg_extra, "Passed wrong user pointer"); - assert_ptr_eq((void *)222, arg_address, "Passed wrong address"); - assert_args_raw(args_raw, 3); - - /* Expand */ - reset_args(); - hook_invoke_expand(hook_expand_xallocx, (void *)222, 333, 444, 555, - args_raw); - assert_d_eq((int)hook_expand_xallocx, arg_type, - "Passed wrong expand type"); - assert_ptr_eq((void *)111, arg_extra, "Passed wrong user pointer"); - assert_ptr_eq((void *)222, arg_address, "Passed wrong address"); - assert_zu_eq(333, arg_old_usize, "Passed wrong old usize"); - assert_zu_eq(444, arg_new_usize, "Passed wrong new usize"); - assert_zu_eq(555, arg_result_raw, "Passed wrong result"); - assert_args_raw(args_raw, 4); - - hook_remove(TSDN_NULL, handle); -} -TEST_END - -TEST_BEGIN(test_hooks_null) { - /* Null hooks should be ignored, not crash. */ - hooks_t hooks1 = {NULL, NULL, NULL, NULL}; - hooks_t hooks2 = {&test_alloc_hook, NULL, NULL, NULL}; - hooks_t hooks3 = {NULL, &test_dalloc_hook, NULL, NULL}; - hooks_t hooks4 = {NULL, NULL, &test_expand_hook, NULL}; - - void *handle1 = hook_install(TSDN_NULL, &hooks1); - void *handle2 = hook_install(TSDN_NULL, &hooks2); - void *handle3 = hook_install(TSDN_NULL, &hooks3); - void *handle4 = hook_install(TSDN_NULL, &hooks4); - - assert_ptr_ne(handle1, NULL, "Hook installation failed"); - assert_ptr_ne(handle2, NULL, "Hook installation failed"); - assert_ptr_ne(handle3, NULL, "Hook installation failed"); - assert_ptr_ne(handle4, NULL, "Hook installation failed"); - - uintptr_t args_raw[4] = {10, 20, 30, 40}; - - call_count = 0; - hook_invoke_alloc(hook_alloc_malloc, NULL, 0, args_raw); - assert_d_eq(call_count, 1, "Called wrong number of times"); - - call_count = 0; - hook_invoke_dalloc(hook_dalloc_free, NULL, args_raw); - assert_d_eq(call_count, 1, "Called wrong number of times"); - - call_count = 0; - hook_invoke_expand(hook_expand_realloc, NULL, 0, 0, 0, args_raw); - assert_d_eq(call_count, 1, "Called wrong number of times"); - - hook_remove(TSDN_NULL, handle1); - hook_remove(TSDN_NULL, handle2); - hook_remove(TSDN_NULL, handle3); - hook_remove(TSDN_NULL, handle4); -} -TEST_END - -TEST_BEGIN(test_hooks_remove) { - hooks_t hooks = {&test_alloc_hook, NULL, NULL, NULL}; - void *handle = hook_install(TSDN_NULL, &hooks); - assert_ptr_ne(handle, NULL, "Hook installation failed"); - call_count = 0; - uintptr_t args_raw[4] = {10, 20, 30, 40}; - hook_invoke_alloc(hook_alloc_malloc, NULL, 0, args_raw); - assert_d_eq(call_count, 1, "Hook not invoked"); - - call_count = 0; - hook_remove(TSDN_NULL, handle); - hook_invoke_alloc(hook_alloc_malloc, NULL, 0, NULL); - assert_d_eq(call_count, 0, "Hook invoked after removal"); - -} -TEST_END - -TEST_BEGIN(test_hooks_alloc_simple) { - /* "Simple" in the sense that we're not in a realloc variant. */ - hooks_t hooks = {&test_alloc_hook, NULL, NULL, (void *)123}; - void *handle = hook_install(TSDN_NULL, &hooks); - assert_ptr_ne(handle, NULL, "Hook installation failed"); - - /* Stop malloc from being optimized away. */ - volatile int err; - void *volatile ptr; - - /* malloc */ - reset(); - ptr = malloc(1); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_alloc_malloc, "Wrong hook type"); - assert_ptr_eq(ptr, arg_result, "Wrong result"); - assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)1, arg_args_raw[0], "Wrong argument"); - free(ptr); - - /* posix_memalign */ - reset(); - err = posix_memalign((void **)&ptr, 1024, 1); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_alloc_posix_memalign, - "Wrong hook type"); - assert_ptr_eq(ptr, arg_result, "Wrong result"); - assert_u64_eq((uintptr_t)err, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)&ptr, arg_args_raw[0], "Wrong argument"); - assert_u64_eq((uintptr_t)1024, arg_args_raw[1], "Wrong argument"); - assert_u64_eq((uintptr_t)1, arg_args_raw[2], "Wrong argument"); - free(ptr); - - /* aligned_alloc */ - reset(); - ptr = aligned_alloc(1024, 1); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_alloc_aligned_alloc, - "Wrong hook type"); - assert_ptr_eq(ptr, arg_result, "Wrong result"); - assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)1024, arg_args_raw[0], "Wrong argument"); - assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong argument"); - free(ptr); - - /* calloc */ - reset(); - ptr = calloc(11, 13); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_alloc_calloc, "Wrong hook type"); - assert_ptr_eq(ptr, arg_result, "Wrong result"); - assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)11, arg_args_raw[0], "Wrong argument"); - assert_u64_eq((uintptr_t)13, arg_args_raw[1], "Wrong argument"); - free(ptr); - - /* memalign */ -#ifdef JEMALLOC_OVERRIDE_MEMALIGN - reset(); - ptr = memalign(1024, 1); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_alloc_memalign, "Wrong hook type"); - assert_ptr_eq(ptr, arg_result, "Wrong result"); - assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)1024, arg_args_raw[0], "Wrong argument"); - assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong argument"); - free(ptr); -#endif /* JEMALLOC_OVERRIDE_MEMALIGN */ - - /* valloc */ -#ifdef JEMALLOC_OVERRIDE_VALLOC - reset(); - ptr = valloc(1); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_alloc_valloc, "Wrong hook type"); - assert_ptr_eq(ptr, arg_result, "Wrong result"); - assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)1, arg_args_raw[0], "Wrong argument"); - free(ptr); -#endif /* JEMALLOC_OVERRIDE_VALLOC */ - - /* mallocx */ - reset(); - ptr = mallocx(1, MALLOCX_LG_ALIGN(10)); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_alloc_mallocx, "Wrong hook type"); - assert_ptr_eq(ptr, arg_result, "Wrong result"); - assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)1, arg_args_raw[0], "Wrong argument"); - assert_u64_eq((uintptr_t)MALLOCX_LG_ALIGN(10), arg_args_raw[1], - "Wrong flags"); - free(ptr); - - hook_remove(TSDN_NULL, handle); -} -TEST_END - -TEST_BEGIN(test_hooks_dalloc_simple) { - /* "Simple" in the sense that we're not in a realloc variant. */ - hooks_t hooks = {NULL, &test_dalloc_hook, NULL, (void *)123}; - void *handle = hook_install(TSDN_NULL, &hooks); - assert_ptr_ne(handle, NULL, "Hook installation failed"); - - void *volatile ptr; - - /* free() */ - reset(); - ptr = malloc(1); - free(ptr); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_dalloc_free, "Wrong hook type"); - assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); - assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); - - /* dallocx() */ - reset(); - ptr = malloc(1); - dallocx(ptr, MALLOCX_TCACHE_NONE); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_dalloc_dallocx, "Wrong hook type"); - assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); - assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); - assert_u64_eq((uintptr_t)MALLOCX_TCACHE_NONE, arg_args_raw[1], - "Wrong raw arg"); - - /* sdallocx() */ - reset(); - ptr = malloc(1); - sdallocx(ptr, 1, MALLOCX_TCACHE_NONE); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_dalloc_sdallocx, "Wrong hook type"); - assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); - assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); - assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong raw arg"); - assert_u64_eq((uintptr_t)MALLOCX_TCACHE_NONE, arg_args_raw[2], - "Wrong raw arg"); - - hook_remove(TSDN_NULL, handle); -} -TEST_END - -TEST_BEGIN(test_hooks_expand_simple) { - /* "Simple" in the sense that we're not in a realloc variant. */ - hooks_t hooks = {NULL, NULL, &test_expand_hook, (void *)123}; - void *handle = hook_install(TSDN_NULL, &hooks); - assert_ptr_ne(handle, NULL, "Hook installation failed"); - - void *volatile ptr; - - /* xallocx() */ - reset(); - ptr = malloc(1); - size_t new_usize = xallocx(ptr, 100, 200, MALLOCX_TCACHE_NONE); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_expand_xallocx, "Wrong hook type"); - assert_ptr_eq(ptr, arg_address, "Wrong pointer expanded"); - assert_u64_eq(arg_old_usize, nallocx(1, 0), "Wrong old usize"); - assert_u64_eq(arg_new_usize, sallocx(ptr, 0), "Wrong new usize"); - assert_u64_eq(new_usize, arg_result_raw, "Wrong result"); - assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong arg"); - assert_u64_eq(100, arg_args_raw[1], "Wrong arg"); - assert_u64_eq(200, arg_args_raw[2], "Wrong arg"); - assert_u64_eq(MALLOCX_TCACHE_NONE, arg_args_raw[3], "Wrong arg"); - - hook_remove(TSDN_NULL, handle); -} -TEST_END - -TEST_BEGIN(test_hooks_realloc_as_malloc_or_free) { - hooks_t hooks = {&test_alloc_hook, &test_dalloc_hook, - &test_expand_hook, (void *)123}; - void *handle = hook_install(TSDN_NULL, &hooks); - assert_ptr_ne(handle, NULL, "Hook installation failed"); - - void *volatile ptr; - - /* realloc(NULL, size) as malloc */ - reset(); - ptr = realloc(NULL, 1); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_alloc_realloc, "Wrong hook type"); - assert_ptr_eq(ptr, arg_result, "Wrong result"); - assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)NULL, arg_args_raw[0], "Wrong argument"); - assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong argument"); - free(ptr); - - /* realloc(ptr, 0) as free */ - ptr = malloc(1); - reset(); - realloc(ptr, 0); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_dalloc_realloc, "Wrong hook type"); - assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); - assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); - assert_u64_eq((uintptr_t)0, arg_args_raw[1], "Wrong raw arg"); - - /* realloc(NULL, 0) as malloc(0) */ - reset(); - ptr = realloc(NULL, 0); - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, (int)hook_alloc_realloc, "Wrong hook type"); - assert_ptr_eq(ptr, arg_result, "Wrong result"); - assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)NULL, arg_args_raw[0], "Wrong argument"); - assert_u64_eq((uintptr_t)0, arg_args_raw[1], "Wrong argument"); - free(ptr); - - hook_remove(TSDN_NULL, handle); -} -TEST_END - -static void -do_realloc_test(void *(*ralloc)(void *, size_t, int), int flags, - int expand_type, int dalloc_type) { - hooks_t hooks = {&test_alloc_hook, &test_dalloc_hook, - &test_expand_hook, (void *)123}; - void *handle = hook_install(TSDN_NULL, &hooks); - assert_ptr_ne(handle, NULL, "Hook installation failed"); - - void *volatile ptr; - void *volatile ptr2; - - /* Realloc in-place, small. */ - ptr = malloc(129); - reset(); - ptr2 = ralloc(ptr, 130, flags); - assert_ptr_eq(ptr, ptr2, "Small realloc moved"); - - assert_d_eq(call_count, 1, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, expand_type, "Wrong hook type"); - assert_ptr_eq(ptr, arg_address, "Wrong address"); - assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong argument"); - assert_u64_eq((uintptr_t)130, arg_args_raw[1], "Wrong argument"); - free(ptr); - - /* - * Realloc in-place, large. Since we can't guarantee the large case - * across all platforms, we stay resilient to moving results. - */ - ptr = malloc(2 * 1024 * 1024); - free(ptr); - ptr2 = malloc(1 * 1024 * 1024); - reset(); - ptr = ralloc(ptr2, 2 * 1024 * 1024, flags); - /* ptr is the new address, ptr2 is the old address. */ - if (ptr == ptr2) { - assert_d_eq(call_count, 1, "Hook not called"); - assert_d_eq(arg_type, expand_type, "Wrong hook type"); - } else { - assert_d_eq(call_count, 2, "Wrong hooks called"); - assert_ptr_eq(ptr, arg_result, "Wrong address"); - assert_d_eq(arg_type, dalloc_type, "Wrong hook type"); - } - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_ptr_eq(ptr2, arg_address, "Wrong address"); - assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)ptr2, arg_args_raw[0], "Wrong argument"); - assert_u64_eq((uintptr_t)2 * 1024 * 1024, arg_args_raw[1], - "Wrong argument"); - free(ptr); - - /* Realloc with move, small. */ - ptr = malloc(8); - reset(); - ptr2 = ralloc(ptr, 128, flags); - assert_ptr_ne(ptr, ptr2, "Small realloc didn't move"); - - assert_d_eq(call_count, 2, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, dalloc_type, "Wrong hook type"); - assert_ptr_eq(ptr, arg_address, "Wrong address"); - assert_ptr_eq(ptr2, arg_result, "Wrong address"); - assert_u64_eq((uintptr_t)ptr2, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong argument"); - assert_u64_eq((uintptr_t)128, arg_args_raw[1], "Wrong argument"); - free(ptr2); - - /* Realloc with move, large. */ - ptr = malloc(1); - reset(); - ptr2 = ralloc(ptr, 2 * 1024 * 1024, flags); - assert_ptr_ne(ptr, ptr2, "Large realloc didn't move"); - - assert_d_eq(call_count, 2, "Hook not called"); - assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); - assert_d_eq(arg_type, dalloc_type, "Wrong hook type"); - assert_ptr_eq(ptr, arg_address, "Wrong address"); - assert_ptr_eq(ptr2, arg_result, "Wrong address"); - assert_u64_eq((uintptr_t)ptr2, (uintptr_t)arg_result_raw, - "Wrong raw result"); - assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong argument"); - assert_u64_eq((uintptr_t)2 * 1024 * 1024, arg_args_raw[1], - "Wrong argument"); - free(ptr2); - - hook_remove(TSDN_NULL, handle); -} - -static void * -realloc_wrapper(void *ptr, size_t size, UNUSED int flags) { - return realloc(ptr, size); -} - -TEST_BEGIN(test_hooks_realloc) { - do_realloc_test(&realloc_wrapper, 0, hook_expand_realloc, - hook_dalloc_realloc); -} -TEST_END - -TEST_BEGIN(test_hooks_rallocx) { - do_realloc_test(&rallocx, MALLOCX_TCACHE_NONE, hook_expand_rallocx, - hook_dalloc_rallocx); -} -TEST_END - -int -main(void) { - /* We assert on call counts. */ - return test_no_reentrancy( - test_hooks_basic, - test_hooks_null, - test_hooks_remove, - test_hooks_alloc_simple, - test_hooks_dalloc_simple, - test_hooks_expand_simple, - test_hooks_realloc_as_malloc_or_free, - test_hooks_realloc, - test_hooks_rallocx); -} diff --git a/deps/jemalloc/test/unit/test_hooks.c b/deps/jemalloc/test/unit/hooks.c similarity index 82% rename from deps/jemalloc/test/unit/test_hooks.c rename to deps/jemalloc/test/unit/hooks.c index ded8698..b70172e 100644 --- a/deps/jemalloc/test/unit/test_hooks.c +++ b/deps/jemalloc/test/unit/hooks.c @@ -12,10 +12,10 @@ func_to_hook(int arg1, int arg2) { return arg1 + arg2; } -#define func_to_hook JEMALLOC_HOOK(func_to_hook, test_hooks_libc_hook) +#define func_to_hook JEMALLOC_HOOK(func_to_hook, hooks_libc_hook) TEST_BEGIN(unhooked_call) { - test_hooks_libc_hook = NULL; + hooks_libc_hook = NULL; hook_called = false; assert_d_eq(3, func_to_hook(1, 2), "Hooking changed return value."); assert_false(hook_called, "Nulling out hook didn't take."); @@ -23,7 +23,7 @@ TEST_BEGIN(unhooked_call) { TEST_END TEST_BEGIN(hooked_call) { - test_hooks_libc_hook = &hook; + hooks_libc_hook = &hook; hook_called = false; assert_d_eq(3, func_to_hook(1, 2), "Hooking changed return value."); assert_true(hook_called, "Hook should have executed."); diff --git a/deps/jemalloc/test/unit/huge.c b/deps/jemalloc/test/unit/huge.c deleted file mode 100644 index ab72cf0..0000000 --- a/deps/jemalloc/test/unit/huge.c +++ /dev/null @@ -1,108 +0,0 @@ -#include "test/jemalloc_test.h" - -/* Threshold: 2 << 20 = 2097152. */ -const char *malloc_conf = "oversize_threshold:2097152"; - -#define HUGE_SZ (2 << 20) -#define SMALL_SZ (8) - -TEST_BEGIN(huge_bind_thread) { - unsigned arena1, arena2; - size_t sz = sizeof(unsigned); - - /* Bind to a manual arena. */ - assert_d_eq(mallctl("arenas.create", &arena1, &sz, NULL, 0), 0, - "Failed to create arena"); - assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena1, - sizeof(arena1)), 0, "Fail to bind thread"); - - void *ptr = mallocx(HUGE_SZ, 0); - assert_ptr_not_null(ptr, "Fail to allocate huge size"); - assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, - sizeof(ptr)), 0, "Unexpected mallctl() failure"); - assert_u_eq(arena1, arena2, "Wrong arena used after binding"); - dallocx(ptr, 0); - - /* Switch back to arena 0. */ - test_skip_if(have_percpu_arena && - PERCPU_ARENA_ENABLED(opt_percpu_arena)); - arena2 = 0; - assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena2, - sizeof(arena2)), 0, "Fail to bind thread"); - ptr = mallocx(SMALL_SZ, MALLOCX_TCACHE_NONE); - assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, - sizeof(ptr)), 0, "Unexpected mallctl() failure"); - assert_u_eq(arena2, 0, "Wrong arena used after binding"); - dallocx(ptr, MALLOCX_TCACHE_NONE); - - /* Then huge allocation should use the huge arena. */ - ptr = mallocx(HUGE_SZ, 0); - assert_ptr_not_null(ptr, "Fail to allocate huge size"); - assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, - sizeof(ptr)), 0, "Unexpected mallctl() failure"); - assert_u_ne(arena2, 0, "Wrong arena used after binding"); - assert_u_ne(arena1, arena2, "Wrong arena used after binding"); - dallocx(ptr, 0); -} -TEST_END - -TEST_BEGIN(huge_mallocx) { - unsigned arena1, arena2; - size_t sz = sizeof(unsigned); - - assert_d_eq(mallctl("arenas.create", &arena1, &sz, NULL, 0), 0, - "Failed to create arena"); - void *huge = mallocx(HUGE_SZ, MALLOCX_ARENA(arena1)); - assert_ptr_not_null(huge, "Fail to allocate huge size"); - assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &huge, - sizeof(huge)), 0, "Unexpected mallctl() failure"); - assert_u_eq(arena1, arena2, "Wrong arena used for mallocx"); - dallocx(huge, MALLOCX_ARENA(arena1)); - - void *huge2 = mallocx(HUGE_SZ, 0); - assert_ptr_not_null(huge, "Fail to allocate huge size"); - assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &huge2, - sizeof(huge2)), 0, "Unexpected mallctl() failure"); - assert_u_ne(arena1, arena2, - "Huge allocation should not come from the manual arena."); - assert_u_ne(arena2, 0, - "Huge allocation should not come from the arena 0."); - dallocx(huge2, 0); -} -TEST_END - -TEST_BEGIN(huge_allocation) { - unsigned arena1, arena2; - - void *ptr = mallocx(HUGE_SZ, 0); - assert_ptr_not_null(ptr, "Fail to allocate huge size"); - size_t sz = sizeof(unsigned); - assert_d_eq(mallctl("arenas.lookup", &arena1, &sz, &ptr, sizeof(ptr)), - 0, "Unexpected mallctl() failure"); - assert_u_gt(arena1, 0, "Huge allocation should not come from arena 0"); - dallocx(ptr, 0); - - ptr = mallocx(HUGE_SZ >> 1, 0); - assert_ptr_not_null(ptr, "Fail to allocate half huge size"); - assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, - sizeof(ptr)), 0, "Unexpected mallctl() failure"); - assert_u_ne(arena1, arena2, "Wrong arena used for half huge"); - dallocx(ptr, 0); - - ptr = mallocx(SMALL_SZ, MALLOCX_TCACHE_NONE); - assert_ptr_not_null(ptr, "Fail to allocate small size"); - assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, - sizeof(ptr)), 0, "Unexpected mallctl() failure"); - assert_u_ne(arena1, arena2, - "Huge and small should be from different arenas"); - dallocx(ptr, 0); -} -TEST_END - -int -main(void) { - return test( - huge_allocation, - huge_mallocx, - huge_bind_thread); -} diff --git a/deps/jemalloc/test/unit/junk.c b/deps/jemalloc/test/unit/junk.c index 57e3ad4..243ced4 100644 --- a/deps/jemalloc/test/unit/junk.c +++ b/deps/jemalloc/test/unit/junk.c @@ -123,13 +123,13 @@ test_junk(size_t sz_min, size_t sz_max) { TEST_BEGIN(test_junk_small) { test_skip_if(!config_fill); - test_junk(1, SC_SMALL_MAXCLASS - 1); + test_junk(1, SMALL_MAXCLASS-1); } TEST_END TEST_BEGIN(test_junk_large) { test_skip_if(!config_fill); - test_junk(SC_SMALL_MAXCLASS + 1, (1U << (SC_LG_LARGE_MINCLASS + 1))); + test_junk(SMALL_MAXCLASS+1, (1U << (LG_LARGE_MINCLASS+1))); } TEST_END diff --git a/deps/jemalloc/test/unit/mallctl.c b/deps/jemalloc/test/unit/mallctl.c index 3a75ac0..1ecbab0 100644 --- a/deps/jemalloc/test/unit/mallctl.c +++ b/deps/jemalloc/test/unit/mallctl.c @@ -1,6 +1,5 @@ #include "test/jemalloc_test.h" -#include "jemalloc/internal/hook.h" #include "jemalloc/internal/util.h" TEST_BEGIN(test_mallctl_errors) { @@ -159,13 +158,11 @@ TEST_BEGIN(test_mallctl_opt) { TEST_MALLCTL_OPT(bool, abort, always); TEST_MALLCTL_OPT(bool, abort_conf, always); - TEST_MALLCTL_OPT(bool, confirm_conf, always); TEST_MALLCTL_OPT(const char *, metadata_thp, always); TEST_MALLCTL_OPT(bool, retain, always); TEST_MALLCTL_OPT(const char *, dss, always); TEST_MALLCTL_OPT(unsigned, narenas, always); TEST_MALLCTL_OPT(const char *, percpu_arena, always); - TEST_MALLCTL_OPT(size_t, oversize_threshold, always); TEST_MALLCTL_OPT(bool, background_thread, always); TEST_MALLCTL_OPT(ssize_t, dirty_decay_ms, always); TEST_MALLCTL_OPT(ssize_t, muzzy_decay_ms, always); @@ -343,9 +340,6 @@ TEST_BEGIN(test_thread_arena) { sz = sizeof(unsigned); assert_d_eq(mallctl("arenas.narenas", (void *)&narenas, &sz, NULL, 0), 0, "Unexpected mallctl() failure"); - if (opt_oversize_threshold != 0) { - narenas--; - } assert_u_eq(narenas, opt_narenas, "Number of arenas incorrect"); if (strcmp(opa, "disabled") == 0) { @@ -582,7 +576,7 @@ TEST_BEGIN(test_arena_i_retain_grow_limit) { assert_d_eq(mallctlbymib(mib, miblen, &default_limit, &sz, NULL, 0), 0, "Unexpected mallctl() failure"); - assert_zu_eq(default_limit, SC_LARGE_MAXCLASS, + assert_zu_eq(default_limit, sz_pind2sz(EXTENT_GROW_MAX_PIND), "Unexpected default for retain_grow_limit"); new_limit = PAGE - 1; @@ -687,8 +681,8 @@ TEST_BEGIN(test_arenas_constants) { TEST_ARENAS_CONSTANT(size_t, quantum, QUANTUM); TEST_ARENAS_CONSTANT(size_t, page, PAGE); - TEST_ARENAS_CONSTANT(unsigned, nbins, SC_NBINS); - TEST_ARENAS_CONSTANT(unsigned, nlextents, SC_NSIZES - SC_NBINS); + TEST_ARENAS_CONSTANT(unsigned, nbins, NBINS); + TEST_ARENAS_CONSTANT(unsigned, nlextents, NSIZES - NBINS); #undef TEST_ARENAS_CONSTANT } @@ -707,7 +701,6 @@ TEST_BEGIN(test_arenas_bin_constants) { TEST_ARENAS_BIN_CONSTANT(uint32_t, nregs, bin_infos[0].nregs); TEST_ARENAS_BIN_CONSTANT(size_t, slab_size, bin_infos[0].slab_size); - TEST_ARENAS_BIN_CONSTANT(uint32_t, nshards, bin_infos[0].n_shards); #undef TEST_ARENAS_BIN_CONSTANT } @@ -722,8 +715,7 @@ TEST_BEGIN(test_arenas_lextent_constants) { assert_zu_eq(name, expected, "Incorrect "#name" size"); \ } while (0) - TEST_ARENAS_LEXTENT_CONSTANT(size_t, size, - SC_LARGE_MINCLASS); + TEST_ARENAS_LEXTENT_CONSTANT(size_t, size, LARGE_MINCLASS); #undef TEST_ARENAS_LEXTENT_CONSTANT } @@ -781,79 +773,6 @@ TEST_BEGIN(test_stats_arenas) { } TEST_END -static void -alloc_hook(void *extra, UNUSED hook_alloc_t type, UNUSED void *result, - UNUSED uintptr_t result_raw, UNUSED uintptr_t args_raw[3]) { - *(bool *)extra = true; -} - -static void -dalloc_hook(void *extra, UNUSED hook_dalloc_t type, - UNUSED void *address, UNUSED uintptr_t args_raw[3]) { - *(bool *)extra = true; -} - -TEST_BEGIN(test_hooks) { - bool hook_called = false; - hooks_t hooks = {&alloc_hook, &dalloc_hook, NULL, &hook_called}; - void *handle = NULL; - size_t sz = sizeof(handle); - int err = mallctl("experimental.hooks.install", &handle, &sz, &hooks, - sizeof(hooks)); - assert_d_eq(err, 0, "Hook installation failed"); - assert_ptr_ne(handle, NULL, "Hook installation gave null handle"); - void *ptr = mallocx(1, 0); - assert_true(hook_called, "Alloc hook not called"); - hook_called = false; - free(ptr); - assert_true(hook_called, "Free hook not called"); - - err = mallctl("experimental.hooks.remove", NULL, NULL, &handle, - sizeof(handle)); - assert_d_eq(err, 0, "Hook removal failed"); - hook_called = false; - ptr = mallocx(1, 0); - free(ptr); - assert_false(hook_called, "Hook called after removal"); -} -TEST_END - -TEST_BEGIN(test_hooks_exhaustion) { - bool hook_called = false; - hooks_t hooks = {&alloc_hook, &dalloc_hook, NULL, &hook_called}; - - void *handle; - void *handles[HOOK_MAX]; - size_t sz = sizeof(handle); - int err; - for (int i = 0; i < HOOK_MAX; i++) { - handle = NULL; - err = mallctl("experimental.hooks.install", &handle, &sz, - &hooks, sizeof(hooks)); - assert_d_eq(err, 0, "Error installation hooks"); - assert_ptr_ne(handle, NULL, "Got NULL handle"); - handles[i] = handle; - } - err = mallctl("experimental.hooks.install", &handle, &sz, &hooks, - sizeof(hooks)); - assert_d_eq(err, EAGAIN, "Should have failed hook installation"); - for (int i = 0; i < HOOK_MAX; i++) { - err = mallctl("experimental.hooks.remove", NULL, NULL, - &handles[i], sizeof(handles[i])); - assert_d_eq(err, 0, "Hook removal failed"); - } - /* Insertion failed, but then we removed some; it should work now. */ - handle = NULL; - err = mallctl("experimental.hooks.install", &handle, &sz, &hooks, - sizeof(hooks)); - assert_d_eq(err, 0, "Hook insertion failed"); - assert_ptr_ne(handle, NULL, "Got NULL handle"); - err = mallctl("experimental.hooks.remove", NULL, NULL, &handle, - sizeof(handle)); - assert_d_eq(err, 0, "Hook removal failed"); -} -TEST_END - int main(void) { return test( @@ -882,7 +801,5 @@ main(void) { test_arenas_lextent_constants, test_arenas_create, test_arenas_lookup, - test_stats_arenas, - test_hooks, - test_hooks_exhaustion); + test_stats_arenas); } diff --git a/deps/jemalloc/test/unit/prof_gdump.c b/deps/jemalloc/test/unit/prof_gdump.c index f7e0aac..fcb434c 100644 --- a/deps/jemalloc/test/unit/prof_gdump.c +++ b/deps/jemalloc/test/unit/prof_gdump.c @@ -29,12 +29,12 @@ TEST_BEGIN(test_gdump) { prof_dump_open = prof_dump_open_intercept; did_prof_dump_open = false; - p = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); + p = mallocx((1U << LG_LARGE_MINCLASS), 0); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_true(did_prof_dump_open, "Expected a profile dump"); did_prof_dump_open = false; - q = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); + q = mallocx((1U << LG_LARGE_MINCLASS), 0); assert_ptr_not_null(q, "Unexpected mallocx() failure"); assert_true(did_prof_dump_open, "Expected a profile dump"); @@ -45,7 +45,7 @@ TEST_BEGIN(test_gdump) { "Unexpected mallctl failure while disabling prof.gdump"); assert(gdump_old); did_prof_dump_open = false; - r = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); + r = mallocx((1U << LG_LARGE_MINCLASS), 0); assert_ptr_not_null(q, "Unexpected mallocx() failure"); assert_false(did_prof_dump_open, "Unexpected profile dump"); @@ -56,7 +56,7 @@ TEST_BEGIN(test_gdump) { "Unexpected mallctl failure while enabling prof.gdump"); assert(!gdump_old); did_prof_dump_open = false; - s = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); + s = mallocx((1U << LG_LARGE_MINCLASS), 0); assert_ptr_not_null(q, "Unexpected mallocx() failure"); assert_true(did_prof_dump_open, "Expected a profile dump"); diff --git a/deps/jemalloc/test/unit/prof_log.c b/deps/jemalloc/test/unit/prof_log.c deleted file mode 100644 index 92fbd7c..0000000 --- a/deps/jemalloc/test/unit/prof_log.c +++ /dev/null @@ -1,148 +0,0 @@ -#include "test/jemalloc_test.h" - -#define N_PARAM 100 -#define N_THREADS 10 - -static void assert_rep() { - assert_b_eq(prof_log_rep_check(), false, "Rep check failed"); -} - -static void assert_log_empty() { - assert_zu_eq(prof_log_bt_count(), 0, - "The log has backtraces; it isn't empty"); - assert_zu_eq(prof_log_thr_count(), 0, - "The log has threads; it isn't empty"); - assert_zu_eq(prof_log_alloc_count(), 0, - "The log has allocations; it isn't empty"); -} - -void *buf[N_PARAM]; - -static void f() { - int i; - for (i = 0; i < N_PARAM; i++) { - buf[i] = malloc(100); - } - for (i = 0; i < N_PARAM; i++) { - free(buf[i]); - } -} - -TEST_BEGIN(test_prof_log_many_logs) { - int i; - - test_skip_if(!config_prof); - - for (i = 0; i < N_PARAM; i++) { - assert_b_eq(prof_log_is_logging(), false, - "Logging shouldn't have started yet"); - assert_d_eq(mallctl("prof.log_start", NULL, NULL, NULL, 0), 0, - "Unexpected mallctl failure when starting logging"); - assert_b_eq(prof_log_is_logging(), true, - "Logging should be started by now"); - assert_log_empty(); - assert_rep(); - f(); - assert_zu_eq(prof_log_thr_count(), 1, "Wrong thread count"); - assert_rep(); - assert_b_eq(prof_log_is_logging(), true, - "Logging should still be on"); - assert_d_eq(mallctl("prof.log_stop", NULL, NULL, NULL, 0), 0, - "Unexpected mallctl failure when stopping logging"); - assert_b_eq(prof_log_is_logging(), false, - "Logging should have turned off"); - } -} -TEST_END - -thd_t thr_buf[N_THREADS]; - -static void *f_thread(void *unused) { - int i; - for (i = 0; i < N_PARAM; i++) { - void *p = malloc(100); - memset(p, 100, sizeof(char)); - free(p); - } - - return NULL; -} - -TEST_BEGIN(test_prof_log_many_threads) { - - test_skip_if(!config_prof); - - int i; - assert_d_eq(mallctl("prof.log_start", NULL, NULL, NULL, 0), 0, - "Unexpected mallctl failure when starting logging"); - for (i = 0; i < N_THREADS; i++) { - thd_create(&thr_buf[i], &f_thread, NULL); - } - - for (i = 0; i < N_THREADS; i++) { - thd_join(thr_buf[i], NULL); - } - assert_zu_eq(prof_log_thr_count(), N_THREADS, - "Wrong number of thread entries"); - assert_rep(); - assert_d_eq(mallctl("prof.log_stop", NULL, NULL, NULL, 0), 0, - "Unexpected mallctl failure when stopping logging"); -} -TEST_END - -static void f3() { - void *p = malloc(100); - free(p); -} - -static void f1() { - void *p = malloc(100); - f3(); - free(p); -} - -static void f2() { - void *p = malloc(100); - free(p); -} - -TEST_BEGIN(test_prof_log_many_traces) { - - test_skip_if(!config_prof); - - assert_d_eq(mallctl("prof.log_start", NULL, NULL, NULL, 0), 0, - "Unexpected mallctl failure when starting logging"); - int i; - assert_rep(); - assert_log_empty(); - for (i = 0; i < N_PARAM; i++) { - assert_rep(); - f1(); - assert_rep(); - f2(); - assert_rep(); - f3(); - assert_rep(); - } - /* - * There should be 8 total backtraces: two for malloc/free in f1(), two - * for malloc/free in f2(), two for malloc/free in f3(), and then two - * for malloc/free in f1()'s call to f3(). However compiler - * optimizations such as loop unrolling might generate more call sites. - * So >= 8 traces are expected. - */ - assert_zu_ge(prof_log_bt_count(), 8, - "Expect at least 8 backtraces given sample workload"); - assert_d_eq(mallctl("prof.log_stop", NULL, NULL, NULL, 0), 0, - "Unexpected mallctl failure when stopping logging"); -} -TEST_END - -int -main(void) { - prof_log_dummy_set(true); - return test_no_reentrancy( - test_prof_log_many_logs, - test_prof_log_many_traces, - test_prof_log_many_threads); -} diff --git a/deps/jemalloc/test/unit/prof_log.sh b/deps/jemalloc/test/unit/prof_log.sh deleted file mode 100644 index 8fcc7d8..0000000 --- a/deps/jemalloc/test/unit/prof_log.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -if [ "x${enable_prof}" = "x1" ] ; then - export MALLOC_CONF="prof:true,lg_prof_sample:0" -fi diff --git a/deps/jemalloc/test/unit/retained.c b/deps/jemalloc/test/unit/retained.c index 7993fd3..d51a598 100644 --- a/deps/jemalloc/test/unit/retained.c +++ b/deps/jemalloc/test/unit/retained.c @@ -107,9 +107,6 @@ TEST_BEGIN(test_retained) { atomic_store_u(&epoch, 0, ATOMIC_RELAXED); unsigned nthreads = ncpus * 2; - if (LG_SIZEOF_PTR < 3 && nthreads > 16) { - nthreads = 16; /* 32-bit platform could run out of vaddr. */ - } VARIABLE_ARRAY(thd_t, threads, nthreads); for (unsigned i = 0; i < nthreads; i++) { thd_create(&threads[i], thd_start, NULL); diff --git a/deps/jemalloc/test/unit/rtree.c b/deps/jemalloc/test/unit/rtree.c index 90adca1..908100f 100644 --- a/deps/jemalloc/test/unit/rtree.c +++ b/deps/jemalloc/test/unit/rtree.c @@ -85,11 +85,11 @@ TEST_END TEST_BEGIN(test_rtree_extrema) { extent_t extent_a, extent_b; - extent_init(&extent_a, NULL, NULL, SC_LARGE_MINCLASS, false, - sz_size2index(SC_LARGE_MINCLASS), 0, - extent_state_active, false, false, true, EXTENT_NOT_HEAD); - extent_init(&extent_b, NULL, NULL, 0, false, SC_NSIZES, 0, - extent_state_active, false, false, true, EXTENT_NOT_HEAD); + extent_init(&extent_a, NULL, NULL, LARGE_MINCLASS, false, + sz_size2index(LARGE_MINCLASS), 0, extent_state_active, false, + false, true); + extent_init(&extent_b, NULL, NULL, 0, false, NSIZES, 0, + extent_state_active, false, false, true); tsdn_t *tsdn = tsdn_fetch(); @@ -125,8 +125,8 @@ TEST_BEGIN(test_rtree_bits) { PAGE + (((uintptr_t)1) << LG_PAGE) - 1}; extent_t extent; - extent_init(&extent, NULL, NULL, 0, false, SC_NSIZES, 0, - extent_state_active, false, false, true, EXTENT_NOT_HEAD); + extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0, + extent_state_active, false, false, true); rtree_t *rtree = &test_rtree; rtree_ctx_t rtree_ctx; @@ -135,7 +135,7 @@ TEST_BEGIN(test_rtree_bits) { for (unsigned i = 0; i < sizeof(keys)/sizeof(uintptr_t); i++) { assert_false(rtree_write(tsdn, rtree, &rtree_ctx, keys[i], - &extent, SC_NSIZES, false), + &extent, NSIZES, false), "Unexpected rtree_write() failure"); for (unsigned j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) { assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, @@ -166,8 +166,8 @@ TEST_BEGIN(test_rtree_random) { rtree_ctx_data_init(&rtree_ctx); extent_t extent; - extent_init(&extent, NULL, NULL, 0, false, SC_NSIZES, 0, - extent_state_active, false, false, true, EXTENT_NOT_HEAD); + extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0, + extent_state_active, false, false, true); assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure"); @@ -177,8 +177,7 @@ TEST_BEGIN(test_rtree_random) { &rtree_ctx, keys[i], false, true); assert_ptr_not_null(elm, "Unexpected rtree_leaf_elm_lookup() failure"); - rtree_leaf_elm_write(tsdn, rtree, elm, &extent, SC_NSIZES, - false); + rtree_leaf_elm_write(tsdn, rtree, elm, &extent, NSIZES, false); assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, keys[i], true), &extent, "rtree_extent_read() should return previously set value"); diff --git a/deps/jemalloc/test/unit/safety_check.c b/deps/jemalloc/test/unit/safety_check.c deleted file mode 100644 index bf4bd86..0000000 --- a/deps/jemalloc/test/unit/safety_check.c +++ /dev/null @@ -1,156 +0,0 @@ -#include "test/jemalloc_test.h" - -#include "jemalloc/internal/safety_check.h" - -/* - * Note that we get called through safety_check.sh, which turns on sampling for - * everything. - */ - -bool fake_abort_called; -void fake_abort(const char *message) { - (void)message; - fake_abort_called = true; -} - -TEST_BEGIN(test_malloc_free_overflow) { - test_skip_if(!config_prof); - test_skip_if(!config_opt_safety_checks); - - safety_check_set_abort(&fake_abort); - /* Buffer overflow! */ - char* ptr = malloc(128); - ptr[128] = 0; - free(ptr); - safety_check_set_abort(NULL); - - assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); - fake_abort_called = false; -} -TEST_END - -TEST_BEGIN(test_mallocx_dallocx_overflow) { - test_skip_if(!config_prof); - test_skip_if(!config_opt_safety_checks); - - safety_check_set_abort(&fake_abort); - /* Buffer overflow! */ - char* ptr = mallocx(128, 0); - ptr[128] = 0; - dallocx(ptr, 0); - safety_check_set_abort(NULL); - - assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); - fake_abort_called = false; -} -TEST_END - -TEST_BEGIN(test_malloc_sdallocx_overflow) { - test_skip_if(!config_prof); - test_skip_if(!config_opt_safety_checks); - - safety_check_set_abort(&fake_abort); - /* Buffer overflow! */ - char* ptr = malloc(128); - ptr[128] = 0; - sdallocx(ptr, 128, 0); - safety_check_set_abort(NULL); - - assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); - fake_abort_called = false; -} -TEST_END - -TEST_BEGIN(test_realloc_overflow) { - test_skip_if(!config_prof); - test_skip_if(!config_opt_safety_checks); - - safety_check_set_abort(&fake_abort); - /* Buffer overflow! */ - char* ptr = malloc(128); - ptr[128] = 0; - ptr = realloc(ptr, 129); - safety_check_set_abort(NULL); - free(ptr); - - assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); - fake_abort_called = false; -} -TEST_END - -TEST_BEGIN(test_rallocx_overflow) { - test_skip_if(!config_prof); - test_skip_if(!config_opt_safety_checks); - - safety_check_set_abort(&fake_abort); - /* Buffer overflow! */ - char* ptr = malloc(128); - ptr[128] = 0; - ptr = rallocx(ptr, 129, 0); - safety_check_set_abort(NULL); - free(ptr); - - assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); - fake_abort_called = false; -} -TEST_END - -TEST_BEGIN(test_xallocx_overflow) { - test_skip_if(!config_prof); - test_skip_if(!config_opt_safety_checks); - - safety_check_set_abort(&fake_abort); - /* Buffer overflow! */ - char* ptr = malloc(128); - ptr[128] = 0; - size_t result = xallocx(ptr, 129, 0, 0); - assert_zu_eq(result, 128, ""); - free(ptr); - assert_b_eq(fake_abort_called, true, "Redzone check didn't fire."); - fake_abort_called = false; - safety_check_set_abort(NULL); -} -TEST_END - -TEST_BEGIN(test_realloc_no_overflow) { - char* ptr = malloc(128); - ptr = realloc(ptr, 256); - ptr[128] = 0; - ptr[255] = 0; - free(ptr); - - ptr = malloc(128); - ptr = realloc(ptr, 64); - ptr[63] = 0; - ptr[0] = 0; - free(ptr); -} -TEST_END - -TEST_BEGIN(test_rallocx_no_overflow) { - char* ptr = malloc(128); - ptr = rallocx(ptr, 256, 0); - ptr[128] = 0; - ptr[255] = 0; - free(ptr); - - ptr = malloc(128); - ptr = rallocx(ptr, 64, 0); - ptr[63] = 0; - ptr[0] = 0; - free(ptr); -} -TEST_END - -int -main(void) { - return test( - test_malloc_free_overflow, - test_mallocx_dallocx_overflow, - test_malloc_sdallocx_overflow, - test_realloc_overflow, - test_rallocx_overflow, - test_xallocx_overflow, - test_realloc_no_overflow, - test_rallocx_no_overflow); -} diff --git a/deps/jemalloc/test/unit/safety_check.sh b/deps/jemalloc/test/unit/safety_check.sh deleted file mode 100644 index 8fcc7d8..0000000 --- a/deps/jemalloc/test/unit/safety_check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -if [ "x${enable_prof}" = "x1" ] ; then - export MALLOC_CONF="prof:true,lg_prof_sample:0" -fi diff --git a/deps/jemalloc/test/unit/sc.c b/deps/jemalloc/test/unit/sc.c deleted file mode 100644 index bf51d8e..0000000 --- a/deps/jemalloc/test/unit/sc.c +++ /dev/null @@ -1,33 +0,0 @@ -#include "test/jemalloc_test.h" - -TEST_BEGIN(test_update_slab_size) { - sc_data_t data; - memset(&data, 0, sizeof(data)); - sc_data_init(&data); - sc_t *tiny = &data.sc[0]; - size_t tiny_size = (ZU(1) << tiny->lg_base) - + (ZU(tiny->ndelta) << tiny->lg_delta); - size_t pgs_too_big = (tiny_size * BITMAP_MAXBITS + PAGE - 1) / PAGE + 1; - sc_data_update_slab_size(&data, tiny_size, tiny_size, (int)pgs_too_big); - assert_zu_lt((size_t)tiny->pgs, pgs_too_big, "Allowed excessive pages"); - - sc_data_update_slab_size(&data, 1, 10 * PAGE, 1); - for (int i = 0; i < data.nbins; i++) { - sc_t *sc = &data.sc[i]; - size_t reg_size = (ZU(1) << sc->lg_base) - + (ZU(sc->ndelta) << sc->lg_delta); - if (reg_size <= PAGE) { - assert_d_eq(sc->pgs, 1, "Ignored valid page size hint"); - } else { - assert_d_gt(sc->pgs, 1, - "Allowed invalid page size hint"); - } - } -} -TEST_END - -int -main(void) { - return test( - test_update_slab_size); -} diff --git a/deps/jemalloc/test/unit/seq.c b/deps/jemalloc/test/unit/seq.c deleted file mode 100644 index 19613b0..0000000 --- a/deps/jemalloc/test/unit/seq.c +++ /dev/null @@ -1,95 +0,0 @@ -#include "test/jemalloc_test.h" - -#include "jemalloc/internal/seq.h" - -typedef struct data_s data_t; -struct data_s { - int arr[10]; -}; - -static void -set_data(data_t *data, int num) { - for (int i = 0; i < 10; i++) { - data->arr[i] = num; - } -} - -static void -assert_data(data_t *data) { - int num = data->arr[0]; - for (int i = 0; i < 10; i++) { - assert_d_eq(num, data->arr[i], "Data consistency error"); - } -} - -seq_define(data_t, data) - -typedef struct thd_data_s thd_data_t; -struct thd_data_s { - seq_data_t data; -}; - -static void * -seq_reader_thd(void *arg) { - thd_data_t *thd_data = (thd_data_t *)arg; - int iter = 0; - data_t local_data; - while (iter < 1000 * 1000 - 1) { - bool success = seq_try_load_data(&local_data, &thd_data->data); - if (success) { - assert_data(&local_data); - assert_d_le(iter, local_data.arr[0], - "Seq read went back in time."); - iter = local_data.arr[0]; - } - } - return NULL; -} - -static void * -seq_writer_thd(void *arg) { - thd_data_t *thd_data = (thd_data_t *)arg; - data_t local_data; - memset(&local_data, 0, sizeof(local_data)); - for (int i = 0; i < 1000 * 1000; i++) { - set_data(&local_data, i); - seq_store_data(&thd_data->data, &local_data); - } - return NULL; -} - -TEST_BEGIN(test_seq_threaded) { - thd_data_t thd_data; - memset(&thd_data, 0, sizeof(thd_data)); - - thd_t reader; - thd_t writer; - - thd_create(&reader, seq_reader_thd, &thd_data); - thd_create(&writer, seq_writer_thd, &thd_data); - - thd_join(reader, NULL); - thd_join(writer, NULL); -} -TEST_END - -TEST_BEGIN(test_seq_simple) { - data_t data; - seq_data_t seq; - memset(&seq, 0, sizeof(seq)); - for (int i = 0; i < 1000 * 1000; i++) { - set_data(&data, i); - seq_store_data(&seq, &data); - set_data(&data, 0); - bool success = seq_try_load_data(&data, &seq); - assert_b_eq(success, true, "Failed non-racing read"); - assert_data(&data); - } -} -TEST_END - -int main(void) { - return test_no_reentrancy( - test_seq_simple, - test_seq_threaded); -} diff --git a/deps/jemalloc/test/unit/size_classes.c b/deps/jemalloc/test/unit/size_classes.c index 6947336..bcff560 100644 --- a/deps/jemalloc/test/unit/size_classes.c +++ b/deps/jemalloc/test/unit/size_classes.c @@ -108,13 +108,8 @@ TEST_BEGIN(test_psize_classes) { size_class, sz_psz2ind(size_class), sz_pind2sz(sz_psz2ind(size_class))); - if (size_class == SC_LARGE_MAXCLASS) { - assert_u_eq(SC_NPSIZES, sz_psz2ind(size_class + 1), - "Next size_class does not round up properly"); - } else { - assert_u_eq(pind + 1, sz_psz2ind(size_class + 1), - "Next size_class does not round up properly"); - } + assert_u_eq(pind+1, sz_psz2ind(size_class+1), + "Next size_class does not round up properly"); assert_zu_eq(size_class, (pind > 0) ? sz_psz2u(sz_pind2sz(pind-1)+1) : sz_psz2u(1), @@ -147,11 +142,11 @@ TEST_BEGIN(test_overflow) { max_size_class = get_max_size_class(); max_psz = max_size_class + PAGE; - assert_u_eq(sz_size2index(max_size_class+1), SC_NSIZES, + assert_u_eq(sz_size2index(max_size_class+1), NSIZES, "sz_size2index() should return NSIZES on overflow"); - assert_u_eq(sz_size2index(ZU(PTRDIFF_MAX)+1), SC_NSIZES, + assert_u_eq(sz_size2index(ZU(PTRDIFF_MAX)+1), NSIZES, "sz_size2index() should return NSIZES on overflow"); - assert_u_eq(sz_size2index(SIZE_T_MAX), SC_NSIZES, + assert_u_eq(sz_size2index(SIZE_T_MAX), NSIZES, "sz_size2index() should return NSIZES on overflow"); assert_zu_eq(sz_s2u(max_size_class+1), 0, @@ -161,11 +156,11 @@ TEST_BEGIN(test_overflow) { assert_zu_eq(sz_s2u(SIZE_T_MAX), 0, "sz_s2u() should return 0 on overflow"); - assert_u_eq(sz_psz2ind(max_size_class+1), SC_NPSIZES, + assert_u_eq(sz_psz2ind(max_size_class+1), NPSIZES, "sz_psz2ind() should return NPSIZES on overflow"); - assert_u_eq(sz_psz2ind(ZU(PTRDIFF_MAX)+1), SC_NPSIZES, + assert_u_eq(sz_psz2ind(ZU(PTRDIFF_MAX)+1), NPSIZES, "sz_psz2ind() should return NPSIZES on overflow"); - assert_u_eq(sz_psz2ind(SIZE_T_MAX), SC_NPSIZES, + assert_u_eq(sz_psz2ind(SIZE_T_MAX), NPSIZES, "sz_psz2ind() should return NPSIZES on overflow"); assert_zu_eq(sz_psz2u(max_size_class+1), max_psz, diff --git a/deps/jemalloc/test/unit/slab.c b/deps/jemalloc/test/unit/slab.c index c56af25..7e662ae 100644 --- a/deps/jemalloc/test/unit/slab.c +++ b/deps/jemalloc/test/unit/slab.c @@ -3,14 +3,13 @@ TEST_BEGIN(test_arena_slab_regind) { szind_t binind; - for (binind = 0; binind < SC_NBINS; binind++) { + for (binind = 0; binind < NBINS; binind++) { size_t regind; extent_t slab; const bin_info_t *bin_info = &bin_infos[binind]; extent_init(&slab, NULL, mallocx(bin_info->slab_size, MALLOCX_LG_ALIGN(LG_PAGE)), bin_info->slab_size, true, - binind, 0, extent_state_active, false, true, true, - EXTENT_NOT_HEAD); + binind, 0, extent_state_active, false, true, true); assert_ptr_not_null(extent_addr_get(&slab), "Unexpected malloc() failure"); for (regind = 0; regind < bin_info->nregs; regind++) { diff --git a/deps/jemalloc/test/unit/stats.c b/deps/jemalloc/test/unit/stats.c index 646768e..231010e 100644 --- a/deps/jemalloc/test/unit/stats.c +++ b/deps/jemalloc/test/unit/stats.c @@ -33,7 +33,7 @@ TEST_BEGIN(test_stats_large) { size_t sz; int expected = config_stats ? 0 : ENOENT; - p = mallocx(SC_SMALL_MAXCLASS + 1, MALLOCX_ARENA(0)); + p = mallocx(SMALL_MAXCLASS+1, MALLOCX_ARENA(0)); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)), @@ -74,10 +74,9 @@ TEST_BEGIN(test_stats_arenas_summary) { uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; - little = mallocx(SC_SMALL_MAXCLASS, MALLOCX_ARENA(0)); + little = mallocx(SMALL_MAXCLASS, MALLOCX_ARENA(0)); assert_ptr_not_null(little, "Unexpected mallocx() failure"); - large = mallocx((1U << SC_LG_LARGE_MINCLASS), - MALLOCX_ARENA(0)); + large = mallocx((1U << LG_LARGE_MINCLASS), MALLOCX_ARENA(0)); assert_ptr_not_null(large, "Unexpected mallocx() failure"); dallocx(little, 0); @@ -149,7 +148,7 @@ TEST_BEGIN(test_stats_arenas_small) { no_lazy_lock(); /* Lazy locking would dodge tcache testing. */ - p = mallocx(SC_SMALL_MAXCLASS, MALLOCX_ARENA(0)); + p = mallocx(SMALL_MAXCLASS, MALLOCX_ARENA(0)); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), @@ -192,7 +191,7 @@ TEST_BEGIN(test_stats_arenas_large) { uint64_t epoch, nmalloc, ndalloc; int expected = config_stats ? 0 : ENOENT; - p = mallocx((1U << SC_LG_LARGE_MINCLASS), MALLOCX_ARENA(0)); + p = mallocx((1U << LG_LARGE_MINCLASS), MALLOCX_ARENA(0)); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)), @@ -228,7 +227,7 @@ gen_mallctl_str(char *cmd, char *name, unsigned arena_ind) { TEST_BEGIN(test_stats_arenas_bins) { void *p; - size_t sz, curslabs, curregs, nonfull_slabs; + size_t sz, curslabs, curregs; uint64_t epoch, nmalloc, ndalloc, nrequests, nfills, nflushes; uint64_t nslabs, nreslabs; int expected = config_stats ? 0 : ENOENT; @@ -289,9 +288,6 @@ TEST_BEGIN(test_stats_arenas_bins) { gen_mallctl_str(cmd, "curslabs", arena_ind); assert_d_eq(mallctl(cmd, (void *)&curslabs, &sz, NULL, 0), expected, "Unexpected mallctl() result"); - gen_mallctl_str(cmd, "nonfull_slabs", arena_ind); - assert_d_eq(mallctl(cmd, (void *)&nonfull_slabs, &sz, NULL, 0), - expected, "Unexpected mallctl() result"); if (config_stats) { assert_u64_gt(nmalloc, 0, @@ -312,8 +308,6 @@ TEST_BEGIN(test_stats_arenas_bins) { "At least one slab should have been allocated"); assert_zu_gt(curslabs, 0, "At least one slab should be currently allocated"); - assert_zu_eq(nonfull_slabs, 0, - "slabs_nonfull should be empty"); } dallocx(p, 0); diff --git a/deps/jemalloc/test/unit/tsd.c b/deps/jemalloc/test/unit/tsd.c index 917884d..6c47913 100644 --- a/deps/jemalloc/test/unit/tsd.c +++ b/deps/jemalloc/test/unit/tsd.c @@ -1,10 +1,5 @@ #include "test/jemalloc_test.h" -/* - * If we're e.g. in debug mode, we *never* enter the fast path, and so shouldn't - * be asserting that we're on one. - */ -static bool originally_fast; static int data_cleanup_count; void @@ -103,11 +98,11 @@ thd_start_reincarnated(void *arg) { tsd_cleanup((void *)tsd); assert_ptr_null(*tsd_arenap_get_unsafe(tsd), "TSD arena should have been cleared."); - assert_u_eq(tsd_state_get(tsd), tsd_state_purgatory, + assert_u_eq(tsd->state, tsd_state_purgatory, "TSD state should be purgatory\n"); free(p); - assert_u_eq(tsd_state_get(tsd), tsd_state_reincarnated, + assert_u_eq(tsd->state, tsd_state_reincarnated, "TSD state should be reincarnated\n"); p = mallocx(1, MALLOCX_TCACHE_NONE); assert_ptr_not_null(p, "Unexpected malloc() failure"); @@ -129,128 +124,6 @@ TEST_BEGIN(test_tsd_reincarnation) { } TEST_END -typedef struct { - atomic_u32_t phase; - atomic_b_t error; -} global_slow_data_t; - -static void * -thd_start_global_slow(void *arg) { - /* PHASE 0 */ - global_slow_data_t *data = (global_slow_data_t *)arg; - free(mallocx(1, 0)); - - tsd_t *tsd = tsd_fetch(); - /* - * No global slowness has happened yet; there was an error if we were - * originally fast but aren't now. - */ - atomic_store_b(&data->error, originally_fast && !tsd_fast(tsd), - ATOMIC_SEQ_CST); - atomic_store_u32(&data->phase, 1, ATOMIC_SEQ_CST); - - /* PHASE 2 */ - while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 2) { - } - free(mallocx(1, 0)); - atomic_store_b(&data->error, tsd_fast(tsd), ATOMIC_SEQ_CST); - atomic_store_u32(&data->phase, 3, ATOMIC_SEQ_CST); - - /* PHASE 4 */ - while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 4) { - } - free(mallocx(1, 0)); - atomic_store_b(&data->error, tsd_fast(tsd), ATOMIC_SEQ_CST); - atomic_store_u32(&data->phase, 5, ATOMIC_SEQ_CST); - - /* PHASE 6 */ - while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 6) { - } - free(mallocx(1, 0)); - /* Only one decrement so far. */ - atomic_store_b(&data->error, tsd_fast(tsd), ATOMIC_SEQ_CST); - atomic_store_u32(&data->phase, 7, ATOMIC_SEQ_CST); - - /* PHASE 8 */ - while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 8) { - } - free(mallocx(1, 0)); - /* - * Both decrements happened; we should be fast again (if we ever - * were) - */ - atomic_store_b(&data->error, originally_fast && !tsd_fast(tsd), - ATOMIC_SEQ_CST); - atomic_store_u32(&data->phase, 9, ATOMIC_SEQ_CST); - - return NULL; -} - -TEST_BEGIN(test_tsd_global_slow) { - global_slow_data_t data = {ATOMIC_INIT(0), ATOMIC_INIT(false)}; - /* - * Note that the "mallocx" here (vs. malloc) is important, since the - * compiler is allowed to optimize away free(malloc(1)) but not - * free(mallocx(1)). - */ - free(mallocx(1, 0)); - tsd_t *tsd = tsd_fetch(); - originally_fast = tsd_fast(tsd); - - thd_t thd; - thd_create(&thd, thd_start_global_slow, (void *)&data.phase); - /* PHASE 1 */ - while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 1) { - /* - * We don't have a portable condvar/semaphore mechanism. - * Spin-wait. - */ - } - assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); - tsd_global_slow_inc(tsd_tsdn(tsd)); - free(mallocx(1, 0)); - assert_false(tsd_fast(tsd), ""); - atomic_store_u32(&data.phase, 2, ATOMIC_SEQ_CST); - - /* PHASE 3 */ - while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 3) { - } - assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); - /* Increase again, so that we can test multiple fast/slow changes. */ - tsd_global_slow_inc(tsd_tsdn(tsd)); - atomic_store_u32(&data.phase, 4, ATOMIC_SEQ_CST); - free(mallocx(1, 0)); - assert_false(tsd_fast(tsd), ""); - - /* PHASE 5 */ - while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 5) { - } - assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); - tsd_global_slow_dec(tsd_tsdn(tsd)); - atomic_store_u32(&data.phase, 6, ATOMIC_SEQ_CST); - /* We only decreased once; things should still be slow. */ - free(mallocx(1, 0)); - assert_false(tsd_fast(tsd), ""); - - /* PHASE 7 */ - while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 7) { - } - assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); - tsd_global_slow_dec(tsd_tsdn(tsd)); - atomic_store_u32(&data.phase, 8, ATOMIC_SEQ_CST); - /* We incremented and then decremented twice; we should be fast now. */ - free(mallocx(1, 0)); - assert_true(!originally_fast || tsd_fast(tsd), ""); - - /* PHASE 9 */ - while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 9) { - } - assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); - - thd_join(thd, NULL); -} -TEST_END - int main(void) { /* Ensure tsd bootstrapped. */ @@ -262,6 +135,5 @@ main(void) { return test_no_reentrancy( test_tsd_main_thread, test_tsd_sub_thread, - test_tsd_reincarnation, - test_tsd_global_slow); + test_tsd_reincarnation); } diff --git a/deps/jemalloc/test/unit/zero.c b/deps/jemalloc/test/unit/zero.c index 271fd5c..553692b 100644 --- a/deps/jemalloc/test/unit/zero.c +++ b/deps/jemalloc/test/unit/zero.c @@ -41,13 +41,13 @@ test_zero(size_t sz_min, size_t sz_max) { TEST_BEGIN(test_zero_small) { test_skip_if(!config_fill); - test_zero(1, SC_SMALL_MAXCLASS - 1); + test_zero(1, SMALL_MAXCLASS-1); } TEST_END TEST_BEGIN(test_zero_large) { test_skip_if(!config_fill); - test_zero(SC_SMALL_MAXCLASS + 1, 1U << (SC_LG_LARGE_MINCLASS + 1)); + test_zero(SMALL_MAXCLASS+1, (1U << (LG_LARGE_MINCLASS+1))); } TEST_END