Do Transparent Huge Pages Help on a VPS? Random Access Over 1 GiB

Results

Recorded September 20, 2026 at 2:02 AM UTC, wall clock 125.5s.

Method

One anonymous 1 GiB mapping, aligned to 2 MiB, is created twice: once with madvise(MADV_NOHUGEPAGE) so the kernel backs it with 4 KiB pages, once with madvise(MADV_HUGEPAGE) so it is backed by 2 MiB pages where the kernel can find contiguous physical memory. Both buffers are filled with the same pseudo-random words, then walked with the same dependent chain: each index is derived from the value just loaded, so no two loads overlap and the figure is the latency of a single access, including any TLB miss and page walk. 50,000,000 accesses per run, both cases warmed up at one tenth of the workload, then five alternating runs each, reported by the fastest. The program reads AnonHugePages from /proc/self/smaps_rollup before and after populating each buffer and reports what fraction of the huge-page case was actually backed by 2 MiB pages; on a fragmented machine that fraction is small and the ratio has to be read together with it. A control mode (./bench 1024 50000000 control) allocates both buffers with 4 KiB pages and reports the same ratio, which is the noise floor of the method: measured at about 1.05 on this machine, so only a ratio outside roughly 0.95 to 1.05 means anything.

Machine

This is a shared virtual server, not an isolated test rig. Absolute throughput will differ on your hardware; the ratio between the two cases is the part that carries over.

Source

The complete program that produced the numbers above. Nothing else was running under our control during the measurement.

/*
 * Measures what 2 MiB pages are worth for random access over a large buffer.
 *
 * The same 1 GiB anonymous mapping is walked twice with the same dependent
 * random chain: once with transparent huge pages refused (MADV_NOHUGEPAGE,
 * so the buffer is backed by 4 KiB pages) and once with them requested
 * (MADV_HUGEPAGE). 1 GiB is 262,144 four-kilobyte pages, far more than any
 * TLB holds, so almost every access in the first case pays a page walk;
 * the same buffer is 512 two-megabyte pages, which fits in a modern L2 TLB.
 *
 * The chain is dependent (each index derives from the value just loaded), so
 * the CPU cannot overlap the misses and the result is the latency of one
 * access, not the throughput of many. The kernel is asked, not trusted: after
 * touching the buffer the program reads AnonHugePages from
 * /proc/self/smaps_rollup and reports how much of the mapping actually got
 * huge pages, so a run on a machine where THP is disabled shows up as such.
 *
 * Build: gcc -O2 -o bench bench.c
 * Run:   ./bench <buffer_mib> <accesses>
 * Output: one JSON object on stdout.
 */
#define _GNU_SOURCE
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>

#define HUGE_ALIGN (2UL * 1024 * 1024)

static double now_seconds(void) {
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC, &ts);
    return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9;
}

/* Bytes of this process's anonymous memory currently backed by huge pages. */
static long anon_huge_pages_kb(void) {
    FILE *f = fopen("/proc/self/smaps_rollup", "r");
    if (!f) return -1;
    char line[256];
    long kb = -1;
    while (fgets(line, sizeof line, f)) {
        if (strncmp(line, "AnonHugePages:", 14) == 0) {
            kb = strtol(line + 14, NULL, 10);
            break;
        }
    }
    fclose(f);
    return kb;
}

typedef struct {
    uint64_t *buf;
    size_t words;
    long huge_kb;
} mapping_t;

static mapping_t map_buffer(size_t bytes, int want_huge) {
    /* Over-allocate so the usable region can be aligned to a 2 MiB boundary;
     * an unaligned mapping cannot be backed by huge pages at all. */
    size_t span = bytes + HUGE_ALIGN;
    void *raw = mmap(NULL, span, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
    if (raw == MAP_FAILED) {
        fprintf(stderr, "mmap failed\n");
        exit(1);
    }
    uintptr_t aligned = ((uintptr_t)raw + HUGE_ALIGN - 1) & ~(HUGE_ALIGN - 1);
    void *buf = (void *)aligned;

    if (madvise(buf, bytes, want_huge ? MADV_HUGEPAGE : MADV_NOHUGEPAGE) != 0) {
        fprintf(stderr, "madvise failed\n");
        exit(1);
    }

    long before = anon_huge_pages_kb();

    /* Populate: every word holds a pseudo-random offset so the chain below
     * has somewhere to go. Touching the memory is also what makes the kernel
     * actually allocate the pages, huge or not. */
    size_t words = bytes / sizeof(uint64_t);
    uint64_t *w = (uint64_t *)buf;
    uint64_t x = 0x9E3779B97F4A7C15ULL;
    for (size_t i = 0; i < words; i++) {
        x ^= x << 13; x ^= x >> 7; x ^= x << 17;
        w[i] = x;
    }

    long after = anon_huge_pages_kb();
    mapping_t m = { w, words, (before >= 0 && after >= 0) ? after - before : -1 };
    return m;
}

static void unmap_buffer(mapping_t m, size_t bytes) {
    /* Unmap by the aligned pointer; the slack before it is leaked on purpose,
     * it is at most 2 MiB and the process exits right after. */
    munmap(m.buf, bytes);
}

/* Dependent random walk: the next index comes from the value just read, so
 * each load must finish before the next can start. Returns ns per access. */
static double walk(mapping_t m, uint64_t accesses, uint64_t *sink) {
    size_t mask = m.words - 1; /* words is a power of two */
    size_t idx = 12345;
    uint64_t acc = 0;
    double start = now_seconds();
    for (uint64_t i = 0; i < accesses; i++) {
        uint64_t v = m.buf[idx];
        acc += v;
        idx = (size_t)(v ^ (i * 0x9E3779B97F4A7C15ULL)) & mask;
    }
    double elapsed = now_seconds() - start;
    *sink += acc;
    return elapsed * 1e9 / (double)accesses;
}

int main(int argc, char **argv) {
    size_t mib = argc > 1 ? strtoull(argv[1], NULL, 10) : 1024;
    uint64_t accesses = argc > 2 ? strtoull(argv[2], NULL, 10) : 50000000ULL;
    size_t bytes = mib * 1024 * 1024;

    if ((bytes & (bytes - 1)) != 0) {
        fprintf(stderr, "buffer size must be a power of two MiB\n");
        return 1;
    }

    uint64_t sink = 0;

    /* "control": both buffers refuse huge pages. The ratio then shows the
     * noise floor of the method itself (allocation order, physical placement),
     * which is what any real difference has to be compared against. */
    int control = argc > 3 && strcmp(argv[3], "control") == 0;

    mapping_t small = map_buffer(bytes, 0);
    mapping_t huge = map_buffer(bytes, control ? 0 : 1);

    /* Warm up both so neither pays for frequency ramp-up the other avoids. */
    walk(small, accesses / 10, &sink);
    walk(huge, accesses / 10, &sink);

    const int repeats = 5;
    double small_best = 1e18, huge_best = 1e18, small_sum = 0, huge_sum = 0;
    for (int r = 0; r < repeats; r++) {
        double s = walk(small, accesses, &sink);
        double h = walk(huge, accesses, &sink);
        if (s < small_best) small_best = s;
        if (h < huge_best) huge_best = h;
        small_sum += s;
        huge_sum += h;
    }

    double huge_fraction = huge.huge_kb >= 0 ? (double)huge.huge_kb * 1024.0 / (double)bytes : -1;

    printf("{\n");
    printf("  \"mode\": \"%s\",\n", control ? "control" : "measure");
    printf("  \"buffer_mib\": %zu,\n", mib);
    printf("  \"accesses\": %llu,\n", (unsigned long long)accesses);
    printf("  \"repeats\": %d,\n", repeats);
    printf("  \"pages_4k_in_buffer\": %zu,\n", bytes / 4096);
    printf("  \"pages_2m_in_buffer\": %zu,\n", bytes / HUGE_ALIGN);
    printf("  \"small_pages_ns_per_access_best\": %.2f,\n", small_best);
    printf("  \"small_pages_ns_per_access_mean\": %.2f,\n", small_sum / repeats);
    printf("  \"huge_pages_ns_per_access_best\": %.2f,\n", huge_best);
    printf("  \"huge_pages_ns_per_access_mean\": %.2f,\n", huge_sum / repeats);
    printf("  \"small_case_anon_huge_pages_kb\": %ld,\n", small.huge_kb);
    printf("  \"huge_case_anon_huge_pages_kb\": %ld,\n", huge.huge_kb);
    printf("  \"huge_case_fraction_backed_by_2m_pages\": %.3f,\n", huge_fraction);
    printf("  \"huge_page_speedup_factor\": %.2f,\n", small_best / huge_best);
    printf("  \"checksum\": %llu\n", (unsigned long long)(sink & 0xffff));
    printf("}\n");

    unmap_buffer(small, bytes);
    unmap_buffer(huge, bytes);
    return 0;
}

Previous runs