Re: [External] : Re: [PATCH 0/1] fsnotify: Check parent watches without child dentry flags

[Partha Satapathy <partha.satapathy@oracle.com>]

On 12-10-2026 14:10, Jan Kara wrote:
> On Fri 09-10-26 17:18:17, Partha Satapathy wrote:
>> Hi Honza,
>> 
>> Thanks for suggesting this workload.
>> The setup and before/after timing comparisons are below.
>> 
>> Architecture:                x86_64
>> CPU(s):                      512
>>   On-line CPU(s) list:       0-511
>> 
>> NUMA:
>>   NUMA node(s):              2
>>   NUMA node0 CPU(s):         0-127,256-383
>>   NUMA node1 CPU(s):         128-255,384-511
>> 
>> Testcase :
>> Using /dev/shm  tmpfs to create  private test directory.
>> Create 1,024 files each fully populated with 4kb.
>> Add IN_MODIFY inotify watch to the directory.
>> Start a listener thread to read notifications and count queue overflows.
>> Start X(1,2,4..1024) client threads; each opens a different file once.
>> Wait until every client is ready, then release them together.
>> Each client starts its timer and performs 1000000
>> one-byte pwrite() at offset0 on its own file.
>> Each client stops its timer and closes its file.
>> Wait for all clients to finish, drain remaining notifications,
>> and stop the listener.
>> Print write times, throughput, notification count, and queue-overflow count.
>> Close inotify and delete the test files and directory.
>> 
>> Note : File creation, open, close, and cleanup are outside the timed
>> write loops.
>> 
>> Run 1:
>> -------
>> 
>> | Clients | Before average time | After average time | Time change |
>> |---:|---:|---:|---:|
>> | 1 | 1.392 s | 1.484 s | +6.6% |
>> | 2 | 2.205 s | 2.402 s | +8.9% |
>> | 4 | 3.969 s | 3.644 s | −8.2% |
>> | 8 | 5.851 s | 5.804 s | −0.8% |
>> | 16 | 10.607 s | 11.538 s | +8.8% |
>> | 32 | 21.093 s | 22.492 s | +6.6% |
>> | 64 | 48.698 s | 49.576 s | +1.8% |
>> | 128 | 100.208 s | 100.467 s | +0.3% |
>> | 256 | 198.773 s | 197.669 s | −0.6% |
>> | 512 | 400.056 s | 399.550 s | −0.1% |
>> 
>> 
>> Run2 :
>> --------
>> 
>> | Clients | Before average time | After average time | Time change |
>> |---:|---:|---:|---:|
>> | 1 | 1.491 s | 1.484 s | −0.4% |
>> | 2 | 2.391 s | 2.402 s | +0.5% |
>> | 4 | 4.050 s | 3.644 s | −10.0% |
>> | 8 | 6.630 s | 5.804 s | −12.5% |
>> | 16 | 11.931 s | 11.538 s | −3.3% |
>> | 32 | 22.513 s | 22.492 s | −0.1% |
>> | 64 | 50.231 s | 49.576 s | −1.3% |
>> | 128 | 100.699 s | 100.467 s | −0.2% |
>> | 256 | 198.771 s | 197.669 s | −0.6% |
>> | 512 | 401.860 s | 399.550 s | −0.6% |
>> | 1,024 | 803.812 s | 806.593 s | +0.3% |
>> 
>> 
>> Positive means After Fix took longer; negative means faster.
>> 
>> 
>> The measured write performance is broadly comparable before and after
>> the fix, with no consistent substantial regression or progressively
>> worsening slowdown as concurrency increases.
>> 
>> The differences vary between baseline batches: the increases at 16
>> and 32 clients in the first comparison disappear or reverse in the
>> second. At 128–1024 clients, average write times differ by approximately
>> 0.6% or less. These shifts are consistent with run-to-run variation;
>> the results do not show a persistent substantial slowdown from the fix.
> 
> Thanks for running these tests. Results are very interesting. Can you
> please share your benchmarking program with me?  I'll check it and try to
> play with it on my end. Thanks!
> 
> 								Honza
> -- 
> Jan Kara <jack@suse.com>
> SUSE Labs, CR
> 

$ cat run_parent_ref_scale.sh
#!/usr/bin/env bash
set -u -o pipefail

bench=./fsnotify_parent_ref_bench
if [[ ! -x "$bench" ]]; then
        printf 'Build %s in the current directory first.\n' "$bench" >&2
        exit 1
fi

results_dir=$(mktemp -d ./parent-ref-results-XXXXXXXX) || exit 1
printf
'kernel=%s\nbenchmark=%s\ntmpfs=/dev/shm\nwrites_per_client=1000000\nmode=watch\n'
\
        "$(uname -r)" "$bench" > "$results_dir/run-info.txt"
printf 'Results: %s\n' "$results_dir"

for i in 1 2 3 4 5; do
for clients in 1 2 4 8 16 32 64 128 256 512 1024; do
        log="$results_dir/clients-${clients}.log"
        printf '\n=== clients=%d ===\n' "$clients"
        "$bench" /dev/shm "$clients" 1000000 watch 2>&1 | tee "$log"
        status=${PIPESTATUS[0]}
        printf 'exit_status=%d\n' "$status" | tee -a "$log"
done
done

printf '\nAll results saved in %s\n' "$results_dir"



$ cat fsnotify_parent_ref_bench.c
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <limits.h>
#include <poll.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/inotify.h>
#include <sys/stat.h>
#include <sys/statfs.h>
#include <time.h>
#include <unistd.h>
#include <linux/magic.h>

#define FILES 1024
#define FILE_SIZE 4096
#define DEFAULT_WRITES 1000000UL

struct gate {
        pthread_mutex_t mutex;
        pthread_cond_t cond;
        unsigned int ready;
        bool start;
        bool abort;
};

struct worker {
        char path[PATH_MAX];
        unsigned long writes;
        unsigned int id;
        struct gate *gate;
        uint64_t begin_ns;
        uint64_t end_ns;
        int error;
};

struct listener {
        int fd;
        atomic_bool stop;
        uint64_t events;
        uint64_t overflows;
        int error;
};

static uint64_t now_ns(void)
{
        struct timespec ts;

        if (clock_gettime(CLOCK_MONOTONIC_RAW, &ts)) {
                perror("clock_gettime");
                exit(EXIT_FAILURE);
        }
        return (uint64_t)ts.tv_sec * 1000000000ULL + ts.tv_nsec;
}

static unsigned long parse_count(const char *s, const char *name)
{
        char *end;
        unsigned long value;

        errno = 0;
        value = strtoul(s, &end, 10);
        if (errno || !*s || *end || !value) {
                fprintf(stderr, "Invalid %s: %s\n", name, s);
                exit(EXIT_FAILURE);
        }
        return value;
}

static void file_path(char *out, size_t size, const char *dir, unsigned
int i)
{
        int n = snprintf(out, size, "%s/file-%04u", dir, i);

        if (n < 0 || (size_t)n >= size) {
                fputs("File path is too long\n", stderr);
                exit(EXIT_FAILURE);
        }
}

static void *run_worker(void *arg)
{
        struct worker *w = arg;
        int fd;
        char byte = (char)w->id;
        unsigned long i;
        bool abort;

        fd = open(w->path, O_WRONLY | O_CLOEXEC);
        if (fd < 0)
                w->error = errno;
        pthread_mutex_lock(&w->gate->mutex);
        w->gate->ready++;
        pthread_cond_broadcast(&w->gate->cond);
        while (!w->gate->start)
                pthread_cond_wait(&w->gate->cond, &w->gate->mutex);
        abort = w->gate->abort;
        pthread_mutex_unlock(&w->gate->mutex);
        if (abort || w->error)
                goto out;

        w->begin_ns = now_ns();
        for (i = 0; i < w->writes; i++) {
                if (pwrite(fd, &byte, 1, 0) != 1) {
                        w->error = errno ? errno : EIO;
                        break;
                }
        }
        w->end_ns = now_ns();
out:
        if (fd >= 0 && close(fd) && !w->error)
                w->error = errno;
        return NULL;
}

static void *drain_events(void *arg)
{
        struct listener *l = arg;
        char buf[65536] __attribute__((aligned(8)));
        struct pollfd pfd = { .fd = l->fd, .events = POLLIN };

        for (;;) {
                int ready = poll(&pfd, 1, atomic_load(&l->stop) ? 0 : 50);

                if (ready < 0) {
                        if (errno == EINTR)
                                continue;
                        l->error = errno;
                        break;
                }
                if (!ready) {
                        if (atomic_load(&l->stop))
                                break;
                        continue;
                }
                for (;;) {
                        ssize_t len = read(l->fd, buf, sizeof(buf));
                        size_t pos = 0;

                        if (len < 0) {
                                if (errno == EINTR)
                                        continue;
                                if (errno == EAGAIN)
                                        break;
                                l->error = errno;
                                return NULL;
                        }
                        while (pos + sizeof(struct inotify_event) <=
(size_t)len) {
                                struct inotify_event *ev =
                                        (struct inotify_event *)(buf + pos);
                                size_t step = sizeof(*ev) + ev->len;

                                if (step > (size_t)len - pos) {
                                        l->error = EPROTO;
                                        return NULL;
                                }
                                l->events++;
                                if (ev->mask & IN_Q_OVERFLOW)
                                        l->overflows++;
                                pos += step;
                        }
                        if (pos != (size_t)len) {
                                l->error = EPROTO;
                                return NULL;
                        }
                }
        }
        return NULL;
}

static void cleanup(const char *dir)
{
        char path[PATH_MAX];
        unsigned int i;

        for (i = 0; i < FILES; i++) {
                file_path(path, sizeof(path), dir, i);
                if (unlink(path) && errno != ENOENT)
                        perror(path);
        }
        if (rmdir(dir))
                perror(dir);
}

int main(int argc, char **argv)
{
        char dir[PATH_MAX], path[PATH_MAX];
        unsigned long writes = DEFAULT_WRITES;
        unsigned long count;
        unsigned int clients, i, created = 0;
        struct statfs fs;
        struct gate gate = {
                .mutex = PTHREAD_MUTEX_INITIALIZER,
                .cond = PTHREAD_COND_INITIALIZER,
        };
        struct listener listener = { .fd = -1 };
        struct worker *workers;
        pthread_t *threads, listener_thread;
        bool watched = true, listener_started = false;
        uint64_t first = UINT64_MAX, last = 0, sum = 0;
        uint64_t shortest = UINT64_MAX, longest = 0;
        char zeroes[FILE_SIZE] = { 0 };
        int rc = EXIT_FAILURE, err;

        if (argc < 3 || argc > 5) {
                fprintf(stderr, "Usage: %s TMPFS_ROOT CLIENTS
[WRITES_PER_CLIENT] [watch|none]\n", argv[0]);
                return rc;
        }
        clients = parse_count(argv[2], "clients");
        if (clients > FILES) {
                fprintf(stderr, "Clients must be at most %d\n", FILES);
                return rc;
        }
        if (argc > 3)
                writes = parse_count(argv[3], "writes per client");
        if (writes > ULONG_MAX / clients) {
                fputs("Total write count is too large\n", stderr);
                return rc;
        }
        if (argc > 4) {
                if (!strcmp(argv[4], "none"))
                        watched = false;
                else if (strcmp(argv[4], "watch")) {
                        fputs("Mode must be watch or none\n", stderr);
                        return rc;
                }
        }
        if (statfs(argv[1], &fs)) {
                perror(argv[1]);
                return rc;
        }
        if (fs.f_type != TMPFS_MAGIC) {
                fprintf(stderr, "%s must be a tmpfs directory\n", argv[1]);
                return rc;
        }
        if (snprintf(dir, sizeof(dir), "%s/fsnotify-parent-ref-XXXXXX",
argv[1]) >=
            (int)sizeof(dir) || !mkdtemp(dir)) {
                perror("mkdtemp");
                return rc;
        }
        workers = calloc(clients, sizeof(*workers));
        threads = calloc(clients, sizeof(*threads));
        if (!workers || !threads) {
                fputs("Cannot allocate worker state\n", stderr);
                goto out;
        }
        for (i = 0; i < FILES; i++) {
                int fd;

                file_path(path, sizeof(path), dir, i);
                fd = open(path, O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC,
0600);
                if (fd < 0) {
                        perror(path);
                        goto out;
                }
                /* Allocate the tmpfs page before the measured write
loop. */
                if (write(fd, zeroes, sizeof(zeroes)) != sizeof(zeroes)) {
                        perror(path);
                        close(fd);
                        goto out;
                }
                if (close(fd)) {
                        perror(path);
                        goto out;
                }
        }
        if (watched) {
                listener.fd = inotify_init1(IN_NONBLOCK | IN_CLOEXEC);
                if (listener.fd < 0 ||
                    inotify_add_watch(listener.fd, dir, IN_MODIFY) < 0) {
                        perror("inotify watch");
                        goto out;
                }
                err = pthread_create(&listener_thread, NULL,
drain_events, &listener);
                if (err) {
                        errno = err;
                        perror("listener thread");
                        goto out;
                }
                listener_started = true;
        }
        for (i = 0; i < clients; i++) {
                workers[i].id = i;
                workers[i].writes = writes;
                workers[i].gate = &gate;
                file_path(workers[i].path, sizeof(workers[i].path), dir, i);
                err = pthread_create(&threads[i], NULL, run_worker,
&workers[i]);
                if (err) {
                        errno = err;
                        perror("worker thread");
                        gate.abort = true;
                        goto release;
                }
                created++;
        }
        pthread_mutex_lock(&gate.mutex);
        while (gate.ready < clients)
                pthread_cond_wait(&gate.cond, &gate.mutex);
        pthread_mutex_unlock(&gate.mutex);
        for (i = 0; i < clients; i++) {
                if (workers[i].error) {
                        fprintf(stderr, "Worker %u open: %s\n", i,
                                strerror(workers[i].error));
                        gate.abort = true;
                        break;
                }
        }
        printf("test_dir=%s files=%d file_size=%d clients=%u
writes_per_client=%lu mode=%s\n",
               dir, FILES, FILE_SIZE, clients, writes,
               watched ? "watch" : "none");
        fflush(stdout);
release:
        pthread_mutex_lock(&gate.mutex);
        gate.start = true;
        pthread_cond_broadcast(&gate.cond);
        pthread_mutex_unlock(&gate.mutex);
        for (i = 0; i < created; i++)
                pthread_join(threads[i], NULL);
        if (listener_started) {
                atomic_store(&listener.stop, true);
                pthread_join(listener_thread, NULL);
                printf("inotify_events=%" PRIu64 " queue_overflows=%"
PRIu64 "\n",
                       listener.events, listener.overflows);
                if (listener.error)
                        fprintf(stderr, "Inotify listener: %s\n",
strerror(listener.error));
                if (listener.overflows)
                        fputs("Inotify queue overflowed; event delivery
load changed during the run\n", stderr);
        }
        if (gate.abort || listener.error)
                goto out;
        for (i = 0; i < clients; i++) {
                uint64_t duration;

                if (workers[i].error) {
                        fprintf(stderr, "Worker %u write: %s\n", i,
                                strerror(workers[i].error));
                        goto out;
                }
                duration = workers[i].end_ns - workers[i].begin_ns;
                if (workers[i].begin_ns < first)
                        first = workers[i].begin_ns;
                if (workers[i].end_ns > last)
                        last = workers[i].end_ns;
                if (duration < shortest)
                        shortest = duration;
                if (duration > longest)
                        longest = duration;
                sum += duration;
        }
        count = (unsigned long)clients * writes;
        printf("total_writes=%lu wall_seconds=%.6f
throughput_writes_per_sec=%.0f mean_worker_ns_per_write=%.1f
shortest_worker_seconds=%.6f longest_worker_seconds=%.6f\n",
               count, (last - first) / 1e9, count * 1e9 / (last - first),
               (double)sum / count, shortest / 1e9, longest / 1e9);
        printf("result=%s\n", listener.overflows ? "QUEUE_OVERFLOW" :
"PASS");
        rc = listener.overflows ? EXIT_FAILURE : EXIT_SUCCESS;
out:
        if (listener.fd >= 0)
                close(listener.fd);
        cleanup(dir);
        free(workers);
        free(threads);
        return rc;
}


Please compile : fsnotify_parent_ref_bench.c -o
fsnotify_parent_ref_bench. After that
run the script ./run_parent_ref_scale.sh

Thanks
Partha