FYI, we noticed the following commit (built with gcc-7): commit: cd17d77705780e2270937fb3cbd2b985adab3edc ("bpf/tools: sync bpf.h") https://git.kernel.org/cgit/linux/kernel/git/next/linux-next.git master in testcase: kernel_selftests with following parameters: group: kselftests-00 test-description: The kernel contains a set of "self tests" under the tools/testing/selftests/ directory. These are intended to be small unit tests to exercise individual code paths in the kernel. test-url: https://www.kernel.org/doc/Documentation/kselftest.txt on test machine: qemu-system-x86_64 -enable-kvm -cpu SandyBridge -smp 2 -m 8G caused below changes (please refer to attached dmesg/kmsg for entire log/backtrace): If you fix the issue, kindly add following tag Reported-by: kernel test robot # selftests: bpf: test_sock_addr.sh # Wait for testing IPv4/IPv6 to become available ... OK # libbpf: load bpf program failed: Permission denied # libbpf: -- BEGIN DUMP LOG --- # libbpf: # ; int connect_v4_prog(struct bpf_sock_addr *ctx) # 0: (bf) r6 = r1 # 1: (b7) r1 = 1544617984 # ; memset(&tuple.ipv4.sport, 0, sizeof(tuple.ipv4.sport)); # 2: (7b) *(u64 *)(r10 -32) = r1 # 3: (18) r1 = 0x100007f00000000 # ; memset(&tuple.ipv4.saddr, 0, sizeof(tuple.ipv4.saddr)); # 5: (7b) *(u64 *)(r10 -40) = r1 # 6: (b7) r7 = 0 # ; struct bpf_sock_tuple tuple = {}; # 7: (63) *(u32 *)(r10 -8) = r7 # 8: (63) *(u32 *)(r10 -12) = r7 # 9: (63) *(u32 *)(r10 -16) = r7 # 10: (63) *(u32 *)(r10 -20) = r7 # 11: (63) *(u32 *)(r10 -24) = r7 # ; if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) # 12: (61) r1 = *(u32 *)(r6 +32) # ; if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) # 13: (bf) r2 = r1 # 14: (07) r2 += -1 # 15: (67) r2 <<= 32 # 16: (77) r2 >>= 32 # 17: (25) if r2 > 0x1 goto pc+24 # R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=1,var_off=(0x0; 0x1)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????0000 fp-16=00000000 fp-24=00000000 fp-32=mmmmmmmm fp-40=mmmmmmmm # ; else if (ctx->type == SOCK_STREAM) # 18: (55) if r1 != 0x1 goto pc+8 # R1=inv1 R2=inv(id=0,umax_value=1,var_off=(0x0; 0x1)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????0000 fp-16=00000000 fp-24=00000000 fp-32=mmmmmmmm fp-40=mmmmmmmm # 19: (bf) r2 = r10 # ; sk = bpf_sk_lookup_tcp(ctx, &tuple, sizeof(tuple.ipv4), # 20: (07) r2 += -40 # 21: (bf) r1 = r6 # 22: (b7) r3 = 12 # 23: (b7) r4 = -1 # 24: (b7) r5 = 0 # 25: (85) call bpf_sk_lookup_tcp#84 # 26: (05) goto pc+7 # ; if (!sk) # 34: (15) if r0 == 0x0 goto pc+7 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????0000 fp-16=00000000 fp-24=00000000 fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; if (sk->src_ip4 != tuple.ipv4.daddr || # 35: (61) r1 = *(u32 *)(r0 +24) # ; if (sk->src_ip4 != tuple.ipv4.daddr || # 36: (61) r2 = *(u32 *)(r10 -36) # ; if (sk->src_ip4 != tuple.ipv4.daddr || # 37: (5d) if r1 != r2 goto pc+2 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????0000 fp-16=00000000 fp-24=00000000 fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_port != DST_REWRITE_PORT4) { # 38: (61) r1 = *(u32 *)(r0 +44) # ; if (sk->src_ip4 != tuple.ipv4.daddr || # 39: (15) if r1 == 0x115c goto pc+4 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????0000 fp-16=00000000 fp-24=00000000 fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; bpf_sk_release(sk); # 40: (bf) r1 = r0 # 41: (85) call bpf_sk_release#86 # ; } # 42: (bf) r0 = r7 # 43: (95) exit # # from 39 to 44: R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv4444 R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????0000 fp-16=00000000 fp-24=00000000 fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; bpf_sk_release(sk); # 44: (bf) r1 = r0 # 45: (85) call bpf_sk_release#86 # 46: (b7) r1 = 23569 # ; ctx->user_port = bpf_htons(DST_REWRITE_PORT4); # 47: (63) *(u32 *)(r6 +24) = r1 # 48: (b7) r1 = 16777343 # ; ctx->user_ip4 = bpf_htonl(DST_REWRITE_IP4); # 49: (63) *(u32 *)(r6 +4) = r1 # invalid bpf_context access off=4 size=4 # processed 42 insns (limit 1000000) max_states_per_insn 0 total_states 10 peak_states 10 mark_read 8 # # libbpf: -- END LOG -- # libbpf: failed to load program 'cgroup/connect4' # libbpf: failed to load object './connect4_prog.o' # libbpf: load bpf program failed: Permission denied # libbpf: -- BEGIN DUMP LOG --- # libbpf: # ; int connect_v6_prog(struct bpf_sock_addr *ctx) # 0: (bf) r6 = r1 # 1: (18) r1 = 0x100000000000000 # ; tuple.ipv6.daddr[0] = bpf_htonl(DST_REWRITE_IP6_0); # 3: (7b) *(u64 *)(r10 -16) = r1 # 4: (b7) r1 = 169476096 # ; memset(&tuple.ipv6.sport, 0, sizeof(tuple.ipv6.sport)); # 5: (63) *(u32 *)(r10 -8) = r1 # 6: (b7) r7 = 0 # ; tuple.ipv6.daddr[0] = bpf_htonl(DST_REWRITE_IP6_0); # 7: (7b) *(u64 *)(r10 -24) = r7 # 8: (7b) *(u64 *)(r10 -32) = r7 # 9: (7b) *(u64 *)(r10 -40) = r7 # ; if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) # 10: (61) r1 = *(u32 *)(r6 +32) # ; if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) # 11: (bf) r2 = r1 # 12: (07) r2 += -1 # 13: (67) r2 <<= 32 # 14: (77) r2 >>= 32 # 15: (25) if r2 > 0x1 goto pc+33 # R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=1,var_off=(0x0; 0x1)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=00000000 fp-32=00000000 fp-40=00000000 # ; else if (ctx->type == SOCK_STREAM) # 16: (55) if r1 != 0x1 goto pc+8 # R1=inv1 R2=inv(id=0,umax_value=1,var_off=(0x0; 0x1)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=00000000 fp-32=00000000 fp-40=00000000 # 17: (bf) r2 = r10 # ; sk = bpf_sk_lookup_tcp(ctx, &tuple, sizeof(tuple.ipv6), # 18: (07) r2 += -40 # 19: (bf) r1 = r6 # 20: (b7) r3 = 36 # 21: (b7) r4 = -1 # 22: (b7) r5 = 0 # 23: (85) call bpf_sk_lookup_tcp#84 # 24: (05) goto pc+7 # ; if (!sk) # 32: (15) if r0 == 0x0 goto pc+16 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 33: (61) r1 = *(u32 *)(r0 +28) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 34: (61) r2 = *(u32 *)(r10 -24) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 35: (5d) if r1 != r2 goto pc+11 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 36: (61) r1 = *(u32 *)(r0 +32) # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 37: (61) r2 = *(u32 *)(r10 -20) # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 38: (5d) if r1 != r2 goto pc+8 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 39: (61) r1 = *(u32 *)(r0 +36) # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 40: (61) r2 = *(u32 *)(r10 -16) # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 41: (5d) if r1 != r2 goto pc+5 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 42: (61) r1 = *(u32 *)(r0 +40) # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 43: (61) r2 = *(u32 *)(r10 -12) # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 44: (5d) if r1 != r2 goto pc+2 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_port != DST_REWRITE_PORT6) { # 45: (61) r1 = *(u32 *)(r0 +44) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 46: (15) if r1 == 0x1a0a goto pc+4 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; bpf_sk_release(sk); # 47: (bf) r1 = r0 # 48: (85) call bpf_sk_release#86 # ; } # 49: (bf) r0 = r7 # 50: (95) exit # # from 46 to 51: R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv6666 R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; bpf_sk_release(sk); # 51: (bf) r1 = r0 # 52: (85) call bpf_sk_release#86 # 53: (b7) r1 = 2586 # ; ctx->user_port = bpf_htons(DST_REWRITE_PORT6); # 54: (63) *(u32 *)(r6 +24) = r1 # 55: (18) r1 = 0x100000000000000 # ; ctx->user_ip6[2] = bpf_htonl(DST_REWRITE_IP6_2); # 57: (7b) *(u64 *)(r6 +16) = r1 # invalid bpf_context access off=16 size=8 # processed 49 insns (limit 1000000) max_states_per_insn 0 total_states 13 peak_states 13 mark_read 11 # # libbpf: -- END LOG -- # libbpf: failed to load program 'cgroup/connect6' # libbpf: failed to load object './connect6_prog.o' # libbpf: load bpf program failed: Permission denied # libbpf: -- BEGIN DUMP LOG --- # libbpf: # ; int connect_v6_prog(struct bpf_sock_addr *ctx) # 0: (bf) r6 = r1 # 1: (18) r1 = 0x100000000000000 # ; tuple.ipv6.daddr[0] = bpf_htonl(DST_REWRITE_IP6_0); # 3: (7b) *(u64 *)(r10 -16) = r1 # 4: (b7) r1 = 169476096 # ; memset(&tuple.ipv6.sport, 0, sizeof(tuple.ipv6.sport)); # 5: (63) *(u32 *)(r10 -8) = r1 # 6: (b7) r7 = 0 # ; tuple.ipv6.daddr[0] = bpf_htonl(DST_REWRITE_IP6_0); # 7: (7b) *(u64 *)(r10 -24) = r7 # 8: (7b) *(u64 *)(r10 -32) = r7 # 9: (7b) *(u64 *)(r10 -40) = r7 # ; if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) # 10: (61) r1 = *(u32 *)(r6 +32) # ; if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) # 11: (bf) r2 = r1 # 12: (07) r2 += -1 # 13: (67) r2 <<= 32 # 14: (77) r2 >>= 32 # 15: (25) if r2 > 0x1 goto pc+33 # R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=1,var_off=(0x0; 0x1)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=00000000 fp-32=00000000 fp-40=00000000 # ; else if (ctx->type == SOCK_STREAM) # 16: (55) if r1 != 0x1 goto pc+8 # R1=inv1 R2=inv(id=0,umax_value=1,var_off=(0x0; 0x1)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=00000000 fp-32=00000000 fp-40=00000000 # 17: (bf) r2 = r10 # ; sk = bpf_sk_lookup_tcp(ctx, &tuple, sizeof(tuple.ipv6), # 18: (07) r2 += -40 # 19: (bf) r1 = r6 # 20: (b7) r3 = 36 # 21: (b7) r4 = -1 # 22: (b7) r5 = 0 # 23: (85) call bpf_sk_lookup_tcp#84 # 24: (05) goto pc+7 # ; if (!sk) # 32: (15) if r0 == 0x0 goto pc+16 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 33: (61) r1 = *(u32 *)(r0 +28) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 34: (61) r2 = *(u32 *)(r10 -24) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 35: (5d) if r1 != r2 goto pc+11 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 36: (61) r1 = *(u32 *)(r0 +32) # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 37: (61) r2 = *(u32 *)(r10 -20) # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 38: (5d) if r1 != r2 goto pc+8 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 39: (61) r1 = *(u32 *)(r0 +36) # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 40: (61) r2 = *(u32 *)(r10 -16) # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 41: (5d) if r1 != r2 goto pc+5 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 42: (61) r1 = *(u32 *)(r0 +40) # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 43: (61) r2 = *(u32 *)(r10 -12) # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 44: (5d) if r1 != r2 goto pc+2 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_port != DST_REWRITE_PORT6) { # 45: (61) r1 = *(u32 *)(r0 +44) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 46: (15) if r1 == 0x1a0a goto pc+4 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; bpf_sk_release(sk); # 47: (bf) r1 = r0 # 48: (85) call bpf_sk_release#86 # ; } # 49: (bf) r0 = r7 # 50: (95) exit # # from 46 to 51: R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv6666 R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; bpf_sk_release(sk); # 51: (bf) r1 = r0 # 52: (85) call bpf_sk_release#86 # 53: (b7) r1 = 2586 # ; ctx->user_port = bpf_htons(DST_REWRITE_PORT6); # 54: (63) *(u32 *)(r6 +24) = r1 # 55: (18) r1 = 0x100000000000000 # ; ctx->user_ip6[2] = bpf_htonl(DST_REWRITE_IP6_2); # 57: (7b) *(u64 *)(r6 +16) = r1 # invalid bpf_context access off=16 size=8 # processed 49 insns (limit 1000000) max_states_per_insn 0 total_states 13 peak_states 13 mark_read 11 # # libbpf: -- END LOG -- # libbpf: failed to load program 'cgroup/connect6' # libbpf: failed to load object './connect6_prog.o' # (test_sock_addr.c:752: errno: Bad file descriptor) >>> Loading program (./connect6_prog.o) error. # # libbpf: load bpf program failed: Permission denied # libbpf: -- BEGIN DUMP LOG --- # libbpf: # ; int connect_v6_prog(struct bpf_sock_addr *ctx) # 0: (bf) r6 = r1 # 1: (18) r1 = 0x100000000000000 # ; tuple.ipv6.daddr[0] = bpf_htonl(DST_REWRITE_IP6_0); # 3: (7b) *(u64 *)(r10 -16) = r1 # 4: (b7) r1 = 169476096 # ; memset(&tuple.ipv6.sport, 0, sizeof(tuple.ipv6.sport)); # 5: (63) *(u32 *)(r10 -8) = r1 # 6: (b7) r7 = 0 # ; tuple.ipv6.daddr[0] = bpf_htonl(DST_REWRITE_IP6_0); # 7: (7b) *(u64 *)(r10 -24) = r7 # 8: (7b) *(u64 *)(r10 -32) = r7 # 9: (7b) *(u64 *)(r10 -40) = r7 # ; if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) # 10: (61) r1 = *(u32 *)(r6 +32) # ; if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) # 11: (bf) r2 = r1 # 12: (07) r2 += -1 # 13: (67) r2 <<= 32 # 14: (77) r2 >>= 32 # 15: (25) if r2 > 0x1 goto pc+33 # R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=1,var_off=(0x0; 0x1)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=00000000 fp-32=00000000 fp-40=00000000 # ; else if (ctx->type == SOCK_STREAM) # 16: (55) if r1 != 0x1 goto pc+8 # R1=inv1 R2=inv(id=0,umax_value=1,var_off=(0x0; 0x1)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=00000000 fp-32=00000000 fp-40=00000000 # 17: (bf) r2 = r10 # ; sk = bpf_sk_lookup_tcp(ctx, &tuple, sizeof(tuple.ipv6), # 18: (07) r2 += -40 # 19: (bf) r1 = r6 # 20: (b7) r3 = 36 # 21: (b7) r4 = -1 # 22: (b7) r5 = 0 # 23: (85) call bpf_sk_lookup_tcp#84 # 24: (05) goto pc+7 # ; if (!sk) # 32: (15) if r0 == 0x0 goto pc+16 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 33: (61) r1 = *(u32 *)(r0 +28) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 34: (61) r2 = *(u32 *)(r10 -24) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 35: (5d) if r1 != r2 goto pc+11 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 36: (61) r1 = *(u32 *)(r0 +32) # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 37: (61) r2 = *(u32 *)(r10 -20) # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 38: (5d) if r1 != r2 goto pc+8 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 39: (61) r1 = *(u32 *)(r0 +36) # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 40: (61) r2 = *(u32 *)(r10 -16) # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 41: (5d) if r1 != r2 goto pc+5 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 42: (61) r1 = *(u32 *)(r0 +40) # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 43: (61) r2 = *(u32 *)(r10 -12) # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 44: (5d) if r1 != r2 goto pc+2 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_port != DST_REWRITE_PORT6) { # 45: (61) r1 = *(u32 *)(r0 +44) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 46: (15) if r1 == 0x1a0a goto pc+4 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; bpf_sk_release(sk); # 47: (bf) r1 = r0 # 48: (85) call bpf_sk_release#86 # ; } # 49: (bf) r0 = r7 # 50: (95) exit # # from 46 to 51: R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv6666 R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; bpf_sk_release(sk); # 51: (bf) r1 = r0 # 52: (85) call bpf_sk_release#86 # 53: (b7) r1 = 2586 # ; ctx->user_port = bpf_htons(DST_REWRITE_PORT6); # 54: (63) *(u32 *)(r6 +24) = r1 # 55: (18) r1 = 0x100000000000000 # ; ctx->user_ip6[2] = bpf_htonl(DST_REWRITE_IP6_2); # 57: (7b) *(u64 *)(r6 +16) = r1 # invalid bpf_context access off=16 size=8 # processed 49 insns (limit 1000000) max_states_per_insn 0 total_states 13 peak_states 13 mark_read 11 # # libbpf: -- END LOG -- # libbpf: failed to load program 'cgroup/connect6' # libbpf: failed to load object './connect6_prog.o' # (test_sock_addr.c:752: errno: Bad file descriptor) >>> Loading program (./connect6_prog.o) error. # # libbpf: load bpf program failed: Permission denied # libbpf: -- BEGIN DUMP LOG --- # libbpf: # ; int connect_v6_prog(struct bpf_sock_addr *ctx) # 0: (bf) r6 = r1 # 1: (18) r1 = 0x100000000000000 # ; tuple.ipv6.daddr[0] = bpf_htonl(DST_REWRITE_IP6_0); # 3: (7b) *(u64 *)(r10 -16) = r1 # 4: (b7) r1 = 169476096 # ; memset(&tuple.ipv6.sport, 0, sizeof(tuple.ipv6.sport)); # 5: (63) *(u32 *)(r10 -8) = r1 # 6: (b7) r7 = 0 # ; tuple.ipv6.daddr[0] = bpf_htonl(DST_REWRITE_IP6_0); # 7: (7b) *(u64 *)(r10 -24) = r7 # 8: (7b) *(u64 *)(r10 -32) = r7 # 9: (7b) *(u64 *)(r10 -40) = r7 # ; if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) # 10: (61) r1 = *(u32 *)(r6 +32) # ; if (ctx->type != SOCK_STREAM && ctx->type != SOCK_DGRAM) # 11: (bf) r2 = r1 # 12: (07) r2 += -1 # 13: (67) r2 <<= 32 # 14: (77) r2 >>= 32 # 15: (25) if r2 > 0x1 goto pc+33 # R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=1,var_off=(0x0; 0x1)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=00000000 fp-32=00000000 fp-40=00000000 # ; else if (ctx->type == SOCK_STREAM) # 16: (55) if r1 != 0x1 goto pc+8 # R1=inv1 R2=inv(id=0,umax_value=1,var_off=(0x0; 0x1)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=00000000 fp-32=00000000 fp-40=00000000 # 17: (bf) r2 = r10 # ; sk = bpf_sk_lookup_tcp(ctx, &tuple, sizeof(tuple.ipv6), # 18: (07) r2 += -40 # 19: (bf) r1 = r6 # 20: (b7) r3 = 36 # 21: (b7) r4 = -1 # 22: (b7) r5 = 0 # 23: (85) call bpf_sk_lookup_tcp#84 # 24: (05) goto pc+7 # ; if (!sk) # 32: (15) if r0 == 0x0 goto pc+16 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 33: (61) r1 = *(u32 *)(r0 +28) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 34: (61) r2 = *(u32 *)(r10 -24) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 35: (5d) if r1 != r2 goto pc+11 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 36: (61) r1 = *(u32 *)(r0 +32) # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 37: (61) r2 = *(u32 *)(r10 -20) # ; sk->src_ip6[1] != tuple.ipv6.daddr[1] || # 38: (5d) if r1 != r2 goto pc+8 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 39: (61) r1 = *(u32 *)(r0 +36) # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 40: (61) r2 = *(u32 *)(r10 -16) # ; sk->src_ip6[2] != tuple.ipv6.daddr[2] || # 41: (5d) if r1 != r2 goto pc+5 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 42: (61) r1 = *(u32 *)(r0 +40) # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 43: (61) r2 = *(u32 *)(r10 -12) # ; sk->src_ip6[3] != tuple.ipv6.daddr[3] || # 44: (5d) if r1 != r2 goto pc+2 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; sk->src_port != DST_REWRITE_PORT6) { # 45: (61) r1 = *(u32 *)(r0 +44) # ; if (sk->src_ip6[0] != tuple.ipv6.daddr[0] || # 46: (15) if r1 == 0x1a0a goto pc+4 # R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; bpf_sk_release(sk); # 47: (bf) r1 = r0 # 48: (85) call bpf_sk_release#86 # ; } # 49: (bf) r0 = r7 # 50: (95) exit # # from 46 to 51: R0=sock(id=0,ref_obj_id=2,off=0,imm=0) R1=inv6666 R2=inv(id=0,umax_value=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(id=0,off=0,imm=0) R7=inv0 R10=fp0,call_-1 fp-8=????mmmm fp-16=mmmmmmmm fp-24=mmmmmmmm fp-32=mmmmmmmm fp-40=mmmmmmmm refs=2 # ; bpf_sk_release(sk); # 51: (bf) r1 = r0 # 52: (85) call bpf_sk_release#86 # 53: (b7) r1 = 2586 # ; ctx->user_port = bpf_htons(DST_REWRITE_PORT6); # 54: (63) *(u32 *)(r6 +24) = r1 # 55: (18) r1 = 0x100000000000000 # ; ctx->user_ip6[2] = bpf_htonl(DST_REWRITE_IP6_2); # 57: (7b) *(u64 *)(r6 +16) = r1 # invalid bpf_context access off=16 size=8 # processed 49 insns (limit 1000000) max_states_per_insn 0 total_states 13 peak_states 13 mark_read 11 # # libbpf: -- END LOG -- # libbpf: failed to load program 'cgroup/connect6' # libbpf: failed to load object './connect6_prog.o' # (test_sock_addr.c:752: errno: Bad file descriptor) >>> Loading program (./connect6_prog.o) error. # # (test_sock_addr.c:1122: errno: Operation not permitted) Fail to send message to server # libbpf: load bpf program failed: Permission denied # libbpf: -- BEGIN DUMP LOG --- # libbpf: # ; int sendmsg_v6_prog(struct bpf_sock_addr *ctx) # 0: (b7) r0 = 0 # ; if (ctx->type != SOCK_DGRAM) # 1: (61) r2 = *(u32 *)(r1 +32) # ; if (ctx->type != SOCK_DGRAM) # 2: (55) if r2 != 0x2 goto pc+20 # R0=inv0 R1=ctx(id=0,off=0,imm=0) R2=inv2 R10=fp0,call_-1 # ; if (ctx->msg_src_ip6[3] == bpf_htonl(1) || # 3: (61) r2 = *(u32 *)(r1 +56) # ; if (ctx->msg_src_ip6[3] == bpf_htonl(1) || # 4: (47) r2 |= 16777216 # 5: (15) if r2 == 0x1000000 goto pc+1 # R0=inv0 R1=ctx(id=0,off=0,imm=0) R2=inv(id=0,umin_value=16777216,umax_value=4294967295,var_off=(0x1000000; 0xfeffffff)) R10=fp0,call_-1 # 6: (05) goto pc+16 # ; } # 23: (95) exit # # from 5 to 7: R0=inv0 R1=ctx(id=0,off=0,imm=0) R2=inv16777216 R10=fp0,call_-1 # ; if (ctx->msg_src_ip6[3] == bpf_htonl(1) || # 7: (b7) r2 = 100663296 # ; ctx->msg_src_ip6[3] = bpf_htonl(SRC_REWRITE_IP6_3); # 8: (63) *(u32 *)(r1 +56) = r2 # 9: (b7) r0 = 0 # ; ctx->msg_src_ip6[2] = bpf_htonl(SRC_REWRITE_IP6_2); # 10: (63) *(u32 *)(r1 +52) = r0 # ; ctx->msg_src_ip6[1] = bpf_htonl(SRC_REWRITE_IP6_1); # 11: (63) *(u32 *)(r1 +48) = r0 # ; ctx->msg_src_ip6[0] = bpf_htonl(SRC_REWRITE_IP6_0); # 12: (63) *(u32 *)(r1 +44) = r0 # ; if ((ctx->user_ip6[0] & 0xFFFF) == bpf_htons(0xFACE) && # 13: (61) r2 = *(u32 *)(r1 +8) # ; if ((ctx->user_ip6[0] & 0xFFFF) == bpf_htons(0xFACE) && # 14: (55) if r2 != 0xcb0cefa goto pc+8 # R0=inv0 R1=ctx(id=0,off=0,imm=0) R2=inv212913914 R10=fp0,call_-1 # 15: (b7) r2 = 2586 # ; ctx->user_port = bpf_htons(DST_REWRITE_PORT6); # 16: (63) *(u32 *)(r1 +24) = r2 # 17: (18) r2 = 0x100000000000000 # ; ctx->user_ip6[2] = bpf_htonl(DST_REWRITE_IP6_2); # 19: (7b) *(u64 *)(r1 +16) = r2 # invalid bpf_context access off=16 size=8 # processed 20 insns (limit 1000000) max_states_per_insn 0 total_states 5 peak_states 5 mark_read 3 # # libbpf: -- END LOG -- # libbpf: failed to load program 'cgroup/sendmsg6' # libbpf: failed to load object './sendmsg6_prog.o' # (test_sock_addr.c:752: errno: Bad file descriptor) >>> Loading program (./sendmsg6_prog.o) error. # # (test_sock_addr.c:1122: errno: Unknown error 524) Fail to send message to server # (test_sock_addr.c:1122: errno: Operation not permitted) Fail to send message to server # Test case: bind4: load prog with wrong expected attach type .. [PASS] # Test case: bind4: attach prog with wrong attach type .. [PASS] # Test case: bind4: rewrite IP & TCP port in .. [PASS] # Test case: bind4: rewrite IP & UDP port in .. [PASS] # Test case: bind6: load prog with wrong expected attach type .. [PASS] # Test case: bind6: attach prog with wrong attach type .. [PASS] # Test case: bind6: rewrite IP & TCP port in .. [PASS] # Test case: bind6: rewrite IP & UDP port in .. [PASS] # Test case: connect4: load prog with wrong expected attach type .. [PASS] # Test case: connect4: attach prog with wrong attach type .. [PASS] # Test case: connect4: rewrite IP & TCP port .. [PASS] # Test case: connect4: rewrite IP & UDP port .. [PASS] # Test case: connect6: load prog with wrong expected attach type .. [PASS] # Test case: connect6: attach prog with wrong attach type .. [FAIL] # Test case: connect6: rewrite IP & TCP port .. [FAIL] # Test case: connect6: rewrite IP & UDP port .. [FAIL] # Test case: sendmsg4: load prog with wrong expected attach type .. [PASS] # Test case: sendmsg4: attach prog with wrong attach type .. [PASS] # Test case: sendmsg4: rewrite IP & port (asm) .. [PASS] # Test case: sendmsg4: rewrite IP & port (C) .. [PASS] # Test case: sendmsg4: deny call .. [PASS] # Test case: sendmsg6: load prog with wrong expected attach type .. [PASS] # Test case: sendmsg6: attach prog with wrong attach type .. [PASS] # Test case: sendmsg6: rewrite IP & port (asm) .. [PASS] # Test case: sendmsg6: rewrite IP & port (C) .. [FAIL] # Test case: sendmsg6: IPv4-mapped IPv6 .. [PASS] # Test case: sendmsg6: set dst IP = [::] (BSD'ism) .. [PASS] # Test case: sendmsg6: preserve dst IP = [::] (BSD'ism) .. [PASS] # Test case: sendmsg6: deny call .. [PASS] # Summary: 25 PASSED, 4 FAILED not ok 34 selftests: bpf: test_sock_addr.sh To reproduce: # build kernel cd linux cp config-5.2.0-rc2-00597-gcd17d77 .config make HOSTCC=gcc-7 CC=gcc-7 ARCH=x86_64 olddefconfig make HOSTCC=gcc-7 CC=gcc-7 ARCH=x86_64 prepare make HOSTCC=gcc-7 CC=gcc-7 ARCH=x86_64 modules_prepare make HOSTCC=gcc-7 CC=gcc-7 ARCH=x86_64 SHELL=/bin/bash make HOSTCC=gcc-7 CC=gcc-7 ARCH=x86_64 bzImage git clone https://github.com/intel/lkp-tests.git cd lkp-tests bin/lkp qemu -k job-script # job-script is attached in this email Thanks, Rong Chen