一、背景
通过内核的新增的mmap_lock tracepoint结合ebpf，实现对mmap_lock的调测，监控线程持锁时间
二、实现
1、社区在mmap_lock增加tracepoint
https://github.com/torvalds/linux/commit/2b5067a8143e34aa3fa57a20fb8a3c40d905f942
https://github.com/torvalds/linux/commit/10994316089c9682f2fbe0be0b1e82bcaf5f4e8c
https://github.com/torvalds/linux/commit/2f1aaf3ea666b737ad717b3d88667225aca23149
在down/up read/write位置增加tracepoint点，可通过tracepoint来统计跟踪down up流程与持锁时间来做维测。
2、ebpf实现用户态跟踪
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
| https://zhuanlan.zhihu.com/p/347241870
https://zhuanlan.zhihu.com/p/450571608 //bpf hello
/* read trace logs from debug fs */
void read_trace_pipe(void)
{
int trace_fd;
trace_fd = open(DEBUGFS "trace_pipe", O_RDONLY, 0);
if (trace_fd < 0)
return;
while (1) {
static char buf[4096];
ssize_t sz;
sz = read(trace_fd, buf, sizeof(buf) - 1);
if (sz> 0) {
buf[sz] = 0;
puts(buf);
}
}
}
|
1
2
3
4
5
6
7
8
9
| https://nakryiko.com/posts/bpf-tips-printk/
bpfprint小技巧：
/* Helper macro to print out debug messages */
#define bpf_printk(fmt, ...) \
({ \
char ____fmt[] = fmt; \
bpf_trace_printk(____fmt, sizeof(____fmt), \
##__VA_ARGS__); \
})
|
1
| https://lwn.net/Articles/818714/ Dumping kernel data structures with BPF BPF iterator机制合入了，可以利用kernel的dump机制把数据结构dump到user space。4月份的文章介绍过
|
1
| tools/testing/selftests/bpf/progs/bpf_iter_task_stack.c //kernel代码中的示例
|
1
| https://github.com/iovisor/bcc/blob/master/docs/reference_guide.md#3-attach_tracepoint
|
1
2
3
4
5
6
| https://www.brendangregg.com/blog/2016-01-18/ebpf-stack-trace-hack.html
//保存stack信息
https://github.com/iovisor/bcc/blob/master/tools/old/memleak.py
int depth = 0;
u64 bp = ctx->bp;
if (!(info->callstack[depth++] = get_frame(&bp))) return depth;
|
1
2
3
4
5
6
| https://man7.org/linux/man-pages/man7/bpf-helpers.7.html
bpf helper.
bpf_get_stack
bpf_get_task_stack // long bpf_get_task_stack(struct task_struct *task, void *buf, u32 size, u64 flags)
bpf_ktime_get_ns // u64 bpf_ktime_get_ns(void)
bpf_get_current_pid_tgid
|
1
2
3
4
5
6
7
8
| //生成ebpf的头文件
bpftool gen skeleton mmap_lock_bpf
clang -g -O2 -target bpf -c mmap_lock.bpf.c -o mmap_lock_bpf
llvm-strip -g mmap_lock_bpf
[root@localhost bazel-bin]# llvm-objdump -t *.bpf.o
clang -g -O2 -target bpf -D__TARGET_ARCH_x86 -I . -I bazel-out/aarch64-fastbuild/bin/external/linux/libbpf/include -c third_party/bpf/schedlat.bpf.c -o bazel-out/aarch64-fastbuild/bin/third_party/bpf/schedlat_bpf.o && llvm-strip -g bazel-out/aarch64-fastbuild/bin/third_party/bpf/schedlat_bpf.o
|
https://facebookmicrosites.github.io/bpf/blog/2020/02/20/bcc-to-libbpf-howto-guide.html //bcc与libbpf写法的转化
https://www.ebpf.top/post/kernel_btf/ //kernel btf
ebpf程序编译：
1
2
3
4
5
6
7
8
9
10
11
12
13
14
| clang -g -O2 -target bpf -D__TARGET_ARCH_x86 -I . -I bazel-out/aarch64-fastbuild/bin/external/linux/libbpf/include -c third_party/bpf/schedlat.bpf.c -o bazel-out/aarch64-fastbuild/bin/third_party/bpf/schedlat_bpf.o && llvm-strip -g bazel-out/aarch64-fastbuild/bin/third_party/bpf/schedlat_bpf.o'
bpftool gen skeleton
/usr/bin/gcc -U_FORTIFY_SOURCE -fstack-protector -Wall -Wunused-but-set-parameter -Wno-free-nonheap-object -fno-omit-frame-pointer -MD -MF bazel-out/aarch64-fastbuild/bin/_objs/schedlat/schedlat.pic.d '-frandom-seed=bazel-out/aarch64-fastbuild/bin/_objs/schedlat/schedlat.pic.o' -fPIC -iquote . -iquote bazel-out/aarch64-fastbuild/bin -iquote external/bazel_tools -iquote bazel-out/aarch64-fastbuild/bin/external/bazel_tools -isystem bazel-out/aarch64-fastbuild/bin/external/linux/libbpf/include -Wno-sign-compare -DGHOST_LOGGING -Wl,-I/home/csluo/ld-linux-aarch64.so.1 -g -fno-canonical-system-headers -Wno-builtin-macro-redefined '-D__DATE__="redacted"' '-D__TIMESTAMP__="redacted"' '-D__TIME__="redacted"' -c bpf/user/schedlat.c -o bazel-out/aarch64-fastbuild/bin/_objs/schedlat/schedlat.pic.o
gcc -o schedlat -Wl,-S -fuse-ld=gold -Wl,-no-as-needed -Wl,-z,relro,-z,now -B/usr/bin -pass-exit-codes -lstdc++ -lm _objs/schedlat/schedlat.pic.o /home/ghost-userspace/bazel-out/aarch64-fastbuild/bin/external/linux/libbpf/lib/libbpf.a -lelf -lz
gcc -o schedla _objs/schedlat/schedlat.pic.o -lbpf -lelf
/usr/bin/gcc -U_FORTIFY_SOURCE -fstack-protector -Wall -Wunused-but-set-parameter -Wno-free-nonheap-object -fno-omit-frame-pointer -Wno-sign-compare -DGHOST_LOGGING -Wl,-I/home/csluo/ld-linux-aarch64.so.1 -g -fno-canonical-system-headers -Wno-builtin-macro-redefined '-D__DATE__="redacted"' '-D__TIMESTAMP__="redacted"' '-D__TIME__="redacted"' -c loader.c -o loader.pic.o
gcc -o mmap_lock_debug loader.pic.o -lbpf -lelf
|
1> 基于BCC的实现
2> 基于libbpf的实现
3、内核ko实现tracepoint跟踪
三、其他知识
1、ebpf程序分类
1
| https://blog.csdn.net/weixin_41036447/article/details/106473865 //ebpf程序分类
|
