eBPF+Ftrace 合璧剑指：no space left on device？

$ sudo docker run --rm -ti busybox ls -hl|wc -l
docker: Error response from daemon: error creating overlay mount to /var/lib/docker/overlay2/40de1c588e43dea75cf80a971d1be474886d873dddee0f00369fc7c8f12b7149-init/merged: 
no space left on device.
See 'docker run --help'.

$ sudo df -Th
Filesystem             Type        Size  Used Avail Use% Mounted on
tmpfs                  tmpfs       392M  1.2M  391M   1% /run
/dev/sda1              ext4         40G   19G   22G  46% /
tmpfs                  tmpfs       2.0G     0  2.0G   0% /dev/shm
tmpfs                  tmpfs       5.0M     0  5.0M   0% /run/lock
/dev/sda15             vfat         98M  5.1M   93M   6% /boot/efi
tmpfs                  tmpfs       392M  4.0K  392M   1% /run/user/1000
overlay                overlay      40G   19G   22G  46% /root/overlay2/merged

$ sudo df -i
Filesystem                 Inodes  IUsed      IFree IUse% Mounted on
tmpfs                      500871    718     500153    1% /run
/dev/sda1                 5186560 338508    4848052    7% /
tmpfs                      500871      1     500870    1% /dev/shm
tmpfs                      500871      3     500868    1% /run/lock
/dev/sda15                      0      0          0     - /boot/efi
tmpfs                      100174     29     100145    1% /run/user/1000
overlay                   5186560 338508    4848052    7% /root/overlay2/merged

$ sudo lsof | grep deleted
empty

#define ENOSPC    28  /* No space left on device */

$ sudo syscount-bpfcc -h
usage: syscount-bpfcc [-h] [-p PID] [-i INTERVAL] [-d DURATION] [-T TOP] [-x] [-e ERRNO] [-L] [-m] [-P] [-l]
Summarize syscall counts and latencies.
optional arguments:
  -h, --help            show this help message and exit
  -p PID, --pid PID     trace only this pid
  -i INTERVAL, --interval INTERVAL
print summary at this interval (seconds)
  -d DURATION, --duration DURATION
                        total duration of trace, in seconds
  -T TOP, --top TOP     print only the top syscalls by count or latency
  -x, --failures        trace only failed syscalls (return < 0)
  -e ERRNO, --errno ERRNO
                        trace only syscalls that return this error (numeric or EPERM, etc.)
  -L, --latency         collect syscall latency
  -m, --milliseconds    display latency in milliseconds (default: microseconds)
  -P, --process         count by process and not by syscall
  -l, --list            print list of recognized syscalls and exit

$ sudo syscount-bpfcc -e ENOSPC
Tracing syscalls, printing top 10... Ctrl+C to quit.
^C[08:34:38]
SYSCALL                   COUNT
mount                         1

$ sudo syscount-bpfcc -e ENOSPC -P
Tracing syscalls, printing top 10... Ctrl+C to quit.
^C[08:35:32]
PID    COMM               COUNT
3010   dockerd                1

$ sudo ./funcgraph -h
USAGE: funcgraph [-aCDhHPtT] [-m maxdepth] [-p PID] [-L TID] [-d secs] funcstring
                 -a              # all info (same as -HPt)
                 -C              # measure on-CPU time only
                 -d seconds      # trace duration, and use buffers
                 -D              # do not show function duration
                 -h              # this usage message
                 -H              # include column headers
                 -m maxdepth     # max stack depth to show
                 -p PID          # trace when this pid is on-CPU
                 -L TID          # trace when this thread is on-CPU
                 -P              # show process names & PIDs
                 -t              # show timestamps
                 -T              # comment function tails
  eg,
       funcgraph do_nanosleep    # trace do_nanosleep() and children
       funcgraph -m 3 do_sys_open # trace do_sys_open() to 3 levels only
       funcgraph -a do_sys_open    # include timestamps and process name
       funcgraph -p 198 do_sys_open # trace vfs_read() for PID 198 only
       funcgraph -d 1 do_sys_open >out # trace 1 sec, then write to file

$ sudo ./funcgraph -m 2 __arm64_sys_mount

$ sudo ./funcgraph -m 5 path_mount > path_mount.log

__arm64_sys_mount()
  -> path_mount()
    -> do_new_mount()
      -> do_add_mount()
        -> graft_tree()
          -> attach_recursive_mnt()
            -> count_mounts()

$ sudo trace-bpfcc 'r::__arm64_sys_mount() "%llx", retval'  \
                   'r::path_mount "%llx", retval' \
'r::do_new_mount "%llx", retval' \
'r::do_add_mount "%llx", retval'\
'r::graft_tree "%llx", retval' \
'r::attach_recursive_mnt "%llx" retval'\
'r::count_mounts "%llx", retval'
PID     TID     COMM            FUNC             -
3010    3017    dockerd         graft_tree       ffffffe4
3010    3017    dockerd         attach_recursive_mnt ffffffe4
3010    3017    dockerd         count_mounts     ffffffe4
3010    3017    dockerd         __arm64_sys_mount ffffffffffffffe4
3010    3017    dockerd         path_mount       ffffffe4
3010    3017    dockerd         do_new_mount     ffffffe4
3010    3017    dockerd         do_add_mount     ffffffe4

int count_mounts(struct mnt_namespace *ns, struct mount *mnt)
{
// 可以加载的最大值是通过 sysctl_mount_max 的变量读取的
  unsigned int max = READ_ONCE(sysctl_mount_max); 
  unsigned int mounts = 0, old, pending, sum;
  struct mount *p;
  for (p = mnt; p; p = next_mnt(p, mnt))
    mounts++;
  old = ns->mounts;  // 当前 namespace 挂载的数量 
  pending = ns->pending_mounts;  // 按照字面意思理解是 pending 的加载数量
  sum = old + pending;   // 加载的总和为 已经加载的数量 + 在路上加载的数量
  if ((old > sum) ||
      (pending > sum) ||
      (max < sum) ||
      (mounts > (max - sum)))  // 那么这些条件的判断也就比较容易理解了
    return -ENOSPC;
  ns->pending_mounts = pending + mounts;
  return 0;
}

$ sudo cat /proc/sys/fs/mount-max
10

SYSCALL_DEFINE5(mount, char __user *, dev_name, char __user *, dir_name,
    char __user *, type, unsigned long, flags, void __user *, data)
{
  int ret;
  char *kernel_type;
  char *kernel_dev;
  void *options;
  kernel_type = copy_mount_string(type); // 子函数 1
  ret = PTR_ERR(kernel_type);
  if (IS_ERR(kernel_type))
    goto out_type;
  kernel_dev = copy_mount_string(dev_name); // 子函数 2
  ret = PTR_ERR(kernel_dev);
  if (IS_ERR(kernel_dev))
    goto out_dev;
  options = copy_mount_options(data); // 子函数 3
  ret = PTR_ERR(options);
  if (IS_ERR(options))
    goto out_data;
  ret = do_mount(kernel_dev, dir_name, kernel_type, flags, options); // 子函数 4
  kfree(options);
out_data:
  kfree(kernel_dev);
out_dev:
  kfree(kernel_type);
out_type:
  return ret;
}

static char *copy_mount_string(const void __user *data)
{
  return data ? strndup_user(data, PATH_MAX) : NULL;
}

$ sudo syscount-bpfcc 
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/bcc/syscall.py", line 379, in <module>
    out = subprocess.check_output(['ausyscall', '--dump'], stderr=subprocess.STDOUT)
  File "/usr/lib/python3.9/subprocess.py", line 424, in check_output
return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
  File "/usr/lib/python3.9/subprocess.py", line 505, in run
    with Popen(*popenargs, **kwargs) as process:
  File "/usr/lib/python3.9/subprocess.py", line 951, in __init__
    self._execute_child(args, executable, preexec_fn, close_fds,
  File "/usr/lib/python3.9/subprocess.py", line 1821, in _execute_child
    raise child_exception_type(errno_num, err_msg, err_filename)
FileNotFoundError: [Errno 2] No such file or directory: 'ausyscall'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
  File "/usr/sbin/syscount-bpfcc", line 20, in <module>
    from bcc.syscall import syscall_name, syscalls
  File "/usr/lib/python3/dist-packages/bcc/syscall.py", line 387, in <module>
    raise Exception("ausyscall: command not found")
Exception: ausyscall: command not found

$ sudo apt-get install auditd