ZFS case : top CPU 100%sy, when no free memory trigger it.

cd /sys/module/zfs/parameters
# grep '' *|sort 
l2arc_feed_again:1
l2arc_feed_min_ms:200
l2arc_feed_secs:1
l2arc_headroom:2
l2arc_headroom_boost:200
l2arc_nocompress:0
l2arc_noprefetch:1
l2arc_norw:0
l2arc_write_boost:8388608
l2arc_write_max:8388608
metaslab_debug_load:0
metaslab_debug_unload:0
spa_asize_inflation:24
spa_config_path:/etc/zfs/zpool.cache
zfetch_array_rd_sz:1048576
zfetch_block_cap:256
zfetch_max_streams:8
zfetch_min_sec_reap:2
zfs_arc_grow_retry:5
zfs_arc_max:10240000000
zfs_arc_memory_throttle_disable:1
zfs_arc_meta_limit:0
zfs_arc_meta_prune:1048576
zfs_arc_min:0
zfs_arc_min_prefetch_lifespan:1000
zfs_arc_p_aggressive_disable:1
zfs_arc_p_dampener_disable:1
zfs_arc_shrink_shift:5
zfs_autoimport_disable:0
zfs_dbuf_state_index:0
zfs_deadman_enabled:1
zfs_deadman_synctime_ms:1000000
zfs_dedup_prefetch:1
zfs_delay_min_dirty_percent:60
zfs_delay_scale:500000
zfs_dirty_data_max:10240000000
zfs_dirty_data_max_max:101595342848
zfs_dirty_data_max_max_percent:25
zfs_dirty_data_max_percent:10
zfs_dirty_data_sync:67108864
zfs_disable_dup_eviction:0
zfs_expire_snapshot:300
zfs_flags:1
zfs_free_min_time_ms:1000
zfs_immediate_write_sz:32768
zfs_mdcomp_disable:0
zfs_nocacheflush:0
zfs_nopwrite_enabled:1
zfs_no_scrub_io:0
zfs_no_scrub_prefetch:0
zfs_pd_blks_max:100
zfs_prefetch_disable:0
zfs_read_chunk_size:1048576
zfs_read_history:0
zfs_read_history_hits:0
zfs_recover:0
zfs_resilver_delay:2
zfs_resilver_min_time_ms:3000
zfs_scan_idle:50
zfs_scan_min_time_ms:1000
zfs_scrub_delay:4
zfs_send_corrupt_data:0
zfs_sync_pass_deferred_free:2
zfs_sync_pass_dont_compress:5
zfs_sync_pass_rewrite:2
zfs_top_maxinflight:32
zfs_txg_history:0
zfs_txg_timeout:5
zfs_vdev_aggregation_limit:131072
zfs_vdev_async_read_max_active:3
zfs_vdev_async_read_min_active:1
zfs_vdev_async_write_active_max_dirty_percent:60
zfs_vdev_async_write_active_min_dirty_percent:30
zfs_vdev_async_write_max_active:10
zfs_vdev_async_write_min_active:1
zfs_vdev_cache_bshift:16
zfs_vdev_cache_max:16384
zfs_vdev_cache_size:0
zfs_vdev_max_active:1000
zfs_vdev_mirror_switch_us:10000
zfs_vdev_read_gap_limit:32768
zfs_vdev_scheduler:noop
zfs_vdev_scrub_max_active:2
zfs_vdev_scrub_min_active:1
zfs_vdev_sync_read_max_active:10
zfs_vdev_sync_read_min_active:10
zfs_vdev_sync_write_max_active:10
zfs_vdev_sync_write_min_active:10
zfs_vdev_write_gap_limit:4096
zfs_zevent_cols:80
zfs_zevent_console:0
zfs_zevent_len_max:768
zil_replay_disable:0
zil_slog_limit:1048576
zio_bulk_flags:0
zio_delay_max:30000
zio_injection_enabled:0
zio_requeue_io_start_cut_in_line:1
zvol_inhibit_dev:0
zvol_major:230
zvol_max_discard_blocks:16384
zvol_threads:32

man /usr/share/man/man5/zfs-module-parameters.5.gz

# zpool get all zp1
NAME  PROPERTY               VALUE                  SOURCE
zp1   size                   40T                    -
zp1   capacity               2%                     -
zp1   altroot                -                      default
zp1   health                 ONLINE                 -
zp1   guid                   15254203672861282738   default
zp1   version                -                      default
zp1   bootfs                 -                      default
zp1   delegation             on                     default
zp1   autoreplace            off                    default
zp1   cachefile              -                      default
zp1   failmode               wait                   default
zp1   listsnapshots          off                    default
zp1   autoexpand             off                    default
zp1   dedupditto             0                      default
zp1   dedupratio             1.00x                  -
zp1   free                   39.0T                  -
zp1   allocated              995G                   -
zp1   readonly               off                    -
zp1   ashift                 12                     local
zp1   comment                -                      default
zp1   expandsize             0                      -
zp1   freeing                0                      default
zp1   feature@async_destroy  enabled                local
zp1   feature@empty_bpobj    active                 local
zp1   feature@lz4_compress   active                 local

# zfs get all zp1/data_a0
NAME         PROPERTY              VALUE                  SOURCE
zp1/data_a0  type                  filesystem             -
zp1/data_a0  creation              Thu Dec 18 10:30 2014  -
zp1/data_a0  used                  98.8G                  -
zp1/data_a0  available             34.1T                  -
zp1/data_a0  referenced            98.8G                  -
zp1/data_a0  compressratio         1.00x                  -
zp1/data_a0  mounted               yes                    -
zp1/data_a0  quota                 none                   default
zp1/data_a0  reservation           none                   default
zp1/data_a0  recordsize            128K                   default
zp1/data_a0  mountpoint            /data_a0               local
zp1/data_a0  sharenfs              off                    default
zp1/data_a0  checksum              on                     default
zp1/data_a0  compression           off                    local
zp1/data_a0  atime                 off                    inherited from zp1
zp1/data_a0  devices               on                     default
zp1/data_a0  exec                  on                     default
zp1/data_a0  setuid                on                     default
zp1/data_a0  readonly              off                    default
zp1/data_a0  zoned                 off                    default
zp1/data_a0  snapdir               hidden                 default
zp1/data_a0  aclinherit            restricted             default
zp1/data_a0  canmount              on                     default
zp1/data_a0  xattr                 sa                     local
zp1/data_a0  copies                1                      default
zp1/data_a0  version               5                      -
zp1/data_a0  utf8only              off                    -
zp1/data_a0  normalization         none                   -
zp1/data_a0  casesensitivity       sensitive              -
zp1/data_a0  vscan                 off                    default
zp1/data_a0  nbmand                off                    default
zp1/data_a0  sharesmb              off                    default
zp1/data_a0  refquota              none                   default
zp1/data_a0  refreservation        none                   default
zp1/data_a0  primarycache          metadata               local
zp1/data_a0  secondarycache        all                    local
zp1/data_a0  usedbysnapshots       0                      -
zp1/data_a0  usedbydataset         98.8G                  -
zp1/data_a0  usedbychildren        0                      -
zp1/data_a0  usedbyrefreservation  0                      -
zp1/data_a0  logbias               latency                default
zp1/data_a0  dedup                 off                    default
zp1/data_a0  mlslabel              none                   default
zp1/data_a0  sync                  standard               default
zp1/data_a0  refcompressratio      1.00x                  -
zp1/data_a0  written               98.8G                  -
zp1/data_a0  logicalused           98.7G                  -
zp1/data_a0  logicalreferenced     98.7G                  -
zp1/data_a0  snapdev               hidden                 default
zp1/data_a0  acltype               off                    default
zp1/data_a0  context               none                   default
zp1/data_a0  fscontext             none                   default
zp1/data_a0  defcontext            none                   default
zp1/data_a0  rootcontext           none                   default
zp1/data_a0  relatime              off                    default

       zfs_arc_shrink_shift (int)
                   log2(fraction of arc to reclaim)
                   Default value: 5.

Description: Semi-regular spikes in I/O latency on an SSD postgres server.
Analysis: The customer reported multi-second I/O latency for a server with flash memory-based solid state disks (SSDs). Since this SSD type was new in production, it was feared that there may be a new drive or firmware problem causing high latency. ZFS latency counters, measured at the VFS interface, confirmed that I/O latency was dismal, sometimes reaching 10 seconds for I/O. The DTrace-based iosnoop tool (DTraceToolkit) was used to trace at the block device level, however, no seriously slow I/O was observed from the SSDs. I plotted the iosnoop traces using R for evidence of queueing behind TXG flushes, but they didn’t support that theory either.
This was difficult to investigate since the slow I/O was intermittent, sometimes only occurring once per hour. Instead of a typical interactive investigation, I developed various ways to log activity from DTrace and kstats, so that clues for the issue could be examined afterwards from the logs. This included capturing which processes were executed using execsnoop, and dumping ZFS metrics from kstat, including arcstats. This showed that various maintenance processes were executing during the hour, and, the ZFS ARC, which was around 210 Gbytes, would sometimes drop by around 6 Gbytes. Having worked performance issues with shrinking ARCs before, I developed a DTrace script to trace ARC reaping along with process execution, and found that it was a match with a cp(1) command. This was part of the maintenance task, which was copying a 30 Gbyte file, hitting the ARC limit and triggering an ARC shrink. Shrinking involves holding ARC hash locks, which can cause latency, especially when shrinking 6 Gbytes worth of buffers. The zfs:zfs_arc_shrink_shift tunable was adjusted to reduce the shrink size, which also made them more frequent. The worst-case I/O improved from 10s to 100ms.

ARC shrink shift
Every second a process runs which checks if data can be removed from the ARC and evicts it. Default max 1/32nd of the ARC can be evicted at a time. This is limited because evicting large amounts of data from ARC stalls all other processes. Back when 8GB was a lot of memory 1/32nd meant 256MB max at a time. When you have 196GB of memory 1/32nd is 6.3GB, which can cause up to 20-30 seconds of unresponsiveness (depending on the record size).
This 1/32nd needs to be changed to make sure the max is set to ~100-200MB again, by adding the following to /etc/system:
set zfs:zfs_arc_shrink_shift=11
(where 11 is 1/2 11 or 1/2048th, 10 is  1/2 10 or 1/1024th etc. Change depending on amount of RAM in your system).

       zfs_vdev_async_write_active_min_dirty_percent (int)
                   When  the  pool  has  less  than  zfs_vdev_async_write_active_min_dirty_percent  dirty  data,   use
                   zfs_vdev_async_write_min_active to limit active async writes.  If the dirty data is between min and
                   max, the active I/O limit is linearly interpolated. See the section "ZFS I/O SCHEDULER".
                   Default value: 30.
       zfs_vdev_async_write_active_max_dirty_percent (int)
                   When  the  pool  has  more  than  zfs_vdev_async_write_active_max_dirty_percent  dirty  data,   use
                   zfs_vdev_async_write_max_active to limit active async writes.  If the dirty data is between min and
                   max, the active I/O limit is linearly interpolated. See the section "ZFS I/O SCHEDULER".
                   Default value: 60.
       zfs_vdev_async_write_max_active (int)
                   Maxium asynchronous write I/Os active to each device.  See the section "ZFS I/O SCHEDULER".
                   Default value: 10.
       zfs_vdev_async_write_min_active (int)
                   Minimum asynchronous write I/Os active to each device.  See the section "ZFS I/O SCHEDULER".
                   Default value: 1.

              |              o---------| <-- zfs_vdev_async_write_max_active
         ^    |             /^         |
         |    |            / |         |
       active |           /  |         |
        I/O   |          /   |         |
       count  |         /    |         |
              |        /     |         |
              |-------o      |         | <-- zfs_vdev_async_write_min_active
             0|_______^______|_________|
              0%      |      |       100% of zfs_dirty_data_max
                      |      |
                      |      ‘-- zfs_vdev_async_write_active_max_dirty_percent
                      ‘--------- zfs_vdev_async_write_active_min_dirty_percent

# free
             total       used       free     shared    buffers     cached
Mem:     396856808  228812456  168044352   21633868      58744   45380060

# cat /proc/spl/kstat/zfs/arcstats |grep size
size                            4    19751851104

# echo 140000000000 > /sys/module/zfs/parameters/zfs_arc_max

zfs_dirty_data_max 降到 arc max 的 1/5 = 28000000000 (可动态调整)

zfs_vdev_async_write_active_min_dirty_percent=10
zfs_vdev_async_write_active_max_dirty_percent=30  (务必小于zfs_delay_min_dirty_percent)
zfs_delay_min_dirty_percent=60

# cd /sys/module/zfs/parameters/
# echo 140000000000 >zfs_arc_max
# echo 28000000000 >zfs_dirty_data_max
# echo 10 > zfs_vdev_async_write_active_min_dirty_percent
# echo 30 > zfs_vdev_async_write_active_max_dirty_percent
# echo 60 > zfs_delay_min_dirty_percent
# echo 11 > zfs_arc_shrink_shift

# vi /etc/modprobe.d/zfs.conf
options zfs zfs_arc_max=140000000000
options zfs zfs_dirty_data_max=28000000000
options zfs zfs_vdev_async_write_active_min_dirty_percent=10
options zfs zfs_vdev_async_write_active_max_dirty_percent=30
options zfs zfs_delay_min_dirty_percent=60
options zfs zfs_arc_shrink_shift=11

# ps -e --width=1024 -o pid,%mem,rss,size,sz,vsz,cmd --sort rss
rss        RSS      resident set size, the non-swapped physical memory that a task has used (in kiloBytes).
                    (alias rssize, rsz).
size       SZ       approximate amount of swap space that would be required if the process were to dirty all writable
                    pages and then be swapped out. This number is very rough!
sz         SZ       size in physical pages of the core image of the process. This includes text, data, and stack
                    space. Device mappings are currently excluded; this is subject to change. See vsz and rss.
vsz        VSZ      virtual memory size of the process in KiB (1024-byte units). Device mappings are currently
                    excluded; this is subject to change. (alias vsize).

06:10:01 AM kbmemfree kbmemused  %memused kbbuffers  kbcached  kbcommit   %commit
06:20:01 AM 219447748 177409060     44.70     23196  22965260  27351828      6.75
06:30:01 AM 219304016 177552792     44.74     24628  23080756  27348820      6.75
06:40:01 AM 218698000 178158808     44.89     26276  23638736  27365764      6.75
06:50:01 AM 218454732 178402076     44.95     27588  23852552  27365664      6.75
07:00:01 AM 218211060 178645748     45.02     28840  24066384  27365736      6.75
07:10:01 AM 218006588 178850220     45.07     30144  24231036  27366528      6.75
07:20:01 AM 217784072 179072736     45.12     31424  24412084  27365496      6.75
07:30:01 AM 217128620 179728188     45.29     32752  24970064  27370048      6.75
07:40:01 AM 216704964 180151844     45.39     34372  25331396  27369700      6.75
07:50:01 AM 216372456 180484352     45.48     35740  25610760  27371348      6.75
08:00:01 AM 216028392 180828416     45.57     37060  25890136  27393748      6.76
08:10:01 AM 214706196 182150612     45.90     38808  27120088  27400288      6.76
08:20:01 AM 213981920 182874888     46.08     42712  27798924  27413000      6.76
08:30:01 AM 213551104 183305704     46.19     44268  28193028  27411516      6.76

vfs_cache_pressure
------------------

This percentage value controls the tendency of the kernel to reclaim
the memory which is used for caching of directory and inode objects.

At the default value of vfs_cache_pressure=100 the kernel will attempt to
reclaim dentries and inodes at a "fair" rate with respect to pagecache and
swapcache reclaim.  Decreasing vfs_cache_pressure causes the kernel to prefer
to retain dentry and inode caches. When vfs_cache_pressure=0, the kernel will
never reclaim dentries and inodes due to memory pressure and this can easily
lead to out-of-memory conditions. Increasing vfs_cache_pressure beyond 100
causes the kernel to prefer to reclaim dentries and inodes.

Increasing vfs_cache_pressure significantly beyond 100 may have negative
performance impact. Reclaim code needs to take various locks to find freeable
directory and inode objects. With vfs_cache_pressure=1000, it will look for
ten times more freeable objects than there are.

# cat /proc/meminfo |grep -i -E "dirt|back"
Dirty:                 0 kB
Writeback:             0 kB
WritebackTmp:          0 kB

==============================================================

dirty_background_bytes

Contains the amount of dirty memory at which the background kernel
flusher threads will start writeback.

If dirty_background_bytes is written, dirty_background_ratio becomes a function
of its value (dirty_background_bytes / the amount of dirtyable system memory).

==============================================================

dirty_background_ratio

Contains, as a percentage of total system memory, the number of pages at which
the background kernel flusher threads will start writing out dirty data.

==============================================================

dirty_bytes

Contains the amount of dirty memory at which a process generating disk writes
will itself start writeback.

If dirty_bytes is written, dirty_ratio becomes a function of its value
(dirty_bytes / the amount of dirtyable system memory).

Note: the minimum value allowed for dirty_bytes is two pages (in bytes); any
value lower than this limit will be ignored and the old configuration will be
retained.

==============================================================

dirty_expire_centisecs

This tunable is used to define when dirty data is old enough to be eligible
for writeout by the kernel flusher threads.  It is expressed in 100'ths
of a second.  Data which has been dirty in-memory for longer than this
interval will be written out next time a flusher thread wakes up.

==============================================================

dirty_ratio

Contains, as a percentage of total system memory, the number of pages at which
a process which is generating disk writes will itself start writing out dirty
data.

==============================================================

dirty_writeback_centisecs

The kernel flusher threads will periodically wake up and write `old' data
out to disk.  This tunable expresses the interval between those wakeups, in
100'ths of a second.

Setting this to zero disables periodic writeback altogether.

/usr/share/doc/kernel-doc-2.6.32/Documentation/sysctl/vm.txt
drop_caches

Writing to this will cause the kernel to drop clean caches, dentries and
inodes from memory, causing that memory to become free.

To free pagecache:
        echo 1 > /proc/sys/vm/drop_caches
To free dentries and inodes:
        echo 2 > /proc/sys/vm/drop_caches
To free pagecache, dentries and inodes:
        echo 3 > /proc/sys/vm/drop_caches

As this is a non-destructive operation and dirty objects are not freeable, the
user should run `sync' first.

crontab -e
30 4 * * * /usr/local/bin/free.sh >>/tmp/free.log 2>&1

# cat /usr/local/bin/free.sh
#!/bin/bash

. /root/.bash_profile
. /etc/profile

echo "`date +%F%T` start drop cache."
free
sync
echo 3 > /proc/sys/vm/drop_caches
echo "`date +%F%T` end drop cache."
free

sysctl -w vm.zone_reclaim_mode=1
sysctl -w vm.dirty_background_bytes=102400000
sysctl -w vm.dirty_bytes=102400000
sysctl -w vm.dirty_expire_centisecs=10
sysctl -w vm.dirty_writeback_centisecs=10
sysctl -w vm.swappiness=0
sysctl -w vm.vfs_cache_pressure=80

# vi /etc/sysctl.conf
vm.zone_reclaim_mode=1
vm.dirty_background_bytes=102400000
vm.dirty_bytes=102400000
vm.dirty_expire_centisecs=10
vm.dirty_writeback_centisecs=10
vm.swappiness=0
vm.vfs_cache_pressure=80


# cd /sys/module/zfs/parameters/
# cat zfs_arc_max 
10240000000

查看arc统计信息/proc/spl/kstat/zfs/arcstats, 可以看到metadata使用了不到2G, 所以给10G差不多了.
不够的话, 以后可以再调整.
meta_size                       4    1952531968

# cat /etc/modprobe.d/zfs.conf 
options zfs zfs_arc_max=10240000000
options zfs zfs_dirty_data_max=800000000
options zfs zfs_vdev_async_write_active_min_dirty_percent=10
options zfs zfs_vdev_async_write_active_max_dirty_percent=30
options zfs zfs_delay_min_dirty_percent=60
options zfs zfs_arc_shrink_shift=11

设置为metadata, 因为LINUX本身也带cache, 没有必要多重cache. 
zfs 和 PostgreSQL 一样有这个多重cache问题, 除非使用directIO.
# zfs set primarycache=metadata zp1
# zfs set primarycache=metadata zp1/data_a0
# zfs set primarycache=metadata zp1/data_a1
# zfs set primarycache=metadata zp1/data_b0
# zfs set primarycache=metadata zp1/data_b1
# zfs set primarycache=metadata zp1/data_c0
# zfs set primarycache=metadata zp1/data_c1
# zfs set primarycache=metadata zp1/data_ssd0
# zfs set primarycache=metadata zp1/data_ssd1

设置为与数据库块大小一致.
# zfs set recordsize=16k zp1/data_a0  wal_block_size=16k
# zfs set recordsize=8k zp1/data_a0  block_size=8k