zfsonlinux/zfs — Issue #466: Memory usage keeps going up
Author: devsk
Created: 2011-11-27 23:54:39+00:00
State: open

I don't know whether its the 3.1.1 issue or ZFS code issue, but after upgrading kernel to 3.1.1 from 2.6.39.4 and ZFS to master, I am starting to swap on a 12GB system with ARC set to 2GB, although swappiness is set to 0. I have rarely seen this (or any for that matter) system swap with swappiness set to 0.

$ cat /tmp/zfs_memory_stats.3.1.1.zcache
cat /proc/slabinfo
cat: /proc/slabinfo: No such file or directory

cat /proc/spl/kmem/slab
--------------------- cache ------------------------------------------------------- ----- slab ------ ---- object -----
name flags size alloc slabsize objsize total alloc max total alloc max
spl_vn_cache 0x00020 0 0 4096 88 0 0 7 0 0 128
spl_vn_file_cache 0x00020 0 0 4096 96 0 0 0 0 0 0
spl_zlib_workspace_cache 0x00040 0 0 8388608 268104 0 0 0 0 0 0
zio_cache 0x00020 753664 293664 16384 912 46 46 1270 782 322 21575
zio_link_cache 0x00020 0 0 4096 48 0 0 495 0 0 22235
zio_buf_512 0x00020 1052508160 386351104 32768 512 32120 32120 45244 995720 754592 1402540
zio_data_buf_512 0x00020 0 0 32768 512 0 0 50 0 0 1540
zio_buf_1024 0x00040 360448 50176 32768 1024 11 11 153 231 49 3200
zio_data_buf_1024 0x00040 0 0 32768 1024 0 0 27 0 0 551
zio_buf_1536 0x00040 458752 41472 65536 1536 7 7 55 217 27 1697
zio_data_buf_1536 0x00040 0 0 65536 1536 0 0 20 0 0 597
zio_buf_2048 0x00040 524288 57344 65536 2048 8 8 61 200 28 1503
zio_data_buf_2048 0x00040 0 0 65536 2048 0 0 21 0 0 522
zio_buf_2560 0x00040 458752 28160 65536 2560 7 7 71 147 11 1480
zio_data_buf_2560 0x00040 0 0 65536 2560 0 0 42 0 0 863
zio_buf_3072 0x00040 262144 21504 65536 3072 4 4 44 72 7 775
zio_data_buf_3072 0x00040 0 0 65536 3072 0 0 44 0 0 784
zio_buf_3584 0x00040 655360 17920 131072 3584 5 5 30 155 5 902
zio_data_buf_3584 0x00040 0 0 131072 3584 0 0 23 0 0 706
zio_buf_4096 0x00040 2883584 233472 262144 4096 11 11 18 341 57 535
zio_data_buf_4096 0x00040 0 0 262144 4096 0 0 15 0 0 449
zio_buf_5120 0x00040 393216 61440 131072 5120 3 3 23 63 12 480
zio_data_buf_5120 0x00040 0 0 131072 5120 0 0 28 0 0 579
zio_buf_6144 0x00040 262144 18432 131072 6144 2 2 40 36 3 717
zio_data_buf_6144 0x00040 0 0 131072 6144 0 0 26 0 0 452
zio_buf_7168 0x00040 786432 50176 262144 7168 3 3 12 93 7 362
zio_data_buf_7168 0x00040 0 0 262144 7168 0 0 13 0 0 396
zio_buf_8192 0x00040 786432 32768 262144 8192 3 3 16 63 4 321
zio_data_buf_8192 0x00040 0 0 262144 8192 0 0 17 0 0 352
zio_buf_10240 0x00040 1048576 71680 262144 10240 4 4 15 84 7 299
zio_data_buf_10240 0x00040 0 0 262144 10240 0 0 20 0 0 410
zio_buf_12288 0x00040 1048576 61440 524288 12288 2 2 16 62 5 495
zio_data_buf_12288 0x00040 0 0 524288 12288 0 0 12 0 0 362
zio_buf_14336 0x00040 1048576 86016 524288 14336 2 2 10 62 6 292
zio_data_buf_14336 0x00040 0 0 524288 14336 0 0 10 0 0 298
zio_buf_16384 0x00040 808976384 416907264 524288 16384 1543 1543 1829 38575 25446 45704
zio_data_buf_16384 0x00040 0 0 524288 16384 0 0 13 0 0 303
zio_buf_20480 0x00040 0 0 524288 20480 0 0 17 0 0 351
zio_data_buf_20480 0x00040 0 0 524288 20480 0 0 15 0 0 305
zio_buf_24576 0x00040 524288 24576 524288 24576 1 1 40 18 1 704
zio_data_buf_24576 0x00040 0 0 524288 24576 0 0 17 0 0 304
zio_buf_28672 0x00040 0 0 1048576 28672 0 0 17 0 0 509
zio_data_buf_28672 0x00040 0 0 1048576 28672 0 0 10 0 0 282
zio_buf_32768 0x00040 2097152 65536 1048576 32768 2 2 19 56 2 511
zio_data_buf_32768 0x00040 0 0 1048576 32768 0 0 10 0 0 270
zio_buf_36864 0x00040 0 0 1048576 36864 0 0 15 0 0 370
zio_data_buf_36864 0x00040 0 0 1048576 36864 0 0 11 0 0 263
zio_buf_40960 0x00040 0 0 1048576 40960 0 0 14 0 0 302
zio_data_buf_40960 0x00040 0 0 1048576 40960 0 0 12 0 0 259
zio_buf_45056 0x00040 0 0 1048576 45056 0 0 84 0 0 1760
zio_data_buf_45056 0x00040 0 0 1048576 45056 0 0 13 0 0 260
zio_buf_49152 0x00040 0 0 1048576 49152 0 0 14 0 0 258
zio_data_buf_49152 0x00040 0 0 1048576 49152 0 0 14 0 0 260
zio_buf_53248 0x00040 0 0 1048576 53248 0 0 15 0 0 262
zio_data_buf_53248 0x00040 1048576 53248 1048576 53248 1 1 15 18 1 256
zio_buf_57344 0x00040 0 0 1048576 57344 0 0 15 0 0 255
zio_data_buf_57344 0x00040 0 0 1048576 57344 0 0 16 0 0 264
zio_buf_61440 0x00040 2097152 61440 2097152 61440 1 1 9 31 1 261
zio_data_buf_61440 0x00040 0 0 2097152 61440 0 0 9 0 0 256
zio_buf_65536 0x00040 0 0 2097152 65536 0 0 9 0 0 256
zio_data_buf_65536 0x00040 0 0 2097152 65536 0 0 9 0 0 259
zio_buf_69632 0x00040 0 0 2097152 69632 0 0 10 0 0 253
zio_data_buf_69632 0x00040 0 0 2097152 69632 0 0 10 0 0 260
zio_buf_73728 0x00040 0 0 2097152 73728 0 0 11 0 0 267
zio_data_buf_73728 0x00040 0 0 2097152 73728 0 0 10 0 0 259
zio_buf_77824 0x00040 0 0 2097152 77824 0 0 11 0 0 256
zio_data_buf_77824 0x00040 0 0 2097152 77824 0 0 11 0 0 260
zio_buf_81920 0x00040 0 0 2097152 81920 0 0 11 0 0 255
zio_data_buf_81920 0x00040 0 0 2097152 81920 0 0 11 0 0 256
zio_buf_86016 0x00040 0 0 2097152 86016 0 0 12 0 0 255
zio_data_buf_86016 0x00040 2097152 86016 2097152 86016 1 1 12 23 1 256
zio_buf_90112 0x00040 0 0 2097152 90112 0 0 16 0 0 352
zio_data_buf_90112 0x00040 4194304 180224 2097152 90112 2 2 12 44 2 256
zio_buf_94208 0x00040 0 0 2097152 94208 0 0 13 0 0 258
zio_data_buf_94208 0x00040 0 0 2097152 94208 0 0 13 0 0 256
zio_buf_98304 0x00040 0 0 2097152 98304 0 0 13 0 0 253
zio_data_buf_98304 0x00040 0 0 2097152 98304 0 0 13 0 0 256
zio_buf_102400 0x00040 0 0 2097152 102400 0 0 14 0 0 263
zio_data_buf_102400 0x00040 0 0 2097152 102400 0 0 14 0 0 256
zio_buf_106496 0x00040 0 0 2097152 106496 0 0 15 0 0 256
zio_data_buf_106496 0x00040 0 0 2097152 106496 0 0 15 0 0 256
zio_buf_110592 0x00040 0 0 2097152 110592 0 0 15 0 0 263
zio_data_buf_110592 0x00040 0 0 2097152 110592 0 0 15 0 0 256
zio_buf_114688 0x00040 0 0 2097152 114688 0 0 17 0 0 287
zio_data_buf_114688 0x00040 0 0 2097152 114688 0 0 16 0 0 257
zio_buf_118784 0x00040 0 0 2097152 118784 0 0 18 0 0 290
zio_data_buf_118784 0x00040 0 0 2097152 118784 0 0 16 0 0 256
zio_buf_122880 0x00040 0 0 2097152 122880 0 0 17 0 0 262
zio_data_buf_122880 0x00040 2097152 122880 2097152 122880 1 1 17 16 1 257
zio_buf_126976 0x00040 0 0 4194304 126976 0 0 10 0 0 285
zio_data_buf_126976 0x00040 0 0 4194304 126976 0 0 9 0 0 256
zio_buf_131072 0x00040 8388608 262144 4194304 131072 2 2 39 62 2 1194
zio_data_buf_131072 0x00040 167772160 124518400 4194304 131072 40 40 207 1240 950 6401
sa_cache 0x00020 7286784 3374880 4096 80 1779 1779 42427 58707 42186 1400074
dnode_t 0x00020 898154496 712389600 16384 944 54819 54819 87622 877104 754650 1401951
dmu_buf_impl_t 0x00020 351313920 231202048 8192 296 42885 42885 60305 1029240 781088 1447315
arc_buf_hdr_t 0x00020 85090304 70180704 8192 288 10387 10387 11454 249288 243683 274888
arc_buf_t 0x00020 6279168 3001712 4096 112 1533 1533 1790 39858 26801 46532
zil_lwb_cache 0x00020 0 0 4096 192 0 0 0 0 0 0
zfs_znode_cache 0x00020 48807936 38811120 16384 920 2979 2979 82358 50643 42186 1400079

cat /proc/spl/kstat/zfs/arcstats
3 1 0x01 61 2928 15736566863 840694116479688
name type data
hits 4 27071108
misses 4 1151721
demand_data_hits 4 694598
demand_data_misses 4 14468
demand_metadata_hits 4 18878109
demand_metadata_misses 4 786355
prefetch_data_hits 4 672
prefetch_data_misses 4 9780
prefetch_metadata_hits 4 7497729
prefetch_metadata_misses 4 341118
mru_hits 4 4446043
mru_ghost_hits 4 309968
mfu_hits 4 15127085
mfu_ghost_hits 4 436465
deleted 4 578463
recycle_miss 4 1464713
mutex_miss 4 3781
evict_skip 4 114102092
evict_l2_cached 4 542301184
evict_l2_eligible 4 4999392256
evict_l2_ineligible 4 56932654592
hash_elements 4 243422
hash_elements_max 4 274016
hash_collisions 4 447730
hash_chains 4 62352
hash_chain_max 4 8
p 4 1336191760
c 4 1968435456
c_min 4 1968435456
c_max 4 1968435456
size 4 1804427328
hdr_size 4 76231680
data_size 4 543160320
other_size 4 1185007968
l2_hits 4 39179
l2_misses 4 1112523
l2_feeds 4 786617
l2_rw_clash 4 0
l2_read_bytes 4 231375872
l2_write_bytes 4 231281664
l2_writes_sent 4 1178
l2_writes_done 4 1178
l2_writes_error 4 0
l2_writes_hdr_miss 4 1
l2_evict_lock_retry 4 0
l2_evict_reading 4 0
l2_free_on_write 4 862
l2_abort_lowmem 4 54061
l2_cksum_bad 4 0
l2_io_error 4 0
l2_size 4 228327936
l2_hdr_size 4 28880
memory_throttle_count 4 1930
memory_direct_count 4 0
memory_indirect_count 4 4
arc_no_grow 4 0
arc_tempreserve 4 0
arc_loaned_bytes 4 0
arc_meta_used 4 1679473216
arc_meta_limit 4 492108864
arc_meta_max 4 3023441968

cat /proc/sys/kernel/spl/k/proc/sys/kernel/spl/kmemmem/kmem_used
cat: /proc/sys/kernel/spl/k/proc/sys/kernel/spl/kmemmem/kmem_used: No such file or directory

cat /proc/sys/kernel/spl/kmem/kmem_max
325256867

free
total used free shared buffers cached
Mem: 12276344 11845252 431092 0 122420 3943392
-/+ buffers/cache: 7779440 4496904
Swap: 12582908 845608 11737300

cat /proc/sys/vm/swappiness
0

This output is after 9days of uptime.

I plan to upgrade to 3.1.3 to see if it helps. It seems to have a shrinker patch.

Let me know if you need some more info.