dmesg - cpu_test_10162018
[ 8259] 0 8259 34113 373 69 0 0 sssd_pam
[ 8260] 0 8260 34114 380 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 408 73 0 0 sssd_ssh
[11911] 0 11911 55169 12312 74 0 0 chef-client
[17952] 110 17952 7369 206 19 0 0 qmgr
[14580] 110 14580 10570 313 25 0 0 tlsmgr
[29141] 113 29141 219163 14997 92 0 0 trace-agent
[31674] 0 31674 8844 3691 20 0 0 atop
[28475] 110 28475 7326 178 18 0 0 showq
[19445] 110 19445 7326 210 18 0 0 pickup
[23650] 992 23650 192508 115128 382 0 0 ruby
[26952] 992 26952 270105 176296 529 0 0 ruby
[27739] 992 27739 271129 185565 532 0 0 ruby
[28539] 992 28539 276249 177551 542 0 0 ruby
[29063] 992 29063 274458 178779 537 0 0 ruby
[29268] 992 29268 3731225 2362564 5855 0 0 ruby
[29458] 992 29458 269852 178295 526 0 0 ruby
[29620] 992 29620 273177 180069 536 0 0 ruby
[30048] 992 30048 273177 180758 536 0 0 ruby
[30203] 992 30203 271133 174376 526 0 0 ruby
[30527] 992 30527 269081 186473 528 0 0 ruby
[30850] 992 30850 274201 178988 539 0 0 ruby
[31035] 992 31035 269081 174873 528 0 0 ruby
[31149] 992 31149 280345 182567 550 0 0 ruby
[31346] 992 31346 273177 176434 536 0 0 ruby
[31429] 992 31429 571165 182773 1115 0 0 ruby
[31467] 992 31467 264985 173758 520 0 0 ruby
[31515] 992 31515 263961 173309 518 0 0 ruby
[31577] 992 31577 597018 196540 1167 0 0 ruby
[13409] 992 13409 267033 175473 524 0 0 ruby
[14242] 992 14242 270105 182971 530 0 0 ruby
[15343] 992 15343 270105 178352 530 0 0 ruby
[18007] 992 18007 269081 180780 528 0 0 ruby
[27547] 992 27547 262937 173768 517 0 0 ruby
[30092] 992 30092 268057 182960 527 0 0 ruby
[30822] 992 30822 572185 186771 1121 0 0 ruby
[ 3328] 992 3328 261146 174719 513 0 0 ruby
[ 4480] 992 4480 262937 176308 515 0 0 ruby
[11189] 992 11189 270105 177762 529 0 0 ruby
[14875] 992 14875 269081 173463 527 0 0 ruby
[ 9602] 992 9602 280345 179665 550 0 0 ruby
[24163] 108 24163 7872 272 19 0 0 ntpd
[27139] 992 27139 267800 178542 526 0 0 ruby
[10904] 992 10904 278297 183072 546 0 0 ruby
[17762] 992 17762 275225 171647 540 0 0 ruby
[21278] 992 21278 251416 165129 496 0 0 ruby
[23913] 0 23913 724330 166286 507 0 0 ruby
[23952] 992 23952 249368 166645 492 0 0 ruby
[25117] 992 25117 235032 154822 462 0 0 ruby
[25738] 992 25738 265752 176999 523 0 0 ruby
[26888] 992 26888 212504 132475 419 0 0 ruby
[27006] 0 27006 11715 238 29 0 0 cron
[27011] 0 27011 11715 238 29 0 0 cron
[27023] 0 27023 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27032] 0 27032 2782 156 10 0 0 bash
[27034] 0 27034 2782 155 11 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[27061] 0 27061 1479 126 8 0 0 sleep
[27069] 0 27069 1479 126 8 0 0 sleep
[28832] 992 28832 211480 132585 417 0 0 ruby
[32223] 0 32223 11715 238 29 0 0 cron
[32227] 0 32227 11715 238 29 0 0 cron
[32231] 0 32231 11715 238 29 0 0 cron
[32242] 0 32242 2782 193 10 0 0 bash
[32245] 0 32245 2782 155 11 0 0 bash
[32252] 0 32252 1479 127 8 0 0 sleep
[32273] 0 32273 2782 156 9 0 0 bash
[32281] 0 32281 1479 125 8 0 0 sleep
[ 389] 992 389 194556 115709 380 0 0 ruby
[ 1860] 993 1860 6897 141 19 0 0 nrpe
[ 1861] 993 1861 6897 129 18 0 0 nrpe
Out of memory: Kill process 29268 (ruby) score 294 or sacrifice child
Killed process 29268 (ruby) total-vm:14924900kB, anon-rss:9449764kB, file-rss:492kB
ruby invoked oom-killer: gfp_mask=0x3000d0, order=1, oom_score_adj=0
ruby cpuset=/ mems_allowed=0
CPU: 4 PID: 28539 Comm: ruby Not tainted 3.13.0-156-generic #206-Ubuntu
Hardware name: Amazon EC2 m5.2xlarge/, BIOS 1.0 10/16/2017
0000000000000000 ffff88076c409b88 ffffffff8173654e ffff8807d7755e40
0000000000000000 ffff88076c409c10 ffffffff81730abd ffff88076c409c50
ffff88076c409be8 0000000000000000 0000000000000000 0000000000000000
Call Trace:
[] dump_stack+0x64/0x80
[] dump_header+0x7f/0x1f1
[] oom_kill_process+0x201/0x360
[] ? security_capable_noaudit+0x1b/0x20
[] out_of_memory+0x471/0x4b0
[] __alloc_pages_nodemask+0xa6c/0xb90
[] copy_process.part.27+0x12b/0x1690
[] ? do_sync_read+0x60/0xa0
[] do_fork+0xe5/0x3c0
[] SyS_clone+0x16/0x20
[] stub_clone+0x69/0x90
[] ? system_call_fastpath+0x1a/0x1f
Mem-Info:
Node 0 DMA per-cpu:
CPU 0: hi: 0, btch: 1 usd: 0
CPU 1: hi: 0, btch: 1 usd: 0
CPU 2: hi: 0, btch: 1 usd: 0
CPU 3: hi: 0, btch: 1 usd: 0
CPU 4: hi: 0, btch: 1 usd: 0
CPU 5: hi: 0, btch: 1 usd: 0
CPU 6: hi: 0, btch: 1 usd: 0
CPU 7: hi: 0, btch: 1 usd: 0
Node 0 DMA32 per-cpu:
CPU 0: hi: 186, btch: 31 usd: 0
CPU 1: hi: 186, btch: 31 usd: 29
CPU 2: hi: 186, btch: 31 usd: 9
CPU 3: hi: 186, btch: 31 usd: 17
CPU 4: hi: 186, btch: 31 usd: 13
CPU 5: hi: 186, btch: 31 usd: 30
CPU 6: hi: 186, btch: 31 usd: 1
CPU 7: hi: 186, btch: 31 usd: 0
Node 0 Normal per-cpu:
CPU 0: hi: 186, btch: 31 usd: 0
CPU 1: hi: 186, btch: 31 usd: 23
CPU 2: hi: 186, btch: 31 usd: 30
CPU 3: hi: 186, btch: 31 usd: 83
CPU 4: hi: 186, btch: 31 usd: 19
CPU 5: hi: 186, btch: 31 usd: 29
CPU 6: hi: 186, btch: 31 usd: 2
CPU 7: hi: 186, btch: 31 usd: 14
active_anon:7856559 inactive_anon:39 isolated_anon:0
active_file:174 inactive_file:25 isolated_file:0
unevictable:3774 dirty:0 writeback:0 unstable:0
free:49124 slab_reclaimable:13958 slab_unreclaimable:13684
mapped:488 shmem:101 pagetables:31978 bounce:0
free_cma:0
Node 0 DMA free:15908kB min:32kB low:40kB high:48kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
lowmem_reserve[]: 0 2976 31358 31358
Node 0 DMA32 free:119752kB min:6408kB low:8008kB high:9612kB active_anon:2899536kB inactive_anon:16kB active_file:28kB inactive_file:0kB unevictable:2696kB isolated(anon):0kB isolated(file):0kB present:3129316kB managed:3050168kB mlocked:2696kB dirty:52kB writeback:0kB mapped:40kB shmem:24kB slab_reclaimable:6120kB slab_unreclaimable:5004kB kernel_stack:544kB pagetables:11608kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:317 all_unreclaimable? yes
lowmem_reserve[]: 0 0 28382 28382
Node 0 Normal free:60836kB min:61136kB low:76420kB high:91704kB active_anon:28526700kB inactive_anon:140kB active_file:668kB inactive_file:228kB unevictable:12400kB isolated(anon):0kB isolated(file):0kB present:29577216kB managed:29063528kB mlocked:12400kB dirty:0kB writeback:0kB mapped:1912kB shmem:380kB slab_reclaimable:49712kB slab_unreclaimable:49732kB kernel_stack:3960kB pagetables:116304kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:1813 all_unreclaimable? yes
lowmem_reserve[]: 0 0 0 0
Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15908kB
Node 0 DMA32: 813*4kB (UEM) 328*8kB (UEM) 373*16kB (UEM) 288*32kB (UEM) 204*64kB (UEM) 111*128kB (UEM) 59*256kB (UEM) 29*512kB (UE) 5*1024kB (UE) 4*2048kB (UE) 7*4096kB (UEMR) = 120260kB
Node 0 Normal: 4670*4kB (UEM) 4766*8kB (UM) 7*16kB (M) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 61016kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
880 total pagecache pages
0 pages in swap cache
Swap cache stats: add 0, delete 0, find 0/0
Free swap = 0kB
Total swap = 0kB
8180631 pages RAM
0 pages HighMem/MovableOnly
128422 pages reserved
[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
[ 478] 0 478 4869 74 13 0 0 upstart-udev-br
[ 482] 0 482 12392 245 27 0 -1000 systemd-udevd
[ 642] 0 642 12164 311 24 0 0 lldpd
[ 663] 0 663 3815 70 12 0 0 upstart-socket-
[ 698] 106 698 12164 160 24 0 0 lldpd
[ 736] 0 736 2555 615 8 0 0 dhclient
[ 986] 102 986 11999 201 26 0 0 dbus-daemon
[ 1026] 0 1026 8756 205 22 0 0 systemd-logind
[ 1033] 0 1033 3819 73 12 0 0 upstart-file-br
[ 1078] 0 1078 3634 164 12 0 0 getty
[ 1081] 0 1081 3634 165 12 0 0 getty
[ 1086] 0 1086 3634 166 12 0 0 getty
[ 1088] 0 1088 3634 166 11 0 0 getty
[ 1089] 0 1089 182070 2283 50 0 0 amazon-ssm-agen
[ 1091] 0 1091 3634 167 12 0 0 getty
[ 1126] 0 1126 1092 109 8 0 0 acpid
[ 1127] 0 1127 5913 191 17 0 0 cron
[ 1129] 0 1129 4785 40 12 0 0 atd
[ 1184] 0 1184 4799 149 15 0 0 irqbalance
[ 1302] 0 1302 2243 66 8 0 0 rngd
[ 1453] 0 1453 8487 156 15 0 0 memlockd
[ 1544] 1 1544 1600 112 9 0 0 uptimed
[ 1548] 0 1548 3634 174 12 0 0 getty
[ 1549] 0 1549 3196 168 12 0 0 getty
[ 7154] 0 7154 46 4 3 0 0 runsvdir
[ 7163] 0 7163 41 1 3 0 0 runsv
[ 7194] 0 7194 1085 127 8 0 0 logger
[ 7195] 0 7195 41 1 3 0 0 runsv
[ 7205] 0 7205 1085 147 8 0 0 logger
[ 9101] 998 9101 9764 1156 24 0 0 python
[ 9136] 109 9136 7478 181 18 0 0 dnsmasq
[17111] 0 17111 3211 77 11 0 0 mcelog
[20922] 0 20922 6289 200 16 0 0 master
[24780] 0 24780 5856 139 16 0 0 rpcbind
[26542] 0 26542 41 1 3 0 0 runsv
[26603] 0 26603 49 1 4 0 0 svlogd
[27249] 0 27249 41 1 3 0 0 runsv
[27398] 0 27398 49 1 4 0 0 svlogd
[29275] 0 29275 41 1 3 0 0 runsv
[29293] 0 29293 1085 79 7 0 0 logger
[ 467] 991 467 8792 1550 15 0 0 redis-server
[ 743] 0 743 41 1 3 0 0 runsv
[ 744] 0 744 49 6 4 0 0 svlogd
[ 1458] 0 1458 41 1 3 0 0 runsv
[ 1459] 0 1459 50 9 4 0 0 svlogd
[ 2423] 993 2423 6897 214 18 0 0 nrpe
[ 2427] 0 2427 19120 3367 34 0 0 ruby
[ 2683] 0 2683 10741 178 25 0 0 syslog-ng
[ 2684] 0 2684 28994 4243 62 0 0 syslog-ng
[ 2692] 0 2692 17384 2821 33 0 0 ruby
[ 2812] 0 2812 7324 353 17 0 0 nginx
[ 8224] 0 8224 41 1 3 0 0 runsv
[ 8269] 0 8269 49 1 4 0 0 svlogd
[12779] 0 12779 41 4 3 0 0 runsv
[12866] 0 12866 1085 146 8 0 0 logger
[12956] 0 12956 41 1 3 0 0 runsv
[13131] 0 13131 1085 146 8 0 0 logger
[13729] 65534 13729 84183 3774 30 0 0 memcached
[13793] 0 13793 4159 774 12 0 0 consul-template
[18314] 0 18314 29462 609 24 0 0 osqueryd
[18316] 0 18316 217702 5846 92 0 0 osqueryd
[21414] 0 21414 15344 282 34 0 -1000 sshd
[23885] 992 23885 167164 1259 39 0 0 raingutter
[23943] 992 23943 56394 22832 115 0 0 bundle
[13766] 0 13766 26264 3483 53 0 0 ruby
[29048] 999 29048 17319 5413 38 0 0 consul
[29867] 113 29867 26431 3788 54 0 0 supervisord
[29891] 113 29891 44805 8696 93 0 0 python
[29892] 113 29892 57087 9217 84 0 0 python
[29896] 113 29896 151800 9576 96 0 0 python
[26311] 33 26311 7846 447 19 0 0 nginx
[ 8234] 0 8234 39672 416 79 0 0 sssd
[ 8238] 0 8238 44386 701 90 0 0 sssd_be
[ 8258] 0 8258 37591 354 75 0 0 sssd_nss
[ 8259] 0 8259 34113 349 69 0 0 sssd_pam
[ 8260] 0 8260 34114 358 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 379 73 0 0 sssd_ssh
[11911] 0 11911 55169 12312 74 0 0 chef-client
[17952] 110 17952 7369 186 19 0 0 qmgr
[14580] 110 14580 10570 313 25 0 0 tlsmgr
[29141] 113 29141 219163 15026 92 0 0 trace-agent
[31674] 0 31674 8844 3691 20 0 0 atop
[28475] 110 28475 7326 212 18 0 0 showq
[19445] 110 19445 7326 197 18 0 0 pickup
[23650] 992 23650 192508 115218 382 0 0 ruby
[26952] 992 26952 270105 175808 529 0 0 ruby
[27739] 992 27739 271129 179704 532 0 0 ruby
[28539] 992 28539 276249 179097 542 0 0 ruby
[29063] 992 29063 274458 179048 537 0 0 ruby
[29458] 992 29458 269852 176759 526 0 0 ruby
[29620] 992 29620 273177 185890 536 0 0 ruby
[30048] 992 30048 273177 180964 536 0 0 ruby
[30203] 992 30203 271133 174368 526 0 0 ruby
[30527] 992 30527 269081 176413 528 0 0 ruby
[30850] 992 30850 274201 180538 539 0 0 ruby
[31035] 992 31035 269081 177030 528 0 0 ruby
[31149] 992 31149 280345 184619 550 0 0 ruby
[31346] 992 31346 273177 176857 536 0 0 ruby
[31429] 992 31429 571165 185309 1115 0 0 ruby
[31467] 992 31467 264985 174396 520 0 0 ruby
[31515] 992 31515 263961 175005 518 0 0 ruby
[31577] 992 31577 597018 196659 1167 0 0 ruby
[13409] 992 13409 267033 175696 524 0 0 ruby
[14242] 992 14242 270105 178988 530 0 0 ruby
[15343] 992 15343 270105 174364 530 0 0 ruby
[18007] 992 18007 1760025 1123932 2845 0 0 ruby
[27547] 992 27547 262937 177079 517 0 0 ruby
[30092] 992 30092 268057 176739 527 0 0 ruby
[30822] 992 30822 572185 183786 1121 0 0 ruby
[ 3328] 992 3328 261146 168709 513 0 0 ruby
[ 4480] 992 4480 262937 170131 515 0 0 ruby
[11189] 992 11189 270105 179043 529 0 0 ruby
[14875] 992 14875 269081 173720 527 0 0 ruby
[ 9602] 992 9602 280345 178414 550 0 0 ruby
[24163] 108 24163 7872 252 19 0 0 ntpd
[27139] 992 27139 267800 175584 526 0 0 ruby
[10904] 992 10904 278297 177517 546 0 0 ruby
[17762] 992 17762 275225 173098 540 0 0 ruby
[21278] 992 21278 251416 160230 496 0 0 ruby
[23913] 0 23913 724330 167228 509 0 0 ruby
[23952] 992 23952 249368 168048 492 0 0 ruby
[25117] 992 25117 236056 153131 465 0 0 ruby
[25738] 992 25738 265752 168056 523 0 0 ruby
[26888] 992 26888 2246168 1442034 3605 0 0 ruby
[27006] 0 27006 11715 238 29 0 0 cron
[27011] 0 27011 11715 238 29 0 0 cron
[27023] 0 27023 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27032] 0 27032 2782 156 10 0 0 bash
[27034] 0 27034 2782 155 11 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[27061] 0 27061 1479 126 8 0 0 sleep
[27069] 0 27069 1479 126 8 0 0 sleep
[28832] 992 28832 212504 133212 418 0 0 ruby
[ 389] 992 389 210456 130747 414 0 0 ruby
[ 1987] 992 1987 211737 131265 416 0 0 ruby
[ 2115] 0 2115 11715 240 29 0 0 cron
[ 2116] 0 2116 11715 240 29 0 0 cron
[ 2118] 0 2118 11715 240 29 0 0 cron
[ 2119] 0 2119 11715 240 29 0 0 cron
[ 2120] 0 2120 11715 240 29 0 0 cron
[ 2121] 0 2121 11715 240 29 0 0 cron
[ 2122] 0 2122 11715 240 29 0 0 cron
[ 2123] 0 2123 11715 240 29 0 0 cron
[ 2126] 0 2126 11715 240 29 0 0 cron
[ 2129] 0 2129 11715 240 29 0 0 cron
[ 2130] 0 2130 11715 240 29 0 0 cron
[ 2131] 0 2131 11715 240 29 0 0 cron
[ 2132] 0 2132 11715 240 29 0 0 cron
[ 2133] 0 2133 11715 240 29 0 0 cron
[ 2134] 0 2134 2782 155 10 0 0 bash
[ 2135] 0 2135 2782 153 10 0 0 bash
[ 2136] 0 2136 2782 156 11 0 0 bash
[ 2138] 0 2138 2782 156 10 0 0 bash
[ 2139] 0 2139 2782 156 10 0 0 bash
[ 2140] 0 2140 2782 154 10 0 0 bash
[ 2141] 0 2141 2782 156 10 0 0 bash
[ 2143] 0 2143 2782 155 10 0 0 bash
[ 2144] 0 2144 2782 156 10 0 0 bash
[ 2145] 0 2145 2782 156 10 0 0 bash
[ 2147] 0 2147 2782 154 10 0 0 bash
[ 2149] 0 2149 2782 154 10 0 0 bash
[ 2150] 0 2150 2782 156 11 0 0 bash
[ 2152] 0 2152 2782 156 10 0 0 bash
[ 2153] 0 2153 1479 127 8 0 0 sleep
[ 2155] 0 2155 1479 126 8 0 0 sleep
[ 2156] 0 2156 1479 126 8 0 0 sleep
[ 2157] 0 2157 1479 126 8 0 0 sleep
[ 2158] 0 2158 1479 126 7 0 0 sleep
[ 2159] 0 2159 1479 127 8 0 0 sleep
[ 2160] 0 2160 1479 127 8 0 0 sleep
[ 2162] 0 2162 1479 127 8 0 0 sleep
[ 2163] 0 2163 1479 126 8 0 0 sleep
[ 2164] 0 2164 1479 126 8 0 0 sleep
[ 2165] 0 2165 1479 127 8 0 0 sleep
[ 2166] 0 2166 1479 127 8 0 0 sleep
[ 2167] 0 2167 1479 126 7 0 0 sleep
[ 2169] 0 2169 1479 126 7 0 0 sleep
[ 2849] 999 2849 4492 206 15 0 0 memcached
[ 2850] 999 2850 4492 78 14 0 0 memcached
[ 2852] 999 2852 2805 163 11 0 0 nc
Out of memory: Kill process 26888 (ruby) score 179 or sacrifice child
Killed process 26888 (ruby) total-vm:8984672kB, anon-rss:5767232kB, file-rss:904kB
ruby invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
ruby cpuset=/ mems_allowed=0
CPU: 2 PID: 15343 Comm: ruby Not tainted 3.13.0-156-generic #206-Ubuntu
Hardware name: Amazon EC2 m5.2xlarge/, BIOS 1.0 10/16/2017
0000000000000000 ffff88073236b960 ffffffff8173654e ffff880103349790
0000000000000000 ffff88073236b9e8 ffffffff81730abd ffffffff8106d6c6
ffff88073236b9c0 ffffffff810cc81c 0000000001320122 0000000000000006
Call Trace:
[] dump_stack+0x64/0x80
[] dump_header+0x7f/0x1f1
[] ? put_online_cpus+0x56/0x80
[] ? rcu_oom_notify+0xcc/0xf0
[] oom_kill_process+0x201/0x360
[] ? security_capable_noaudit+0x1b/0x20
[] out_of_memory+0x471/0x4b0
[] __alloc_pages_nodemask+0xa6c/0xb90
[] alloc_pages_current+0xa3/0x160
[] __page_cache_alloc+0x97/0xc0
[] filemap_fault+0x185/0x400
[] __do_fault+0x75/0x560
[] handle_mm_fault+0x234/0xeb0
[] __do_page_fault+0x183/0x570
[] ? SYSC_recvfrom+0x127/0x160
[] ? poll_select_copy_remaining+0xdc/0x130
[] do_page_fault+0x1a/0x70
[] ? SyS_select+0xbd/0x100
[] do_async_page_fault+0x29/0xe0
[] async_page_fault+0x28/0x30
Mem-Info:
Node 0 DMA per-cpu:
CPU 0: hi: 0, btch: 1 usd: 0
CPU 1: hi: 0, btch: 1 usd: 0
CPU 2: hi: 0, btch: 1 usd: 0
CPU 3: hi: 0, btch: 1 usd: 0
CPU 4: hi: 0, btch: 1 usd: 0
CPU 5: hi: 0, btch: 1 usd: 0
CPU 6: hi: 0, btch: 1 usd: 0
CPU 7: hi: 0, btch: 1 usd: 0
Node 0 DMA32 per-cpu:
CPU 0: hi: 186, btch: 31 usd: 0
CPU 1: hi: 186, btch: 31 usd: 0
CPU 2: hi: 186, btch: 31 usd: 0
CPU 3: hi: 186, btch: 31 usd: 0
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 0
CPU 6: hi: 186, btch: 31 usd: 0
CPU 7: hi: 186, btch: 31 usd: 0
Node 0 Normal per-cpu:
CPU 0: hi: 186, btch: 31 usd: 0
CPU 1: hi: 186, btch: 31 usd: 0
CPU 2: hi: 186, btch: 31 usd: 0
CPU 3: hi: 186, btch: 31 usd: 0
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 0
CPU 6: hi: 186, btch: 31 usd: 0
CPU 7: hi: 186, btch: 31 usd: 0
active_anon:7857431 inactive_anon:39 isolated_anon:0
active_file:169 inactive_file:0 isolated_file:0
unevictable:3774 dirty:115 writeback:0 unstable:0
free:49174 slab_reclaimable:14042 slab_unreclaimable:13716
mapped:248 shmem:101 pagetables:31504 bounce:0
free_cma:0
Node 0 DMA free:15908kB min:32kB low:40kB high:48kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
lowmem_reserve[]: 0 2976 31358 31358
Node 0 DMA32 free:119872kB min:6408kB low:8008kB high:9612kB active_anon:2898976kB inactive_anon:16kB active_file:108kB inactive_file:0kB unevictable:2696kB isolated(anon):0kB isolated(file):0kB present:3129316kB managed:3050168kB mlocked:2696kB dirty:24kB writeback:0kB mapped:128kB shmem:24kB slab_reclaimable:6104kB slab_unreclaimable:4980kB kernel_stack:528kB pagetables:11712kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:163 all_unreclaimable? yes
lowmem_reserve[]: 0 0 28382 28382
Node 0 Normal free:60916kB min:61136kB low:76420kB high:91704kB active_anon:28530748kB inactive_anon:140kB active_file:568kB inactive_file:28kB unevictable:12400kB isolated(anon):0kB isolated(file):0kB present:29577216kB managed:29063528kB mlocked:12400kB dirty:436kB writeback:0kB mapped:864kB shmem:380kB slab_reclaimable:50064kB slab_unreclaimable:49884kB kernel_stack:3888kB pagetables:114304kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:918 all_unreclaimable? yes
lowmem_reserve[]: 0 0 0 0
Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15908kB
Node 0 DMA32: 661*4kB (UEM) 291*8kB (UEM) 382*16kB (UEM) 288*32kB (UEM) 218*64kB (UEM) 111*128kB (UEM) 59*256kB (UEM) 29*512kB (UE) 5*1024kB (UE) 4*2048kB (UE) 7*4096kB (UEMR) = 120396kB
Node 0 Normal: 6753*4kB (UEM) 3790*8kB (UM) 7*16kB (UM) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 61540kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
367 total pagecache pages
0 pages in swap cache
Swap cache stats: add 0, delete 0, find 0/0
Free swap = 0kB
Total swap = 0kB
8180631 pages RAM
0 pages HighMem/MovableOnly
128422 pages reserved
[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
[ 478] 0 478 4869 74 13 0 0 upstart-udev-br
[ 482] 0 482 12392 245 27 0 -1000 systemd-udevd
[ 642] 0 642 12164 311 24 0 0 lldpd
[ 663] 0 663 3815 70 12 0 0 upstart-socket-
[ 698] 106 698 12164 138 24 0 0 lldpd
[ 736] 0 736 2555 615 8 0 0 dhclient
[ 986] 102 986 11999 201 26 0 0 dbus-daemon
[ 1026] 0 1026 8756 205 22 0 0 systemd-logind
[ 1033] 0 1033 3819 73 12 0 0 upstart-file-br
[ 1078] 0 1078 3634 164 12 0 0 getty
[ 1081] 0 1081 3634 165 12 0 0 getty
[ 1086] 0 1086 3634 166 12 0 0 getty
[ 1088] 0 1088 3634 166 11 0 0 getty
[ 1089] 0 1089 182070 2283 50 0 0 amazon-ssm-agen
[ 1091] 0 1091 3634 167 12 0 0 getty
[ 1126] 0 1126 1092 109 8 0 0 acpid
[ 1127] 0 1127 5913 189 17 0 0 cron
[ 1129] 0 1129 4785 40 12 0 0 atd
[ 1184] 0 1184 4799 148 15 0 0 irqbalance
[ 1302] 0 1302 2243 62 8 0 0 rngd
[ 1453] 0 1453 8487 156 15 0 0 memlockd
[ 1544] 1 1544 1600 112 9 0 0 uptimed
[ 1548] 0 1548 3634 174 12 0 0 getty
[ 1549] 0 1549 3196 168 12 0 0 getty
[ 7154] 0 7154 46 4 3 0 0 runsvdir
[ 7163] 0 7163 41 1 3 0 0 runsv
[ 7194] 0 7194 1085 127 8 0 0 logger
[ 7195] 0 7195 41 1 3 0 0 runsv
[ 7205] 0 7205 1085 147 8 0 0 logger
[ 9101] 998 9101 9764 1156 24 0 0 python
[ 9136] 109 9136 7478 203 18 0 0 dnsmasq
[17111] 0 17111 3211 77 11 0 0 mcelog
[20922] 0 20922 6289 200 16 0 0 master
[24780] 0 24780 5856 178 16 0 0 rpcbind
[26542] 0 26542 41 1 3 0 0 runsv
[26603] 0 26603 49 1 4 0 0 svlogd
[27249] 0 27249 41 1 3 0 0 runsv
[27398] 0 27398 49 1 4 0 0 svlogd
[29275] 0 29275 41 1 3 0 0 runsv
[29293] 0 29293 1085 79 7 0 0 logger
[ 467] 991 467 8792 1533 15 0 0 redis-server
[ 743] 0 743 41 1 3 0 0 runsv
[ 744] 0 744 49 6 4 0 0 svlogd
[ 1458] 0 1458 41 1 3 0 0 runsv
[ 1459] 0 1459 50 9 4 0 0 svlogd
[ 2423] 993 2423 6897 214 18 0 0 nrpe
[ 2427] 0 2427 19120 3358 34 0 0 ruby
[ 2683] 0 2683 10741 178 25 0 0 syslog-ng
[ 2684] 0 2684 28994 4247 62 0 0 syslog-ng
[ 2692] 0 2692 17384 2809 33 0 0 ruby
[ 2812] 0 2812 7324 353 17 0 0 nginx
[ 8224] 0 8224 41 1 3 0 0 runsv
[ 8269] 0 8269 49 1 4 0 0 svlogd
[12779] 0 12779 41 4 3 0 0 runsv
[12866] 0 12866 1085 146 8 0 0 logger
[12956] 0 12956 41 1 3 0 0 runsv
[13131] 0 13131 1085 146 8 0 0 logger
[13729] 65534 13729 84183 3797 30 0 0 memcached
[13793] 0 13793 4159 774 12 0 0 consul-template
[18314] 0 18314 29462 655 24 0 0 osqueryd
[18316] 0 18316 217702 5846 92 0 0 osqueryd
[21414] 0 21414 15344 282 34 0 -1000 sshd
[23885] 992 23885 167164 1243 39 0 0 raingutter
[23943] 992 23943 56394 22824 115 0 0 bundle
[13766] 0 13766 26264 3448 53 0 0 ruby
[29048] 999 29048 17319 5413 38 0 0 consul
[29867] 113 29867 26431 3750 54 0 0 supervisord
[29891] 113 29891 44805 8700 93 0 0 python
[29892] 113 29892 57087 9210 84 0 0 python
[29896] 113 29896 151800 9573 96 0 0 python
[26311] 33 26311 7846 447 19 0 0 nginx
[ 8234] 0 8234 39672 417 79 0 0 sssd
[ 8238] 0 8238 44386 689 90 0 0 sssd_be
[ 8258] 0 8258 37591 354 75 0 0 sssd_nss
[ 8259] 0 8259 34113 350 69 0 0 sssd_pam
[ 8260] 0 8260 34114 357 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 380 73 0 0 sssd_ssh
[11911] 0 11911 55169 12312 74 0 0 chef-client
[17952] 110 17952 7369 186 19 0 0 qmgr
[14580] 110 14580 10570 313 25 0 0 tlsmgr
[29141] 113 29141 219163 15090 92 0 0 trace-agent
[31674] 0 31674 8844 3691 20 0 0 atop
[28475] 110 28475 7326 212 18 0 0 showq
[19445] 110 19445 7326 197 18 0 0 pickup
[23650] 992 23650 192508 115135 382 0 0 ruby
[26952] 992 26952 270105 176599 529 0 0 ruby
[27739] 992 27739 271129 180201 532 0 0 ruby
[28539] 992 28539 276249 179122 542 0 0 ruby
[29063] 992 29063 274458 180344 537 0 0 ruby
[29458] 992 29458 269852 176642 526 0 0 ruby
[29620] 992 29620 273177 177732 536 0 0 ruby
[30048] 992 30048 273177 181154 536 0 0 ruby
[30203] 992 30203 271133 172741 526 0 0 ruby
[30527] 992 30527 269081 177337 528 0 0 ruby
[30850] 992 30850 274201 178135 539 0 0 ruby
[31035] 992 31035 269081 176927 528 0 0 ruby
[31149] 992 31149 280345 182103 550 0 0 ruby
[31346] 992 31346 273177 177478 536 0 0 ruby
[31429] 992 31429 571165 181713 1115 0 0 ruby
[31467] 992 31467 264985 174258 520 0 0 ruby
[31515] 992 31515 263961 170207 518 0 0 ruby
[31577] 992 31577 1365018 759871 2305 0 0 ruby
[13409] 992 13409 267033 175625 524 0 0 ruby
[14242] 992 14242 270105 178987 530 0 0 ruby
[15343] 992 15343 270105 174715 530 0 0 ruby
[18007] 992 18007 3049241 1918112 4741 0 0 ruby
[27547] 992 27547 262937 177493 517 0 0 ruby
[30092] 992 30092 268057 177439 527 0 0 ruby
[30822] 992 30822 572185 183714 1121 0 0 ruby
[ 3328] 992 3328 261146 168700 513 0 0 ruby
[ 4480] 992 4480 262937 170746 515 0 0 ruby
[11189] 992 11189 270105 180024 529 0 0 ruby
[14875] 992 14875 269081 173845 527 0 0 ruby
[ 9602] 992 9602 280345 178342 550 0 0 ruby
[24163] 108 24163 7872 240 19 0 0 ntpd
[27139] 992 27139 267800 175708 526 0 0 ruby
[10904] 992 10904 278297 177379 546 0 0 ruby
[17762] 992 17762 275225 176045 540 0 0 ruby
[21278] 992 21278 251416 161630 496 0 0 ruby
[23913] 0 23913 724330 167602 510 0 0 ruby
[23952] 992 23952 249368 169349 492 0 0 ruby
[25117] 992 25117 236056 153699 465 0 0 ruby
[25738] 992 25738 265752 170380 523 0 0 ruby
[27006] 0 27006 11715 238 29 0 0 cron
[27011] 0 27011 11715 238 29 0 0 cron
[27023] 0 27023 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27032] 0 27032 2782 156 10 0 0 bash
[27034] 0 27034 2782 155 11 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[27061] 0 27061 1479 126 8 0 0 sleep
[27069] 0 27069 1479 126 8 0 0 sleep
[28832] 992 28832 212504 133122 418 0 0 ruby
[ 389] 992 389 210456 130998 414 0 0 ruby
[ 1987] 992 1987 211737 131322 417 0 0 ruby
[ 2115] 0 2115 11715 238 29 0 0 cron
[ 2118] 0 2118 11715 238 29 0 0 cron
[ 2119] 0 2119 11715 238 29 0 0 cron
[ 2121] 0 2121 11715 238 29 0 0 cron
[ 2122] 0 2122 11715 238 29 0 0 cron
[ 2123] 0 2123 11715 238 29 0 0 cron
[ 2129] 0 2129 11715 238 29 0 0 cron
[ 2130] 0 2130 11715 238 29 0 0 cron
[ 2131] 0 2131 11715 238 29 0 0 cron
[ 2132] 0 2132 11715 238 29 0 0 cron
[ 2133] 0 2133 11715 238 29 0 0 cron
[ 2134] 0 2134 2782 155 10 0 0 bash
[ 2135] 0 2135 2782 153 10 0 0 bash
[ 2138] 0 2138 2782 156 10 0 0 bash
[ 2141] 0 2141 2782 156 10 0 0 bash
[ 2143] 0 2143 2782 155 10 0 0 bash
[ 2144] 0 2144 2782 156 10 0 0 bash
[ 2145] 0 2145 2782 156 10 0 0 bash
[ 2147] 0 2147 2782 154 10 0 0 bash
[ 2149] 0 2149 2782 154 10 0 0 bash
[ 2150] 0 2150 2782 156 11 0 0 bash
[ 2152] 0 2152 2782 156 10 0 0 bash
[ 2153] 0 2153 1479 127 8 0 0 sleep
[ 2156] 0 2156 1479 126 8 0 0 sleep
[ 2158] 0 2158 1479 126 7 0 0 sleep
[ 2159] 0 2159 1479 127 8 0 0 sleep
[ 2160] 0 2160 1479 127 8 0 0 sleep
[ 2162] 0 2162 1479 127 8 0 0 sleep
[ 2163] 0 2163 1479 126 8 0 0 sleep
[ 2165] 0 2165 1479 127 8 0 0 sleep
[ 2166] 0 2166 1479 127 8 0 0 sleep
[ 2167] 0 2167 1479 126 7 0 0 sleep
[ 2169] 0 2169 1479 126 7 0 0 sleep
[ 3025] 992 3025 194556 115654 380 0 0 ruby
Out of memory: Kill process 18007 (ruby) score 239 or sacrifice child
Killed process 18007 (ruby) total-vm:12196964kB, anon-rss:7672032kB, file-rss:412kB
ruby invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
ruby cpuset=/ mems_allowed=0
CPU: 1 PID: 30527 Comm: ruby Not tainted 3.13.0-156-generic #206-Ubuntu
Hardware name: Amazon EC2 m5.2xlarge/, BIOS 1.0 10/16/2017
0000000000000000 ffff8806cc2bf960 ffffffff8173654e ffff8806b9b5af20
0000000000000000 ffff8806cc2bf9e8 ffffffff81730abd ffffffff81c43580
ffff88068ff9da00 0000000000000015 0000000000000000 ffffffff81c43580
Call Trace:
[] dump_stack+0x64/0x80
[] dump_header+0x7f/0x1f1
[] oom_kill_process+0x201/0x360
[] ? security_capable_noaudit+0x1b/0x20
[] out_of_memory+0x471/0x4b0
[] __alloc_pages_nodemask+0xa6c/0xb90
[] alloc_pages_current+0xa3/0x160
[] __page_cache_alloc+0x97/0xc0
[] filemap_fault+0x185/0x400
[] __do_fault+0x75/0x560
[] handle_mm_fault+0x234/0xeb0
[] __do_page_fault+0x183/0x570
[] ? set_next_entity+0x95/0xb0
[] ? __switch_to+0x350/0x530
[] do_page_fault+0x1a/0x70
[] do_async_page_fault+0x29/0xe0
[] async_page_fault+0x28/0x30
Mem-Info:
Node 0 DMA per-cpu:
CPU 0: hi: 0, btch: 1 usd: 0
CPU 1: hi: 0, btch: 1 usd: 0
CPU 2: hi: 0, btch: 1 usd: 0
CPU 3: hi: 0, btch: 1 usd: 0
CPU 4: hi: 0, btch: 1 usd: 0
CPU 5: hi: 0, btch: 1 usd: 0
CPU 6: hi: 0, btch: 1 usd: 0
CPU 7: hi: 0, btch: 1 usd: 0
Node 0 DMA32 per-cpu:
CPU 0: hi: 186, btch: 31 usd: 0
CPU 1: hi: 186, btch: 31 usd: 0
CPU 2: hi: 186, btch: 31 usd: 0
CPU 3: hi: 186, btch: 31 usd: 0
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 0
CPU 6: hi: 186, btch: 31 usd: 0
CPU 7: hi: 186, btch: 31 usd: 0
Node 0 Normal per-cpu:
CPU 0: hi: 186, btch: 31 usd: 0
CPU 1: hi: 186, btch: 31 usd: 30
CPU 2: hi: 186, btch: 31 usd: 66
CPU 3: hi: 186, btch: 31 usd: 176
CPU 4: hi: 186, btch: 31 usd: 78
CPU 5: hi: 186, btch: 31 usd: 1
CPU 6: hi: 186, btch: 31 usd: 1
CPU 7: hi: 186, btch: 31 usd: 0
active_anon:7857904 inactive_anon:39 isolated_anon:0
active_file:210 inactive_file:0 isolated_file:9
unevictable:3774 dirty:35 writeback:0 unstable:0
free:49109 slab_reclaimable:13940 slab_unreclaimable:13708
mapped:187 shmem:101 pagetables:30929 bounce:0
free_cma:0
Node 0 DMA free:15908kB min:32kB low:40kB high:48kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
lowmem_reserve[]: 0 2976 31358 31358
Node 0 DMA32 free:119704kB min:6408kB low:8008kB high:9612kB active_anon:2900272kB inactive_anon:16kB active_file:292kB inactive_file:0kB unevictable:2696kB isolated(anon):0kB isolated(file):0kB present:3129316kB managed:3050168kB mlocked:2696kB dirty:28kB writeback:0kB mapped:0kB shmem:24kB slab_reclaimable:6032kB slab_unreclaimable:4956kB kernel_stack:536kB pagetables:11296kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:733 all_unreclaimable? yes
lowmem_reserve[]: 0 0 28382 28382
Node 0 Normal free:60824kB min:61136kB low:76420kB high:91704kB active_anon:28531344kB inactive_anon:140kB active_file:548kB inactive_file:0kB unevictable:12400kB isolated(anon):0kB isolated(file):36kB present:29577216kB managed:29063528kB mlocked:12400kB dirty:112kB writeback:0kB mapped:928kB shmem:380kB slab_reclaimable:49728kB slab_unreclaimable:49876kB kernel_stack:3672kB pagetables:112420kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:1204 all_unreclaimable? yes
lowmem_reserve[]: 0 0 0 0
Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15908kB
Node 0 DMA32: 657*4kB (UEM) 341*8kB (UEM) 415*16kB (UEM) 286*32kB (UE) 201*64kB (UEM) 111*128kB (UEM) 59*256kB (UEM) 29*512kB (UE) 5*1024kB (UE) 4*2048kB (UE) 7*4096kB (UEMR) = 120156kB
Node 0 Normal: 8508*4kB (UE) 2825*8kB (U) 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 60728kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
443 total pagecache pages
0 pages in swap cache
Swap cache stats: add 0, delete 0, find 0/0
Free swap = 0kB
Total swap = 0kB
8180631 pages RAM
0 pages HighMem/MovableOnly
128422 pages reserved
[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
[ 478] 0 478 4869 74 13 0 0 upstart-udev-br
[ 482] 0 482 12392 245 27 0 -1000 systemd-udevd
[ 642] 0 642 12164 311 24 0 0 lldpd
[ 663] 0 663 3815 70 12 0 0 upstart-socket-
[ 698] 106 698 12164 138 24 0 0 lldpd
[ 736] 0 736 2555 615 8 0 0 dhclient
[ 986] 102 986 11999 201 26 0 0 dbus-daemon
[ 1026] 0 1026 8756 205 22 0 0 systemd-logind
[ 1033] 0 1033 3819 73 12 0 0 upstart-file-br
[ 1078] 0 1078 3634 164 12 0 0 getty
[ 1081] 0 1081 3634 165 12 0 0 getty
[ 1086] 0 1086 3634 166 12 0 0 getty
[ 1088] 0 1088 3634 166 11 0 0 getty
[ 1089] 0 1089 182070 2283 50 0 0 amazon-ssm-agen
[ 1091] 0 1091 3634 167 12 0 0 getty
[ 1126] 0 1126 1092 109 8 0 0 acpid
[ 1127] 0 1127 5913 187 17 0 0 cron
[ 1129] 0 1129 4785 40 12 0 0 atd
[ 1184] 0 1184 4799 148 15 0 0 irqbalance
[ 1302] 0 1302 2243 60 8 0 0 rngd
[ 1453] 0 1453 8487 156 15 0 0 memlockd
[ 1544] 1 1544 1600 112 9 0 0 uptimed
[ 1548] 0 1548 3634 174 12 0 0 getty
[ 1549] 0 1549 3196 168 12 0 0 getty
[ 7154] 0 7154 46 4 3 0 0 runsvdir
[ 7163] 0 7163 41 1 3 0 0 runsv
[ 7194] 0 7194 1085 127 8 0 0 logger
[ 7195] 0 7195 41 1 3 0 0 runsv
[ 7205] 0 7205 1085 147 8 0 0 logger
[ 9101] 998 9101 9764 1156 24 0 0 python
[ 9136] 109 9136 7478 186 18 0 0 dnsmasq
[17111] 0 17111 3211 77 11 0 0 mcelog
[20922] 0 20922 6289 200 16 0 0 master
[24780] 0 24780 5856 178 16 0 0 rpcbind
[26542] 0 26542 41 1 3 0 0 runsv
[26603] 0 26603 49 1 4 0 0 svlogd
[27249] 0 27249 41 1 3 0 0 runsv
[27398] 0 27398 49 1 4 0 0 svlogd
[29275] 0 29275 41 1 3 0 0 runsv
[29293] 0 29293 1085 79 7 0 0 logger
[ 467] 991 467 8792 1554 15 0 0 redis-server
[ 743] 0 743 41 1 3 0 0 runsv
[ 744] 0 744 49 6 4 0 0 svlogd
[ 1458] 0 1458 41 1 3 0 0 runsv
[ 1459] 0 1459 50 9 4 0 0 svlogd
[ 2423] 993 2423 6897 220 18 0 0 nrpe
[ 2427] 0 2427 19120 3356 34 0 0 ruby
[ 2683] 0 2683 10741 178 25 0 0 syslog-ng
[ 2684] 0 2684 28994 4225 62 0 0 syslog-ng
[ 2692] 0 2692 17384 2806 33 0 0 ruby
[ 2812] 0 2812 7324 353 17 0 0 nginx
[ 8224] 0 8224 41 1 3 0 0 runsv
[ 8269] 0 8269 49 1 4 0 0 svlogd
[12779] 0 12779 41 4 3 0 0 runsv
[12866] 0 12866 1085 146 8 0 0 logger
[12956] 0 12956 41 1 3 0 0 runsv
[13131] 0 13131 1085 146 8 0 0 logger
[13729] 65534 13729 84183 3755 30 0 0 memcached
[13793] 0 13793 4159 774 12 0 0 consul-template
[18314] 0 18314 29462 645 24 0 0 osqueryd
[18316] 0 18316 217702 5846 92 0 0 osqueryd
[21414] 0 21414 15344 282 34 0 -1000 sshd
[23885] 992 23885 167164 1243 39 0 0 raingutter
[23943] 992 23943 56394 22810 115 0 0 bundle
[13766] 0 13766 26264 3453 53 0 0 ruby
[29048] 999 29048 17319 5413 38 0 0 consul
[29867] 113 29867 26431 3777 54 0 0 supervisord
[29891] 113 29891 44805 8693 93 0 0 python
[29892] 113 29892 57087 9187 84 0 0 python
[29896] 113 29896 151800 9566 96 0 0 python
[26311] 33 26311 7846 479 19 0 0 nginx
[ 8234] 0 8234 39672 421 79 0 0 sssd
[ 8238] 0 8238 44386 689 90 0 0 sssd_be
[ 8258] 0 8258 37591 351 75 0 0 sssd_nss
[ 8259] 0 8259 34113 352 69 0 0 sssd_pam
[ 8260] 0 8260 34114 355 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 382 73 0 0 sssd_ssh
[11911] 0 11911 55169 12312 74 0 0 chef-client
[17952] 110 17952 7369 186 19 0 0 qmgr
[14580] 110 14580 10570 313 25 0 0 tlsmgr
[29141] 113 29141 219163 15132 92 0 0 trace-agent
[31674] 0 31674 8844 3691 20 0 0 atop
[28475] 110 28475 7326 212 18 0 0 showq
[19445] 110 19445 7326 197 18 0 0 pickup
[23650] 992 23650 192508 115135 382 0 0 ruby
[26952] 992 26952 270105 173761 529 0 0 ruby
[27739] 992 27739 271129 180760 532 0 0 ruby
[28539] 992 28539 276249 175755 542 0 0 ruby
[29063] 992 29063 274458 182049 537 0 0 ruby
[29458] 992 29458 269852 177873 526 0 0 ruby
[29620] 992 29620 273177 177700 536 0 0 ruby
[30048] 992 30048 273177 179461 536 0 0 ruby
[30203] 992 30203 271133 173692 526 0 0 ruby
[30527] 992 30527 269081 179698 528 0 0 ruby
[30850] 992 30850 274201 180508 539 0 0 ruby
[31035] 992 31035 269081 172628 528 0 0 ruby
[31149] 992 31149 280345 183596 550 0 0 ruby
[31346] 992 31346 273177 177574 536 0 0 ruby
[31429] 992 31429 571165 181818 1115 0 0 ruby
[31467] 992 31467 264985 173231 520 0 0 ruby
[31515] 992 31515 263961 170369 518 0 0 ruby
[31577] 992 31577 2610202 1603831 4097 0 0 ruby
[13409] 992 13409 267033 176754 524 0 0 ruby
[14242] 992 14242 270105 178992 530 0 0 ruby
[15343] 992 15343 270105 174503 530 0 0 ruby
[27547] 992 27547 262937 173382 517 0 0 ruby
[30092] 992 30092 268057 178463 527 0 0 ruby
[30822] 992 30822 572185 183738 1121 0 0 ruby
[ 3328] 992 3328 261146 168788 513 0 0 ruby
[ 4480] 992 4480 262937 171832 515 0 0 ruby
[11189] 992 11189 270105 182414 529 0 0 ruby
[14875] 992 14875 269081 172766 527 0 0 ruby
[ 9602] 992 9602 280345 182148 550 0 0 ruby
[24163] 108 24163 7872 276 19 0 0 ntpd
[27139] 992 27139 267800 175694 526 0 0 ruby
[10904] 992 10904 1690393 1093698 2749 0 0 ruby
[17762] 992 17762 275225 176039 540 0 0 ruby
[21278] 992 21278 251416 165103 496 0 0 ruby
[23913] 0 23913 724330 168318 511 0 0 ruby
[23952] 992 23952 249368 166056 492 0 0 ruby
[25117] 992 25117 236056 154105 465 0 0 ruby
[25738] 992 25738 265752 170431 523 0 0 ruby
[27006] 0 27006 11715 238 29 0 0 cron
[27011] 0 27011 11715 238 29 0 0 cron
[27023] 0 27023 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27032] 0 27032 2782 156 10 0 0 bash
[27034] 0 27034 2782 155 11 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[27061] 0 27061 1479 126 8 0 0 sleep
[27069] 0 27069 1479 126 8 0 0 sleep
[28832] 992 28832 212504 133184 418 0 0 ruby
[ 389] 992 389 212504 133695 419 0 0 ruby
[ 1987] 992 1987 212761 131827 418 0 0 ruby
[ 2115] 0 2115 11715 238 29 0 0 cron
[ 2119] 0 2119 11715 238 29 0 0 cron
[ 2123] 0 2123 11715 238 29 0 0 cron
[ 2134] 0 2134 2782 155 10 0 0 bash
[ 2138] 0 2138 2782 156 10 0 0 bash
[ 2145] 0 2145 2782 192 10 0 0 bash
[ 2153] 0 2153 1479 127 8 0 0 sleep
[ 2160] 0 2160 1479 127 8 0 0 sleep
[ 3025] 992 3025 208408 128364 410 0 0 ruby
[ 3425] 992 3425 209689 129002 412 0 0 ruby
[ 4214] 0 4214 2792 199 10 0 0 bash
[ 4215] 0 4215 1478 120 8 0 0 logger
[ 4220] 0 4220 2793 60 10 0 0 bash
[ 4221] 0 4221 2793 69 10 0 0 bash
Out of memory: Kill process 31577 (ruby) score 200 or sacrifice child
Killed process 31577 (ruby) total-vm:10440808kB, anon-rss:6414948kB, file-rss:376kB
thread.rb:70 invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
thread.rb:70 cpuset=/ mems_allowed=0
CPU: 0 PID: 23981 Comm: thread.rb:70 Not tainted 3.13.0-156-generic #206-Ubuntu
Hardware name: Amazon EC2 m5.2xlarge/, BIOS 1.0 10/16/2017
0000000000000000 ffff8807d6717960 ffffffff8173654e ffff88007bd11790
0000000000000000 ffff8807d67179e8 ffffffff81730abd ffffffff8106d6c6
ffff8807d67179c0 ffffffff810cc81c 0000000001320122 0000000000000001
Call Trace:
[] dump_stack+0x64/0x80
[] dump_header+0x7f/0x1f1
[] ? put_online_cpus+0x56/0x80
[] ? rcu_oom_notify+0xcc/0xf0
[] oom_kill_process+0x201/0x360
[] ? security_capable_noaudit+0x1b/0x20
[] out_of_memory+0x471/0x4b0
[] __alloc_pages_nodemask+0xa6c/0xb90
[] alloc_pages_current+0xa3/0x160
[] __page_cache_alloc+0x97/0xc0
[] filemap_fault+0x185/0x400
[] __do_fault+0x75/0x560
[] ? ttwu_do_wakeup+0x19/0x100
[] handle_mm_fault+0x234/0xeb0
[] ? core_sys_select+0x236/0x2e0
[] __do_page_fault+0x183/0x570
[] ? do_futex+0x152/0x760
[] ? read_tsc+0x9/0x20
[] ? ktime_get_ts+0x4d/0xf0
[] ? poll_select_copy_remaining+0xdc/0x130
[] do_page_fault+0x1a/0x70
[] ? SyS_select+0xbd/0x100
[] do_async_page_fault+0x29/0xe0
[] async_page_fault+0x28/0x30
Mem-Info:
Node 0 DMA per-cpu:
CPU 0: hi: 0, btch: 1 usd: 0
CPU 1: hi: 0, btch: 1 usd: 0
CPU 2: hi: 0, btch: 1 usd: 0
CPU 3: hi: 0, btch: 1 usd: 0
CPU 4: hi: 0, btch: 1 usd: 0
CPU 5: hi: 0, btch: 1 usd: 0
CPU 6: hi: 0, btch: 1 usd: 0
CPU 7: hi: 0, btch: 1 usd: 0
Node 0 DMA32 per-cpu:
CPU 0: hi: 186, btch: 31 usd: 3
CPU 1: hi: 186, btch: 31 usd: 27
CPU 2: hi: 186, btch: 31 usd: 25
CPU 3: hi: 186, btch: 31 usd: 30
CPU 4: hi: 186, btch: 31 usd: 41
CPU 5: hi: 186, btch: 31 usd: 180
CPU 6: hi: 186, btch: 31 usd: 25
CPU 7: hi: 186, btch: 31 usd: 53
Node 0 Normal per-cpu:
CPU 0: hi: 186, btch: 31 usd: 121
CPU 1: hi: 186, btch: 31 usd: 69
CPU 2: hi: 186, btch: 31 usd: 108
CPU 3: hi: 186, btch: 31 usd: 142
CPU 4: hi: 186, btch: 31 usd: 20
CPU 5: hi: 186, btch: 31 usd: 126
CPU 6: hi: 186, btch: 31 usd: 58
CPU 7: hi: 186, btch: 31 usd: 81
active_anon:7849211 inactive_anon:39 isolated_anon:0
active_file:146 inactive_file:16 isolated_file:0
unevictable:3774 dirty:7 writeback:0 unstable:0
free:52187 slab_reclaimable:13989 slab_unreclaimable:13686
mapped:187 shmem:101 pagetables:35575 bounce:0
free_cma:0
Node 0 DMA free:15908kB min:32kB low:40kB high:48kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
lowmem_reserve[]: 0 2976 31358 31358
Node 0 DMA32 free:120752kB min:6408kB low:8008kB high:9612kB active_anon:2897696kB inactive_anon:16kB active_file:128kB inactive_file:0kB unevictable:2696kB isolated(anon):0kB isolated(file):0kB present:3129316kB managed:3050168kB mlocked:2696kB dirty:40kB writeback:0kB mapped:28kB shmem:24kB slab_reclaimable:6160kB slab_unreclaimable:4948kB kernel_stack:544kB pagetables:12900kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:1203 all_unreclaimable? yes
lowmem_reserve[]: 0 0 28382 28382
Node 0 Normal free:72088kB min:61136kB low:76420kB high:91704kB active_anon:28499148kB inactive_anon:140kB active_file:456kB inactive_file:320kB unevictable:12400kB isolated(anon):0kB isolated(file):0kB present:29577216kB managed:29063528kB mlocked:12400kB dirty:0kB writeback:0kB mapped:720kB shmem:380kB slab_reclaimable:49796kB slab_unreclaimable:49796kB kernel_stack:3728kB pagetables:129400kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:1325 all_unreclaimable? yes
lowmem_reserve[]: 0 0 0 0
Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15908kB
Node 0 DMA32: 800*4kB (UEM) 343*8kB (UEM) 343*16kB (UE) 280*32kB (UE) 217*64kB (UEM) 110*128kB (UEM) 58*256kB (UEM) 28*512kB (UE) 4*1024kB (UE) 5*2048kB (UE) 7*4096kB (UEMR) = 120552kB
Node 0 Normal: 6868*4kB (UEM) 5011*8kB (UEM) 20*16kB (UM) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 71976kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
818 total pagecache pages
0 pages in swap cache
Swap cache stats: add 0, delete 0, find 0/0
Free swap = 0kB
Total swap = 0kB
8180631 pages RAM
0 pages HighMem/MovableOnly
128422 pages reserved
[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
[ 478] 0 478 4869 74 13 0 0 upstart-udev-br
[ 482] 0 482 12392 245 27 0 -1000 systemd-udevd
[ 642] 0 642 12164 311 24 0 0 lldpd
[ 663] 0 663 3815 70 12 0 0 upstart-socket-
[ 698] 106 698 12164 155 24 0 0 lldpd
[ 736] 0 736 2555 615 8 0 0 dhclient
[ 986] 102 986 11999 201 26 0 0 dbus-daemon
[ 1026] 0 1026 8756 205 22 0 0 systemd-logind
[ 1033] 0 1033 3819 73 12 0 0 upstart-file-br
[ 1078] 0 1078 3634 164 12 0 0 getty
[ 1081] 0 1081 3634 165 12 0 0 getty
[ 1086] 0 1086 3634 166 12 0 0 getty
[ 1088] 0 1088 3634 166 11 0 0 getty
[ 1089] 0 1089 182070 2283 50 0 0 amazon-ssm-agen
[ 1091] 0 1091 3634 167 12 0 0 getty
[ 1126] 0 1126 1092 109 8 0 0 acpid
[ 1127] 0 1127 5913 189 17 0 0 cron
[ 1129] 0 1129 4785 40 12 0 0 atd
[ 1184] 0 1184 4799 148 15 0 0 irqbalance
[ 1302] 0 1302 2243 52 8 0 0 rngd
[ 1453] 0 1453 8487 156 15 0 0 memlockd
[ 1544] 1 1544 1600 112 9 0 0 uptimed
[ 1548] 0 1548 3634 174 12 0 0 getty
[ 1549] 0 1549 3196 168 12 0 0 getty
[ 7154] 0 7154 46 4 3 0 0 runsvdir
[ 7163] 0 7163 41 1 3 0 0 runsv
[ 7194] 0 7194 1085 127 8 0 0 logger
[ 7195] 0 7195 41 1 3 0 0 runsv
[ 7205] 0 7205 1085 147 8 0 0 logger
[ 9101] 998 9101 9764 1156 24 0 0 python
[ 9136] 109 9136 7478 197 18 0 0 dnsmasq
[17111] 0 17111 3211 77 11 0 0 mcelog
[20922] 0 20922 6289 179 16 0 0 master
[24780] 0 24780 5856 162 16 0 0 rpcbind
[26542] 0 26542 41 1 3 0 0 runsv
[26603] 0 26603 49 1 4 0 0 svlogd
[27249] 0 27249 41 1 3 0 0 runsv
[27398] 0 27398 49 1 4 0 0 svlogd
[29275] 0 29275 41 1 3 0 0 runsv
[29293] 0 29293 1085 79 7 0 0 logger
[ 467] 991 467 8792 1545 15 0 0 redis-server
[ 743] 0 743 41 1 3 0 0 runsv
[ 744] 0 744 49 6 4 0 0 svlogd
[ 1458] 0 1458 41 1 3 0 0 runsv
[ 1459] 0 1459 50 9 4 0 0 svlogd
[ 2423] 993 2423 6897 220 18 0 0 nrpe
[ 2427] 0 2427 19120 3373 34 0 0 ruby
[ 2683] 0 2683 10741 178 25 0 0 syslog-ng
[ 2684] 0 2684 28994 4235 62 0 0 syslog-ng
[ 2692] 0 2692 17384 2807 33 0 0 ruby
[ 2812] 0 2812 7324 353 17 0 0 nginx
[ 8224] 0 8224 41 1 3 0 0 runsv
[ 8269] 0 8269 49 1 4 0 0 svlogd
[12779] 0 12779 41 4 3 0 0 runsv
[12866] 0 12866 1085 146 8 0 0 logger
[12956] 0 12956 41 1 3 0 0 runsv
[13131] 0 13131 1085 146 8 0 0 logger
[13729] 65534 13729 84183 3786 30 0 0 memcached
[13793] 0 13793 4159 778 12 0 0 consul-template
[18314] 0 18314 29462 619 24 0 0 osqueryd
[18316] 0 18316 217702 5846 92 0 0 osqueryd
[21414] 0 21414 15344 282 34 0 -1000 sshd
[23885] 992 23885 167164 1261 39 0 0 raingutter
[23943] 992 23943 56394 22834 115 0 0 bundle
[13766] 0 13766 26264 3470 53 0 0 ruby
[29048] 999 29048 17319 5413 38 0 0 consul
[29867] 113 29867 26431 3758 54 0 0 supervisord
[29891] 113 29891 44805 8678 93 0 0 python
[29892] 113 29892 57087 9330 84 0 0 python
[29896] 113 29896 151800 9573 96 0 0 python
[26311] 33 26311 7846 482 19 0 0 nginx
[ 8234] 0 8234 39672 424 79 0 0 sssd
[ 8238] 0 8238 44386 689 90 0 0 sssd_be
[ 8258] 0 8258 37591 367 75 0 0 sssd_nss
[ 8259] 0 8259 34113 356 69 0 0 sssd_pam
[ 8260] 0 8260 34114 355 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 386 73 0 0 sssd_ssh
[11911] 0 11911 55169 12312 74 0 0 chef-client
[17952] 110 17952 7369 186 19 0 0 qmgr
[14580] 110 14580 10570 313 25 0 0 tlsmgr
[29141] 113 29141 219163 14902 92 0 0 trace-agent
[31674] 0 31674 8844 3691 20 0 0 atop
[28475] 110 28475 7326 183 18 0 0 showq
[19445] 110 19445 7326 184 18 0 0 pickup
[23650] 992 23650 192508 115124 382 0 0 ruby
[26952] 992 26952 270105 174331 529 0 0 ruby
[27739] 992 27739 271129 185240 532 0 0 ruby
[28539] 992 28539 276249 176603 542 0 0 ruby
[29063] 992 29063 274458 179426 537 0 0 ruby
[29458] 992 29458 269852 178039 526 0 0 ruby
[29620] 992 29620 273177 180614 536 0 0 ruby
[30048] 992 30048 273177 180118 536 0 0 ruby
[30203] 992 30203 271133 172067 526 0 0 ruby
[30527] 992 30527 269081 178285 528 0 0 ruby
[30850] 992 30850 274201 180307 539 0 0 ruby
[31035] 992 31035 3979033 2419102 6012 0 0 ruby
[31149] 992 31149 280345 180924 550 0 0 ruby
[31346] 992 31346 273177 176606 536 0 0 ruby
[31429] 992 31429 571165 182808 1115 0 0 ruby
[31467] 992 31467 264985 176114 520 0 0 ruby
[31515] 992 31515 263961 171307 518 0 0 ruby
[13409] 992 13409 267033 174116 524 0 0 ruby
[14242] 992 14242 270105 182107 530 0 0 ruby
[15343] 992 15343 270105 176866 530 0 0 ruby
[27547] 992 27547 262937 173662 517 0 0 ruby
[30092] 992 30092 268057 175114 527 0 0 ruby
[30822] 992 30822 572185 185686 1121 0 0 ruby
[ 3328] 992 3328 261146 169305 513 0 0 ruby
[ 4480] 992 4480 262937 170880 515 0 0 ruby
[11189] 992 11189 270105 178953 529 0 0 ruby
[14875] 992 14875 269081 173991 527 0 0 ruby
[ 9602] 992 9602 280345 179864 550 0 0 ruby
[24163] 108 24163 7872 253 19 0 0 ntpd
[27139] 992 27139 267800 179762 526 0 0 ruby
[10904] 992 10904 3050779 224978 4746 0 0 ruby
[17762] 992 17762 275225 154403 540 0 0 ruby
[21278] 992 21278 251416 161657 496 0 0 ruby
[23913] 0 23913 724330 170125 515 0 0 ruby
[23952] 992 23952 249368 167826 492 0 0 ruby
[25117] 992 25117 236056 155787 465 0 0 ruby
[25738] 992 25738 590617 196762 1156 0 0 ruby
[27011] 0 27011 11715 238 29 0 0 cron
[27023] 0 27023 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27034] 0 27034 2782 155 11 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[27061] 0 27061 1479 126 8 0 0 sleep
[28832] 992 28832 247320 166910 486 0 0 ruby
[ 389] 992 389 245272 165008 482 0 0 ruby
[ 1987] 992 1987 213785 133095 420 0 0 ruby
[ 3025] 992 3025 209432 130576 413 0 0 ruby
[ 3425] 992 3425 210713 130059 414 0 0 ruby
[ 4347] 992 4347 210456 131188 415 0 0 ruby
[ 4507] 0 4507 11715 238 29 0 0 cron
[ 4511] 0 4511 11715 238 29 0 0 cron
[ 4513] 0 4513 11715 238 29 0 0 cron
[ 4514] 0 4514 11715 238 29 0 0 cron
[ 4515] 0 4515 11715 238 29 0 0 cron
[ 4525] 0 4525 11715 238 29 0 0 cron
[ 4526] 0 4526 2782 155 10 0 0 bash
[ 4528] 0 4528 2782 154 10 0 0 bash
[ 4532] 0 4532 2782 155 10 0 0 bash
[ 4538] 0 4538 1479 127 8 0 0 sleep
[ 4539] 0 4539 1479 127 8 0 0 sleep
[ 4540] 0 4540 1479 126 8 0 0 sleep
[ 4546] 0 4546 2782 154 10 0 0 bash
[ 4551] 0 4551 2782 154 9 0 0 bash
[ 4553] 0 4553 2782 156 10 0 0 bash
[ 4554] 0 4554 1479 127 7 0 0 sleep
[ 4556] 0 4556 1479 127 8 0 0 sleep
[ 4557] 0 4557 1479 126 8 0 0 sleep
[ 6656] 0 6656 4078 46 11 0 0 syslog-ng-ctl
Out of memory: Kill process 31035 (ruby) score 301 or sacrifice child
Killed process 31035 (ruby) total-vm:15916132kB, anon-rss:9675992kB, file-rss:416kB
ruby invoked oom-killer: gfp_mask=0x280da, order=0, oom_score_adj=0
ruby cpuset=/ mems_allowed=0
CPU: 1 PID: 3025 Comm: ruby Not tainted 3.13.0-156-generic #206-Ubuntu
Hardware name: Amazon EC2 m5.2xlarge/, BIOS 1.0 10/16/2017
0000000000000000 ffff880104953a60 ffffffff8173654e ffff88068f2b9790
0000000000000000 ffff880104953ae8 ffffffff81730abd ffffffff8106d6c6
ffff880104953ac0 ffffffff810cc81c 0000000001320122 0000000000000007
Call Trace:
[] dump_stack+0x64/0x80
[] dump_header+0x7f/0x1f1
[] ? put_online_cpus+0x56/0x80
[] ? rcu_oom_notify+0xcc/0xf0
[] oom_kill_process+0x201/0x360
[] ? security_capable_noaudit+0x1b/0x20
[] out_of_memory+0x471/0x4b0
[] __alloc_pages_nodemask+0xa6c/0xb90
[] alloc_pages_vma+0x9a/0x140
[] handle_mm_fault+0xaaf/0xeb0
[] __do_page_fault+0x183/0x570
[] ? blk_finish_plug+0x14/0x40
[] ? SyS_madvise+0x3a5/0x800
[] ? __switch_to+0x350/0x530
[] do_page_fault+0x1a/0x70
[] do_async_page_fault+0x29/0xe0
[] async_page_fault+0x28/0x30
Mem-Info:
Node 0 DMA per-cpu:
CPU 0: hi: 0, btch: 1 usd: 0
CPU 1: hi: 0, btch: 1 usd: 0
CPU 2: hi: 0, btch: 1 usd: 0
CPU 3: hi: 0, btch: 1 usd: 0
CPU 4: hi: 0, btch: 1 usd: 0
CPU 5: hi: 0, btch: 1 usd: 0
CPU 6: hi: 0, btch: 1 usd: 0
CPU 7: hi: 0, btch: 1 usd: 0
Node 0 DMA32 per-cpu:
CPU 0: hi: 186, btch: 31 usd: 0
CPU 1: hi: 186, btch: 31 usd: 0
CPU 2: hi: 186, btch: 31 usd: 0
CPU 3: hi: 186, btch: 31 usd: 0
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 0
CPU 6: hi: 186, btch: 31 usd: 0
CPU 7: hi: 186, btch: 31 usd: 0
Node 0 Normal per-cpu:
CPU 0: hi: 186, btch: 31 usd: 0
CPU 1: hi: 186, btch: 31 usd: 0
CPU 2: hi: 186, btch: 31 usd: 0
CPU 3: hi: 186, btch: 31 usd: 0
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 0
CPU 6: hi: 186, btch: 31 usd: 0
CPU 7: hi: 186, btch: 31 usd: 0
active_anon:7853850 inactive_anon:39 isolated_anon:0
active_file:112 inactive_file:36 isolated_file:0
unevictable:3774 dirty:0 writeback:0 unstable:0
free:49139 slab_reclaimable:13986 slab_unreclaimable:13645
mapped:404 shmem:101 pagetables:35642 bounce:0
free_cma:0
Node 0 DMA free:15908kB min:32kB low:40kB high:48kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
lowmem_reserve[]: 0 2976 31358 31358
Node 0 DMA32 free:119792kB min:6408kB low:8008kB high:9612kB active_anon:2899244kB inactive_anon:16kB active_file:104kB inactive_file:8kB unevictable:2696kB isolated(anon):0kB isolated(file):0kB present:3129316kB managed:3050168kB mlocked:2696kB dirty:0kB writeback:0kB mapped:308kB shmem:24kB slab_reclaimable:6156kB slab_unreclaimable:4920kB kernel_stack:584kB pagetables:13068kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:169 all_unreclaimable? yes
lowmem_reserve[]: 0 0 28382 28382
Node 0 Normal free:60856kB min:61136kB low:76420kB high:91704kB active_anon:28516156kB inactive_anon:140kB active_file:344kB inactive_file:136kB unevictable:12400kB isolated(anon):0kB isolated(file):0kB present:29577216kB managed:29063528kB mlocked:12400kB dirty:0kB writeback:0kB mapped:1308kB shmem:380kB slab_reclaimable:49788kB slab_unreclaimable:49660kB kernel_stack:3872kB pagetables:129500kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:770 all_unreclaimable? yes
lowmem_reserve[]: 0 0 0 0
Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15908kB
Node 0 DMA32: 806*4kB (UEM) 365*8kB (UEM) 285*16kB (UEM) 291*32kB (UEM) 221*64kB (UEM) 110*128kB (UEM) 58*256kB (UEM) 28*512kB (UE) 4*1024kB (UE) 5*2048kB (UE) 7*4096kB (UEMR) = 120432kB
Node 0 Normal: 2503*4kB (UEM) 3884*8kB (UEM) 1008*16kB (UEM) 2*32kB (M) 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 61372kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
734 total pagecache pages
0 pages in swap cache
Swap cache stats: add 0, delete 0, find 0/0
Free swap = 0kB
Total swap = 0kB
8180631 pages RAM
0 pages HighMem/MovableOnly
128422 pages reserved
[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
[ 478] 0 478 4869 74 13 0 0 upstart-udev-br
[ 482] 0 482 12392 245 27 0 -1000 systemd-udevd
[ 642] 0 642 12164 311 24 0 0 lldpd
[ 663] 0 663 3815 70 12 0 0 upstart-socket-
[ 698] 106 698 12164 125 24 0 0 lldpd
[ 736] 0 736 2555 615 8 0 0 dhclient
[ 986] 102 986 11999 201 26 0 0 dbus-daemon
[ 1026] 0 1026 8756 205 22 0 0 systemd-logind
[ 1033] 0 1033 3819 73 12 0 0 upstart-file-br
[ 1078] 0 1078 3634 164 12 0 0 getty
[ 1081] 0 1081 3634 165 12 0 0 getty
[ 1086] 0 1086 3634 166 12 0 0 getty
[ 1088] 0 1088 3634 166 11 0 0 getty
[ 1089] 0 1089 182070 2283 50 0 0 amazon-ssm-agen
[ 1091] 0 1091 3634 167 12 0 0 getty
[ 1126] 0 1126 1092 109 8 0 0 acpid
[ 1127] 0 1127 5913 191 17 0 0 cron
[ 1129] 0 1129 4785 40 12 0 0 atd
[ 1184] 0 1184 4799 148 15 0 0 irqbalance
[ 1302] 0 1302 2243 46 8 0 0 rngd
[ 1453] 0 1453 8487 156 15 0 0 memlockd
[ 1544] 1 1544 1600 112 9 0 0 uptimed
[ 1548] 0 1548 3634 174 12 0 0 getty
[ 1549] 0 1549 3196 168 12 0 0 getty
[ 7154] 0 7154 46 4 3 0 0 runsvdir
[ 7163] 0 7163 41 1 3 0 0 runsv
[ 7194] 0 7194 1085 127 8 0 0 logger
[ 7195] 0 7195 41 1 3 0 0 runsv
[ 7205] 0 7205 1085 147 8 0 0 logger
[ 9101] 998 9101 9764 1156 24 0 0 python
[ 9136] 109 9136 7478 162 18 0 0 dnsmasq
[17111] 0 17111 3211 77 11 0 0 mcelog
[20922] 0 20922 6289 221 16 0 0 master
[24780] 0 24780 5856 154 16 0 0 rpcbind
[26542] 0 26542 41 1 3 0 0 runsv
[26603] 0 26603 49 1 4 0 0 svlogd
[27249] 0 27249 41 1 3 0 0 runsv
[27398] 0 27398 49 1 4 0 0 svlogd
[29275] 0 29275 41 1 3 0 0 runsv
[29293] 0 29293 1085 79 7 0 0 logger
[ 467] 991 467 8792 1544 15 0 0 redis-server
[ 743] 0 743 41 1 3 0 0 runsv
[ 744] 0 744 49 6 4 0 0 svlogd
[ 1458] 0 1458 41 1 3 0 0 runsv
[ 1459] 0 1459 50 9 4 0 0 svlogd
[ 2423] 993 2423 6897 217 18 0 0 nrpe
[ 2427] 0 2427 19120 3371 34 0 0 ruby
[ 2683] 0 2683 10741 178 25 0 0 syslog-ng
[ 2684] 0 2684 28994 4270 62 0 0 syslog-ng
[ 2692] 0 2692 17384 2808 33 0 0 ruby
[ 2812] 0 2812 7324 353 17 0 0 nginx
[ 8224] 0 8224 41 1 3 0 0 runsv
[ 8269] 0 8269 49 1 4 0 0 svlogd
[12779] 0 12779 41 4 3 0 0 runsv
[12866] 0 12866 1085 146 8 0 0 logger
[12956] 0 12956 41 1 3 0 0 runsv
[13131] 0 13131 1085 146 8 0 0 logger
[13729] 65534 13729 84183 3755 30 0 0 memcached
[13793] 0 13793 4159 778 12 0 0 consul-template
[18314] 0 18314 29462 641 24 0 0 osqueryd
[18316] 0 18316 217702 5846 92 0 0 osqueryd
[21414] 0 21414 15344 286 34 0 -1000 sshd
[23885] 992 23885 167164 1261 39 0 0 raingutter
[23943] 992 23943 56394 22826 115 0 0 bundle
[13766] 0 13766 26264 3468 53 0 0 ruby
[29048] 999 29048 17319 5401 38 0 0 consul
[29867] 113 29867 26431 3784 54 0 0 supervisord
[29891] 113 29891 44805 8695 93 0 0 python
[29892] 113 29892 57087 9208 84 0 0 python
[29896] 113 29896 151800 9573 96 0 0 python
[26311] 33 26311 7846 455 19 0 0 nginx
[ 8234] 0 8234 39672 441 79 0 0 sssd
[ 8238] 0 8238 44386 750 90 0 0 sssd_be
[ 8258] 0 8258 37591 408 75 0 0 sssd_nss
[ 8259] 0 8259 34113 372 69 0 0 sssd_pam
[ 8260] 0 8260 34114 387 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 406 73 0 0 sssd_ssh
[11911] 0 11911 55169 12312 74 0 0 chef-client
[17952] 110 17952 7369 186 19 0 0 qmgr
[14580] 110 14580 10570 313 25 0 0 tlsmgr
[29141] 113 29141 219163 15150 92 0 0 trace-agent
[31674] 0 31674 8844 3691 20 0 0 atop
[28475] 110 28475 7326 154 18 0 0 showq
[19445] 110 19445 7326 232 18 0 0 pickup
[23650] 992 23650 192508 115136 382 0 0 ruby
[26952] 992 26952 270105 183365 529 0 0 ruby
[27739] 992 27739 271129 180572 532 0 0 ruby
[28539] 992 28539 276249 178218 542 0 0 ruby
[29063] 992 29063 274458 182290 537 0 0 ruby
[29458] 992 29458 269852 182542 526 0 0 ruby
[29620] 992 29620 273177 187906 536 0 0 ruby
[30048] 992 30048 273177 185065 536 0 0 ruby
[30203] 992 30203 271133 171843 526 0 0 ruby
[30527] 992 30527 269081 176731 528 0 0 ruby
[30850] 992 30850 274201 183410 539 0 0 ruby
[31149] 992 31149 280345 182211 550 0 0 ruby
[31346] 992 31346 273177 175798 536 0 0 ruby
[31429] 992 31429 571165 187465 1115 0 0 ruby
[31467] 992 31467 264985 177167 520 0 0 ruby
[31515] 992 31515 263961 170401 518 0 0 ruby
[13409] 992 13409 267033 175794 524 0 0 ruby
[14242] 992 14242 270105 179593 530 0 0 ruby
[15343] 992 15343 270105 179067 530 0 0 ruby
[27547] 992 27547 262937 174424 517 0 0 ruby
[30092] 992 30092 268057 179046 527 0 0 ruby
[30822] 992 30822 572185 185089 1121 0 0 ruby
[ 3328] 992 3328 261146 171525 513 0 0 ruby
[ 4480] 992 4480 262937 176646 515 0 0 ruby
[11189] 992 11189 270105 181082 529 0 0 ruby
[14875] 992 14875 269081 177198 527 0 0 ruby
[ 9602] 992 9602 280345 179004 550 0 0 ruby
[24163] 108 24163 7872 250 19 0 0 ntpd
[27139] 992 27139 267800 174947 526 0 0 ruby
[10904] 992 10904 3050779 202955 4746 0 0 ruby
[17762] 992 17762 275225 175443 540 0 0 ruby
[21278] 992 21278 251416 164655 496 0 0 ruby
[23913] 0 23913 724330 173471 521 0 0 ruby
[23952] 992 23952 257560 167996 507 0 0 ruby
[25117] 992 25117 245272 165608 484 0 0 ruby
[25738] 992 25738 590617 194928 1156 0 0 ruby
[27011] 0 27011 11715 238 29 0 0 cron
[27023] 0 27023 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27034] 0 27034 2782 155 11 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[27061] 0 27061 1479 126 8 0 0 sleep
[28832] 992 28832 259865 171836 511 0 0 ruby
[ 389] 992 389 259608 171105 511 0 0 ruby
[ 1987] 992 1987 230169 149873 453 0 0 ruby
[ 3025] 992 3025 3436056 2273373 5425 0 0 ruby
[ 3425] 992 3425 242457 160670 475 0 0 ruby
[ 4347] 992 4347 228888 149349 451 0 0 ruby
[ 6746] 992 6746 211480 131944 416 0 0 ruby
[ 9773] 0 9773 11715 238 29 0 0 cron
[ 9774] 0 9774 11715 238 29 0 0 cron
[ 9776] 0 9776 11715 238 29 0 0 cron
[ 9777] 0 9777 11715 238 29 0 0 cron
[ 9778] 0 9778 11715 238 29 0 0 cron
[ 9779] 0 9779 11715 238 29 0 0 cron
[ 9780] 0 9780 11715 238 29 0 0 cron
[ 9781] 0 9781 11715 238 29 0 0 cron
[ 9784] 0 9784 11715 238 29 0 0 cron
[ 9787] 0 9787 11715 238 29 0 0 cron
[ 9788] 0 9788 11715 238 29 0 0 cron
[ 9789] 0 9789 11715 238 29 0 0 cron
[ 9790] 0 9790 11715 238 29 0 0 cron
[ 9791] 0 9791 11715 238 29 0 0 cron
[ 9792] 0 9792 2782 154 10 0 0 bash
[ 9793] 0 9793 2782 154 10 0 0 bash
[ 9795] 0 9795 1479 127 8 0 0 sleep
[ 9796] 0 9796 2782 155 10 0 0 bash
[ 9798] 0 9798 1479 126 9 0 0 sleep
[ 9799] 0 9799 2782 156 10 0 0 bash
[ 9800] 0 9800 2782 156 10 0 0 bash
[ 9801] 0 9801 1479 127 8 0 0 sleep
[ 9802] 0 9802 2782 155 10 0 0 bash
[ 9803] 0 9803 1479 126 8 0 0 sleep
[ 9804] 0 9804 2782 155 10 0 0 bash
[ 9805] 0 9805 1479 125 8 0 0 sleep
[ 9806] 0 9806 2782 155 9 0 0 bash
[ 9807] 0 9807 1479 127 8 0 0 sleep
[ 9809] 0 9809 1479 126 8 0 0 sleep
[ 9812] 0 9812 2782 156 10 0 0 bash
[ 9815] 0 9815 1479 126 8 0 0 sleep
[ 9817] 0 9817 2782 155 10 0 0 bash
[ 9818] 0 9818 2782 155 11 0 0 bash
[ 9819] 0 9819 1479 126 7 0 0 sleep
[ 9820] 0 9820 2782 156 10 0 0 bash
[ 9821] 0 9821 1479 126 8 0 0 sleep
[ 9822] 0 9822 2782 155 10 0 0 bash
[ 9824] 0 9824 1479 126 8 0 0 sleep
[ 9825] 0 9825 2782 155 11 0 0 bash
[ 9826] 0 9826 1479 127 8 0 0 sleep
[ 9828] 0 9828 1479 127 8 0 0 sleep
[ 9833] 0 9833 1479 127 7 0 0 sleep
[10523] 0 10523 17384 2650 29 0 0 ruby
Out of memory: Kill process 3025 (ruby) score 283 or sacrifice child
Killed process 3025 (ruby) total-vm:13744224kB, anon-rss:9092908kB, file-rss:584kB
consul invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
consul cpuset=/ mems_allowed=0
CPU: 2 PID: 29059 Comm: consul Not tainted 3.13.0-156-generic #206-Ubuntu
Hardware name: Amazon EC2 m5.2xlarge/, BIOS 1.0 10/16/2017
0000000000000000 ffff88018b329960 ffffffff8173654e ffff88018be85e40
0000000000000000 ffff88018b3299e8 ffffffff81730abd ffffffff81c43580
0000000000001aa1 0000000000000015 0000000000000000 ffffffff81c43580
Call Trace:
[] dump_stack+0x64/0x80
[] dump_header+0x7f/0x1f1
[] oom_kill_process+0x201/0x360
[] ? security_capable_noaudit+0x1b/0x20
[] out_of_memory+0x471/0x4b0
[] __alloc_pages_nodemask+0xa6c/0xb90
[] alloc_pages_current+0xa3/0x160
[] __page_cache_alloc+0x97/0xc0
[] filemap_fault+0x185/0x400
[] __do_fault+0x75/0x560
[] handle_mm_fault+0x234/0xeb0
[] __do_page_fault+0x183/0x570
[] ? do_futex+0x10a/0x760
[] ? ktime_get_ts+0x4d/0xf0
[] ? poll_select_copy_remaining+0x101/0x130
[] ? read_tsc+0x9/0x20
[] do_page_fault+0x1a/0x70
[] do_async_page_fault+0x29/0xe0
[] async_page_fault+0x28/0x30
Mem-Info:
Node 0 DMA per-cpu:
CPU 0: hi: 0, btch: 1 usd: 0
CPU 1: hi: 0, btch: 1 usd: 0
CPU 2: hi: 0, btch: 1 usd: 0
CPU 3: hi: 0, btch: 1 usd: 0
CPU 4: hi: 0, btch: 1 usd: 0
CPU 5: hi: 0, btch: 1 usd: 0
CPU 6: hi: 0, btch: 1 usd: 0
CPU 7: hi: 0, btch: 1 usd: 0
Node 0 DMA32 per-cpu:
CPU 0: hi: 186, btch: 31 usd: 29
CPU 1: hi: 186, btch: 31 usd: 4
CPU 2: hi: 186, btch: 31 usd: 30
CPU 3: hi: 186, btch: 31 usd: 16
CPU 4: hi: 186, btch: 31 usd: 44
CPU 5: hi: 186, btch: 31 usd: 7
CPU 6: hi: 186, btch: 31 usd: 20
CPU 7: hi: 186, btch: 31 usd: 24
Node 0 Normal per-cpu:
CPU 0: hi: 186, btch: 31 usd: 29
CPU 1: hi: 186, btch: 31 usd: 5
CPU 2: hi: 186, btch: 31 usd: 23
CPU 3: hi: 186, btch: 31 usd: 37
CPU 4: hi: 186, btch: 31 usd: 157
CPU 5: hi: 186, btch: 31 usd: 41
CPU 6: hi: 186, btch: 31 usd: 41
CPU 7: hi: 186, btch: 31 usd: 16
active_anon:7856497 inactive_anon:39 isolated_anon:0
active_file:110 inactive_file:61 isolated_file:0
unevictable:3774 dirty:108 writeback:0 unstable:0
free:49269 slab_reclaimable:14059 slab_unreclaimable:13642
mapped:360 shmem:101 pagetables:31618 bounce:0
free_cma:0
Node 0 DMA free:15908kB min:32kB low:40kB high:48kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
lowmem_reserve[]: 0 2976 31358 31358
Node 0 DMA32 free:119904kB min:6408kB low:8008kB high:9612kB active_anon:2901648kB inactive_anon:16kB active_file:44kB inactive_file:84kB unevictable:2696kB isolated(anon):0kB isolated(file):0kB present:3129316kB managed:3050168kB mlocked:2696kB dirty:132kB writeback:0kB mapped:164kB shmem:24kB slab_reclaimable:6180kB slab_unreclaimable:4960kB kernel_stack:536kB pagetables:10920kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:1397 all_unreclaimable? yes
lowmem_reserve[]: 0 0 28382 28382
Node 0 Normal free:61264kB min:61136kB low:76420kB high:91704kB active_anon:28524340kB inactive_anon:140kB active_file:396kB inactive_file:160kB unevictable:12400kB isolated(anon):0kB isolated(file):0kB present:29577216kB managed:29063528kB mlocked:12400kB dirty:300kB writeback:0kB mapped:1276kB shmem:380kB slab_reclaimable:50056kB slab_unreclaimable:49608kB kernel_stack:3952kB pagetables:115552kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:215 all_unreclaimable? no
lowmem_reserve[]: 0 0 0 0
Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15908kB
Node 0 DMA32: 961*4kB (UE) 466*8kB (UE) 375*16kB (UEM) 286*32kB (UE) 174*64kB (UE) 105*128kB (UEM) 58*256kB (UEM) 28*512kB (UE) 4*1024kB (UE) 5*2048kB (UE) 7*4096kB (UEMR) = 119492kB
Node 0 Normal: 3104*4kB (UEM) 4645*8kB (UEM) 472*16kB (UM) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 61224kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
944 total pagecache pages
0 pages in swap cache
Swap cache stats: add 0, delete 0, find 0/0
Free swap = 0kB
Total swap = 0kB
8180631 pages RAM
0 pages HighMem/MovableOnly
128422 pages reserved
[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
[ 478] 0 478 4869 74 13 0 0 upstart-udev-br
[ 482] 0 482 12392 245 27 0 -1000 systemd-udevd
[ 642] 0 642 12164 311 24 0 0 lldpd
[ 663] 0 663 3815 70 12 0 0 upstart-socket-
[ 698] 106 698 12164 140 24 0 0 lldpd
[ 736] 0 736 2555 617 8 0 0 dhclient
[ 986] 102 986 11999 201 26 0 0 dbus-daemon
[ 1026] 0 1026 8756 205 22 0 0 systemd-logind
[ 1033] 0 1033 3819 73 12 0 0 upstart-file-br
[ 1078] 0 1078 3634 164 12 0 0 getty
[ 1081] 0 1081 3634 165 12 0 0 getty
[ 1086] 0 1086 3634 166 12 0 0 getty
[ 1088] 0 1088 3634 166 11 0 0 getty
[ 1089] 0 1089 182070 2287 50 0 0 amazon-ssm-agen
[ 1091] 0 1091 3634 167 12 0 0 getty
[ 1126] 0 1126 1092 109 8 0 0 acpid
[ 1127] 0 1127 5913 191 17 0 0 cron
[ 1129] 0 1129 4785 40 12 0 0 atd
[ 1184] 0 1184 4799 148 15 0 0 irqbalance
[ 1302] 0 1302 2243 42 8 0 0 rngd
[ 1453] 0 1453 8487 156 15 0 0 memlockd
[ 1544] 1 1544 1600 112 9 0 0 uptimed
[ 1548] 0 1548 3634 174 12 0 0 getty
[ 1549] 0 1549 3196 168 12 0 0 getty
[ 7154] 0 7154 46 4 3 0 0 runsvdir
[ 7163] 0 7163 41 1 3 0 0 runsv
[ 7194] 0 7194 1085 127 8 0 0 logger
[ 7195] 0 7195 41 1 3 0 0 runsv
[ 7205] 0 7205 1085 147 8 0 0 logger
[ 9101] 998 9101 9764 1153 24 0 0 python
[ 9136] 109 9136 7478 207 18 0 0 dnsmasq
[17111] 0 17111 3211 77 11 0 0 mcelog
[20922] 0 20922 6289 193 16 0 0 master
[24780] 0 24780 5856 147 16 0 0 rpcbind
[26542] 0 26542 41 1 3 0 0 runsv
[26603] 0 26603 49 1 4 0 0 svlogd
[27249] 0 27249 41 1 3 0 0 runsv
[27398] 0 27398 49 1 4 0 0 svlogd
[29275] 0 29275 41 1 3 0 0 runsv
[29293] 0 29293 1085 79 7 0 0 logger
[ 467] 991 467 8792 1522 15 0 0 redis-server
[ 743] 0 743 41 1 3 0 0 runsv
[ 744] 0 744 49 6 4 0 0 svlogd
[ 1458] 0 1458 41 1 3 0 0 runsv
[ 1459] 0 1459 50 9 4 0 0 svlogd
[ 2423] 993 2423 6897 221 18 0 0 nrpe
[ 2427] 0 2427 19120 3385 34 0 0 ruby
[ 2683] 0 2683 10741 178 25 0 0 syslog-ng
[ 2684] 0 2684 28994 4271 62 0 0 syslog-ng
[ 2692] 0 2692 17384 2810 33 0 0 ruby
[ 2812] 0 2812 7324 353 17 0 0 nginx
[ 8224] 0 8224 41 1 3 0 0 runsv
[ 8269] 0 8269 49 1 4 0 0 svlogd
[12779] 0 12779 41 4 3 0 0 runsv
[12866] 0 12866 1085 146 8 0 0 logger
[12956] 0 12956 41 1 3 0 0 runsv
[13131] 0 13131 1085 146 8 0 0 logger
[13729] 65534 13729 84183 3764 30 0 0 memcached
[13793] 0 13793 4159 778 12 0 0 consul-template
[18314] 0 18314 29462 613 24 0 0 osqueryd
[18316] 0 18316 217702 5851 92 0 0 osqueryd
[21414] 0 21414 15344 294 34 0 -1000 sshd
[23885] 992 23885 167164 1237 39 0 0 raingutter
[23943] 992 23943 56394 22818 115 0 0 bundle
[13766] 0 13766 26264 3464 53 0 0 ruby
[29048] 999 29048 17319 5390 38 0 0 consul
[29867] 113 29867 26431 3748 54 0 0 supervisord
[29891] 113 29891 44805 8701 93 0 0 python
[29892] 113 29892 57087 9218 84 0 0 python
[29896] 113 29896 151802 9574 96 0 0 python
[26311] 33 26311 7846 430 19 0 0 nginx
[ 8234] 0 8234 39672 380 79 0 0 sssd
[ 8238] 0 8238 44386 742 90 0 0 sssd_be
[ 8258] 0 8258 37591 366 75 0 0 sssd_nss
[ 8259] 0 8259 34113 309 69 0 0 sssd_pam
[ 8260] 0 8260 34114 324 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 343 73 0 0 sssd_ssh
[11911] 0 11911 55169 12312 74 0 0 chef-client
[17952] 110 17952 7369 228 19 0 0 qmgr
[14580] 110 14580 10570 334 25 0 0 tlsmgr
[29141] 113 29141 219163 15716 92 0 0 trace-agent
[31674] 0 31674 8844 3691 20 0 0 atop
[19445] 110 19445 7326 219 18 0 0 pickup
[23650] 992 23650 192508 115163 382 0 0 ruby
[26952] 992 26952 270105 175722 529 0 0 ruby
[27739] 992 27739 271129 186408 532 0 0 ruby
[28539] 992 28539 276249 178852 542 0 0 ruby
[29063] 992 29063 279325 180732 541 0 0 ruby
[29458] 992 29458 269852 183843 526 0 0 ruby
[29620] 992 29620 273177 187377 536 0 0 ruby
[30048] 992 30048 273177 181234 536 0 0 ruby
[30203] 992 30203 271133 177167 526 0 0 ruby
[30527] 992 30527 269081 181873 528 0 0 ruby
[30850] 992 30850 3463961 2263256 5451 0 0 ruby
[31149] 992 31149 280345 183765 550 0 0 ruby
[31346] 992 31346 273177 179860 536 0 0 ruby
[31429] 992 31429 571165 188924 1115 0 0 ruby
[31467] 992 31467 264985 174397 520 0 0 ruby
[31515] 992 31515 269085 177066 522 0 0 ruby
[13409] 992 13409 267033 180861 524 0 0 ruby
[14242] 992 14242 270105 179968 530 0 0 ruby
[15343] 992 15343 270105 180992 530 0 0 ruby
[27547] 992 27547 263961 173768 518 0 0 ruby
[30092] 992 30092 268057 178700 527 0 0 ruby
[30822] 992 30822 572185 183538 1121 0 0 ruby
[ 3328] 992 3328 261146 169580 513 0 0 ruby
[ 4480] 992 4480 262937 173392 515 0 0 ruby
[11189] 992 11189 270105 184040 529 0 0 ruby
[14875] 992 14875 269081 174794 527 0 0 ruby
[ 9602] 992 9602 280345 177689 550 0 0 ruby
[24163] 108 24163 7872 288 19 0 0 ntpd
[27139] 992 27139 274205 183191 531 0 0 ruby
[17762] 992 17762 275225 174889 540 0 0 ruby
[21278] 992 21278 256536 163591 506 0 0 ruby
[23913] 0 23913 740714 179181 534 0 0 ruby
[23952] 992 23952 257560 168138 507 0 0 ruby
[25117] 992 25117 245272 164081 484 0 0 ruby
[25738] 992 25738 590617 198572 1156 0 0 ruby
[27011] 0 27011 11715 238 29 0 0 cron
[27023] 0 27023 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27034] 0 27034 2782 155 11 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[27061] 0 27061 1479 126 8 0 0 sleep
[28832] 992 28832 261913 169010 514 0 0 ruby
[ 389] 992 389 262680 175691 517 0 0 ruby
[ 1987] 992 1987 261913 169053 514 0 0 ruby
[ 3425] 992 3425 275225 179937 539 0 0 ruby
[ 4347] 992 4347 258584 171194 510 0 0 ruby
[ 6746] 992 6746 262680 166073 517 0 0 ruby
[10691] 992 10691 212504 132929 418 0 0 ruby
[14760] 110 14760 7326 201 17 0 0 showq
[14951] 992 14951 211480 132667 417 0 0 ruby
[17392] 0 17392 11715 239 29 0 0 cron
[17393] 0 17393 11715 239 29 0 0 cron
[17394] 0 17394 11715 239 29 0 0 cron
[17395] 0 17395 11715 239 29 0 0 cron
[17396] 0 17396 11715 239 29 0 0 cron
[17397] 0 17397 11715 239 29 0 0 cron
[17398] 0 17398 11715 239 29 0 0 cron
[17399] 0 17399 11715 239 29 0 0 cron
[17400] 0 17400 11715 239 29 0 0 cron
[17401] 0 17401 11715 239 29 0 0 cron
[17403] 0 17403 11715 239 29 0 0 cron
[17406] 0 17406 11715 239 29 0 0 cron
[17408] 0 17408 11715 239 29 0 0 cron
[17409] 0 17409 11715 239 29 0 0 cron
[17410] 0 17410 11715 239 29 0 0 cron
[17411] 0 17411 11715 239 29 0 0 cron
[17412] 0 17412 2782 156 10 0 0 bash
[17413] 0 17413 2782 155 10 0 0 bash
[17415] 0 17415 2782 155 10 0 0 bash
[17416] 0 17416 2782 155 10 0 0 bash
[17417] 0 17417 2782 155 10 0 0 bash
[17418] 0 17418 2782 156 10 0 0 bash
[17419] 0 17419 1479 126 8 0 0 sleep
[17420] 0 17420 2782 156 10 0 0 bash
[17421] 0 17421 2782 154 10 0 0 bash
[17422] 0 17422 2782 155 10 0 0 bash
[17423] 0 17423 2782 155 9 0 0 bash
[17424] 0 17424 2782 155 10 0 0 bash
[17425] 0 17425 2782 155 10 0 0 bash
[17426] 0 17426 2782 156 10 0 0 bash
[17427] 0 17427 1479 126 7 0 0 sleep
[17429] 0 17429 1479 126 8 0 0 sleep
[17430] 0 17430 1479 127 8 0 0 sleep
[17431] 0 17431 2782 155 10 0 0 bash
[17432] 0 17432 1479 126 8 0 0 sleep
[17433] 0 17433 1479 126 8 0 0 sleep
[17435] 0 17435 1479 127 8 0 0 sleep
[17436] 0 17436 1479 127 8 0 0 sleep
[17437] 0 17437 2782 154 10 0 0 bash
[17438] 0 17438 1479 127 8 0 0 sleep
[17439] 0 17439 1479 127 8 0 0 sleep
[17440] 0 17440 1479 126 7 0 0 sleep
[17442] 0 17442 1479 126 8 0 0 sleep
[17443] 0 17443 2782 156 11 0 0 bash
[17444] 0 17444 1479 126 8 0 0 sleep
[17445] 0 17445 1479 126 8 0 0 sleep
[17446] 0 17446 1479 126 8 0 0 sleep
[17448] 0 17448 1479 127 8 0 0 sleep
[17668] 113 17668 1088 165 8 0 0 mpstat
Out of memory: Kill process 30850 (ruby) score 282 or sacrifice child
Killed process 30850 (ruby) total-vm:13855844kB, anon-rss:9048904kB, file-rss:536kB
osqueryd invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
osqueryd cpuset=/ mems_allowed=0
CPU: 5 PID: 18504 Comm: osqueryd Not tainted 3.13.0-156-generic #206-Ubuntu
Hardware name: Amazon EC2 m5.2xlarge/, BIOS 1.0 10/16/2017
0000000000000000 ffff880106a25960 ffffffff8173654e ffff880775f49790
0000000000000000 ffff880106a259e8 ffffffff81730abd ffffffff8106d6c6
ffff880106a259c0 ffffffff810cc81c 0000000001320122 0000000000000000
Call Trace:
[] dump_stack+0x64/0x80
[] dump_header+0x7f/0x1f1
[] ? put_online_cpus+0x56/0x80
[] ? rcu_oom_notify+0xcc/0xf0
[] oom_kill_process+0x201/0x360
[] ? security_capable_noaudit+0x1b/0x20
[] out_of_memory+0x471/0x4b0
[] __alloc_pages_nodemask+0xa6c/0xb90
[] alloc_pages_current+0xa3/0x160
[] __page_cache_alloc+0x97/0xc0
[] filemap_fault+0x185/0x400
[] __do_fault+0x75/0x560
[] handle_mm_fault+0x234/0xeb0
[] ? get_futex_key+0x1d8/0x2c0
[] __do_page_fault+0x183/0x570
[] ? do_futex+0x10a/0x760
[] ? mark_tsc_unstable+0x60/0x60
[] do_page_fault+0x1a/0x70
[] do_async_page_fault+0x29/0xe0
[] async_page_fault+0x28/0x30
Mem-Info:
Node 0 DMA per-cpu:
CPU 0: hi: 0, btch: 1 usd: 0
CPU 1: hi: 0, btch: 1 usd: 0
CPU 2: hi: 0, btch: 1 usd: 0
CPU 3: hi: 0, btch: 1 usd: 0
CPU 4: hi: 0, btch: 1 usd: 0
CPU 5: hi: 0, btch: 1 usd: 0
CPU 6: hi: 0, btch: 1 usd: 0
CPU 7: hi: 0, btch: 1 usd: 0
Node 0 DMA32 per-cpu:
CPU 0: hi: 186, btch: 31 usd: 0
CPU 1: hi: 186, btch: 31 usd: 0
CPU 2: hi: 186, btch: 31 usd: 0
CPU 3: hi: 186, btch: 31 usd: 81
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 0
CPU 6: hi: 186, btch: 31 usd: 0
CPU 7: hi: 186, btch: 31 usd: 0
Node 0 Normal per-cpu:
CPU 0: hi: 186, btch: 31 usd: 183
CPU 1: hi: 186, btch: 31 usd: 61
CPU 2: hi: 186, btch: 31 usd: 55
CPU 3: hi: 186, btch: 31 usd: 115
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 54
CPU 6: hi: 186, btch: 31 usd: 24
CPU 7: hi: 186, btch: 31 usd: 30
active_anon:7856739 inactive_anon:39 isolated_anon:0
active_file:109 inactive_file:0 isolated_file:75
unevictable:3776 dirty:20 writeback:0 unstable:0
free:49229 slab_reclaimable:14157 slab_unreclaimable:13646
mapped:552 shmem:101 pagetables:31643 bounce:0
free_cma:0
Node 0 DMA free:15908kB min:32kB low:40kB high:48kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
lowmem_reserve[]: 0 2976 31358 31358
Node 0 DMA32 free:119880kB min:6408kB low:8008kB high:9612kB active_anon:2899188kB inactive_anon:16kB active_file:108kB inactive_file:28kB unevictable:2700kB isolated(anon):0kB isolated(file):0kB present:3129316kB managed:3050168kB mlocked:2700kB dirty:100kB writeback:0kB mapped:388kB shmem:24kB slab_reclaimable:6236kB slab_unreclaimable:4912kB kernel_stack:568kB pagetables:12448kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:2210 all_unreclaimable? yes
lowmem_reserve[]: 0 0 28382 28382
Node 0 Normal free:61128kB min:61136kB low:76420kB high:91704kB active_anon:28527768kB inactive_anon:140kB active_file:328kB inactive_file:0kB unevictable:12404kB isolated(anon):0kB isolated(file):300kB present:29577216kB managed:29063528kB mlocked:12404kB dirty:0kB writeback:0kB mapped:1820kB shmem:380kB slab_reclaimable:50392kB slab_unreclaimable:49672kB kernel_stack:3904kB pagetables:114124kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:548 all_unreclaimable? yes
lowmem_reserve[]: 0 0 0 0
Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15908kB
Node 0 DMA32: 775*4kB (UEM) 352*8kB (UEM) 364*16kB (UEM) 290*32kB (UEM) 214*64kB (UEM) 105*128kB (UEM) 59*256kB (UEM) 28*512kB (UE) 4*1024kB (UE) 5*2048kB (UE) 7*4096kB (UEMR) = 120604kB
Node 0 Normal: 6529*4kB (UEM) 3841*8kB (U) 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 60940kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
415 total pagecache pages
0 pages in swap cache
Swap cache stats: add 0, delete 0, find 0/0
Free swap = 0kB
Total swap = 0kB
8180631 pages RAM
0 pages HighMem/MovableOnly
128422 pages reserved
[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
[ 478] 0 478 4869 74 13 0 0 upstart-udev-br
[ 482] 0 482 12392 245 27 0 -1000 systemd-udevd
[ 642] 0 642 12164 311 24 0 0 lldpd
[ 663] 0 663 3815 70 12 0 0 upstart-socket-
[ 698] 106 698 12164 169 24 0 0 lldpd
[ 736] 0 736 2555 617 8 0 0 dhclient
[ 986] 102 986 11999 201 26 0 0 dbus-daemon
[ 1026] 0 1026 8756 205 22 0 0 systemd-logind
[ 1033] 0 1033 3819 73 12 0 0 upstart-file-br
[ 1078] 0 1078 3634 164 12 0 0 getty
[ 1081] 0 1081 3634 165 12 0 0 getty
[ 1086] 0 1086 3634 166 12 0 0 getty
[ 1088] 0 1088 3634 166 11 0 0 getty
[ 1089] 0 1089 182070 2287 50 0 0 amazon-ssm-agen
[ 1091] 0 1091 3634 167 12 0 0 getty
[ 1126] 0 1126 1092 109 8 0 0 acpid
[ 1127] 0 1127 5913 191 17 0 0 cron
[ 1129] 0 1129 4785 40 12 0 0 atd
[ 1184] 0 1184 4799 148 15 0 0 irqbalance
[ 1302] 0 1302 2243 38 8 0 0 rngd
[ 1453] 0 1453 8487 156 15 0 0 memlockd
[ 1544] 1 1544 1600 112 9 0 0 uptimed
[ 1548] 0 1548 3634 174 12 0 0 getty
[ 1549] 0 1549 3196 168 12 0 0 getty
[ 7154] 0 7154 46 4 3 0 0 runsvdir
[ 7163] 0 7163 41 1 3 0 0 runsv
[ 7194] 0 7194 1085 127 8 0 0 logger
[ 7195] 0 7195 41 1 3 0 0 runsv
[ 7205] 0 7205 1085 147 8 0 0 logger
[ 9101] 998 9101 9764 1156 24 0 0 python
[ 9136] 109 9136 7478 172 18 0 0 dnsmasq
[17111] 0 17111 3211 77 11 0 0 mcelog
[20922] 0 20922 6289 214 16 0 0 master
[24780] 0 24780 5856 139 16 0 0 rpcbind
[26542] 0 26542 41 1 3 0 0 runsv
[26603] 0 26603 49 1 4 0 0 svlogd
[27249] 0 27249 41 1 3 0 0 runsv
[27398] 0 27398 49 1 4 0 0 svlogd
[29275] 0 29275 41 1 3 0 0 runsv
[29293] 0 29293 1085 79 7 0 0 logger
[ 467] 991 467 8792 1514 15 0 0 redis-server
[ 743] 0 743 41 1 3 0 0 runsv
[ 744] 0 744 49 6 4 0 0 svlogd
[ 1458] 0 1458 41 1 3 0 0 runsv
[ 1459] 0 1459 50 10 4 0 0 svlogd
[ 2423] 993 2423 6897 218 18 0 0 nrpe
[ 2427] 0 2427 19120 3389 34 0 0 ruby
[ 2683] 0 2683 10741 178 25 0 0 syslog-ng
[ 2684] 0 2684 28994 4273 62 0 0 syslog-ng
[ 2692] 0 2692 17384 2809 33 0 0 ruby
[ 2812] 0 2812 7324 353 17 0 0 nginx
[ 8224] 0 8224 41 1 3 0 0 runsv
[ 8269] 0 8269 49 1 4 0 0 svlogd
[12779] 0 12779 41 4 3 0 0 runsv
[12866] 0 12866 1085 146 8 0 0 logger
[12956] 0 12956 41 1 3 0 0 runsv
[13131] 0 13131 1085 146 8 0 0 logger
[13729] 65534 13729 84183 3755 30 0 0 memcached
[13793] 0 13793 4159 799 12 0 0 consul-template
[18314] 0 18314 29462 603 24 0 0 osqueryd
[18316] 0 18316 217702 5851 92 0 0 osqueryd
[21414] 0 21414 15344 298 34 0 -1000 sshd
[23885] 992 23885 167164 1277 39 0 0 raingutter
[23943] 992 23943 56394 22810 115 0 0 bundle
[13766] 0 13766 26264 3480 53 0 0 ruby
[29048] 999 29048 17319 5405 38 0 0 consul
[29867] 113 29867 26431 3780 54 0 0 supervisord
[29891] 113 29891 44805 8700 93 0 0 python
[29892] 113 29892 57087 9255 84 0 0 python
[29896] 113 29896 151802 9573 96 0 0 python
[26311] 33 26311 7846 468 19 0 0 nginx
[ 8234] 0 8234 39672 441 79 0 0 sssd
[ 8238] 0 8238 44386 750 90 0 0 sssd_be
[ 8258] 0 8258 37591 408 75 0 0 sssd_nss
[ 8259] 0 8259 34113 372 69 0 0 sssd_pam
[ 8260] 0 8260 34114 387 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 406 73 0 0 sssd_ssh
[11911] 0 11911 55169 12312 74 0 0 chef-client
[17952] 110 17952 7369 228 19 0 0 qmgr
[14580] 110 14580 10570 302 25 0 0 tlsmgr
[29141] 113 29141 219163 15706 92 0 0 trace-agent
[31674] 0 31674 8844 3727 20 0 0 atop
[19445] 110 19445 7326 206 18 0 0 pickup
[23650] 992 23650 192508 115162 382 0 0 ruby
[26952] 992 26952 270105 176821 529 0 0 ruby
[27739] 992 27739 271129 179505 532 0 0 ruby
[28539] 992 28539 276249 177374 542 0 0 ruby
[29063] 992 29063 279325 184815 541 0 0 ruby
[29458] 992 29458 269852 179305 526 0 0 ruby
[29620] 992 29620 273177 179861 536 0 0 ruby
[30048] 992 30048 273177 183500 536 0 0 ruby
[30203] 992 30203 1919773 1220426 3043 0 0 ruby
[30527] 992 30527 269081 180548 528 0 0 ruby
[31149] 992 31149 280345 182411 550 0 0 ruby
[31346] 992 31346 273177 181900 536 0 0 ruby
[31429] 992 31429 571165 184078 1115 0 0 ruby
[31467] 992 31467 264985 179246 520 0 0 ruby
[31515] 992 31515 269085 175892 522 0 0 ruby
[13409] 992 13409 267033 176641 524 0 0 ruby
[14242] 992 14242 270105 178424 530 0 0 ruby
[15343] 992 15343 270105 178529 530 0 0 ruby
[27547] 992 27547 263961 174126 518 0 0 ruby
[30092] 992 30092 268057 181116 527 0 0 ruby
[30822] 992 30822 572185 185136 1121 0 0 ruby
[ 3328] 992 3328 1891354 1090627 2862 0 0 ruby
[ 4480] 992 4480 262937 174223 515 0 0 ruby
[11189] 992 11189 270105 185222 529 0 0 ruby
[14875] 992 14875 269081 175875 527 0 0 ruby
[ 9602] 992 9602 280345 185029 550 0 0 ruby
[24163] 108 24163 7872 300 19 0 0 ntpd
[27139] 992 27139 274205 181504 531 0 0 ruby
[17762] 992 17762 275225 177235 540 0 0 ruby
[21278] 992 21278 256536 168301 506 0 0 ruby
[23913] 0 23913 740714 186300 552 0 0 ruby
[23952] 992 23952 257560 169421 507 0 0 ruby
[25117] 992 25117 245272 166229 484 0 0 ruby
[25738] 992 25738 591641 197455 1157 0 0 ruby
[27011] 0 27011 11715 238 29 0 0 cron
[27023] 0 27023 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27034] 0 27034 2782 155 11 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[27061] 0 27061 1479 126 8 0 0 sleep
[28832] 992 28832 261913 178653 514 0 0 ruby
[ 389] 992 389 263961 175825 518 0 0 ruby
[ 1987] 992 1987 261913 175858 514 0 0 ruby
[ 3425] 992 3425 275225 185176 539 0 0 ruby
[ 4347] 992 4347 258584 173658 510 0 0 ruby
[ 6746] 992 6746 263961 171621 518 0 0 ruby
[10691] 992 10691 270105 169447 529 0 0 ruby
[14760] 110 14760 7326 171 17 0 0 showq
[14951] 992 14951 263704 174899 518 0 0 ruby
[17860] 992 17860 258841 164276 510 0 0 ruby
[20360] 0 20360 11715 238 29 0 0 cron
[20389] 0 20389 2782 155 10 0 0 bash
[20410] 0 20410 1479 126 7 0 0 sleep
[25556] 0 25556 11715 238 29 0 0 cron
[25557] 0 25557 11715 238 29 0 0 cron
[25559] 0 25559 11715 238 29 0 0 cron
[25560] 0 25560 11715 238 29 0 0 cron
[25561] 0 25561 11715 238 29 0 0 cron
[25562] 0 25562 11715 238 29 0 0 cron
[25563] 0 25563 11715 238 29 0 0 cron
[25564] 0 25564 11715 238 29 0 0 cron
[25567] 0 25567 11715 238 29 0 0 cron
[25570] 0 25570 11715 238 29 0 0 cron
[25571] 0 25571 11715 238 29 0 0 cron
[25572] 0 25572 11715 238 29 0 0 cron
[25573] 0 25573 11715 238 29 0 0 cron
[25574] 0 25574 11715 238 29 0 0 cron
[25577] 0 25577 2782 155 10 0 0 bash
[25578] 0 25578 2782 155 10 0 0 bash
[25579] 0 25579 2782 155 9 0 0 bash
[25583] 0 25583 2782 154 10 0 0 bash
[25584] 0 25584 1479 126 8 0 0 sleep
[25586] 0 25586 2782 155 10 0 0 bash
[25587] 0 25587 2782 155 10 0 0 bash
[25588] 0 25588 1479 126 8 0 0 sleep
[25589] 0 25589 2782 154 9 0 0 bash
[25590] 0 25590 2782 155 9 0 0 bash
[25592] 0 25592 1479 126 8 0 0 sleep
[25593] 0 25593 1479 127 8 0 0 sleep
[25594] 0 25594 2782 155 10 0 0 bash
[25595] 0 25595 2782 155 10 0 0 bash
[25596] 0 25596 2782 156 10 0 0 bash
[25598] 0 25598 1479 127 8 0 0 sleep
[25599] 0 25599 1479 126 8 0 0 sleep
[25600] 0 25600 2782 155 9 0 0 bash
[25601] 0 25601 2782 155 10 0 0 bash
[25602] 0 25602 2782 155 10 0 0 bash
[25603] 0 25603 1479 127 8 0 0 sleep
[25605] 0 25605 1479 126 7 0 0 sleep
[25606] 0 25606 1479 127 8 0 0 sleep
[25607] 0 25607 1479 127 7 0 0 sleep
[25608] 0 25608 1479 127 7 0 0 sleep
[25611] 0 25611 1479 127 7 0 0 sleep
[25615] 0 25615 1479 126 8 0 0 sleep
[25616] 0 25616 1479 126 8 0 0 sleep
[26246] 113 26246 1090 162 7 0 0 iostat
Out of memory: Kill process 30203 (ruby) score 152 or sacrifice child
Killed process 30203 (ruby) total-vm:7679092kB, anon-rss:4881132kB, file-rss:572kB
consul invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
consul cpuset=/ mems_allowed=0
CPU: 1 PID: 29094 Comm: consul Not tainted 3.13.0-156-generic #206-Ubuntu
Hardware name: Amazon EC2 m5.2xlarge/, BIOS 1.0 10/16/2017
0000000000000000 ffff88009649d960 ffffffff8173654e ffff8807dfdec6b0
0000000000000000 ffff88009649d9e8 ffffffff81730abd ffffffff8106d6c6
ffff88009649d9c0 ffffffff810cc81c ffff88009649da18 0000000000000001
Call Trace:
[] dump_stack+0x64/0x80
[] dump_header+0x7f/0x1f1
[] ? put_online_cpus+0x56/0x80
[] ? rcu_oom_notify+0xcc/0xf0
[] oom_kill_process+0x201/0x360
[] ? security_capable_noaudit+0x1b/0x20
[] out_of_memory+0x471/0x4b0
[] __alloc_pages_nodemask+0xa6c/0xb90
[] alloc_pages_current+0xa3/0x160
[] __page_cache_alloc+0x97/0xc0
[] filemap_fault+0x185/0x400
[] __do_fault+0x75/0x560
[] handle_mm_fault+0x234/0xeb0
[] ? try_to_wake_up+0x203/0x2e0
[] __do_page_fault+0x183/0x570
[] ? do_futex+0x10a/0x760
[] ? ktime_get_ts+0x4d/0xf0
[] ? poll_select_copy_remaining+0x101/0x130
[] do_page_fault+0x1a/0x70
[] do_async_page_fault+0x29/0xe0
[] async_page_fault+0x28/0x30
Mem-Info:
Node 0 DMA per-cpu:
CPU 0: hi: 0, btch: 1 usd: 0
CPU 1: hi: 0, btch: 1 usd: 0
CPU 2: hi: 0, btch: 1 usd: 0
CPU 3: hi: 0, btch: 1 usd: 0
CPU 4: hi: 0, btch: 1 usd: 0
CPU 5: hi: 0, btch: 1 usd: 0
CPU 6: hi: 0, btch: 1 usd: 0
CPU 7: hi: 0, btch: 1 usd: 0
Node 0 DMA32 per-cpu:
CPU 0: hi: 186, btch: 31 usd: 0
CPU 1: hi: 186, btch: 31 usd: 0
CPU 2: hi: 186, btch: 31 usd: 0
CPU 3: hi: 186, btch: 31 usd: 0
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 0
CPU 6: hi: 186, btch: 31 usd: 0
CPU 7: hi: 186, btch: 31 usd: 0
Node 0 Normal per-cpu:
CPU 0: hi: 186, btch: 31 usd: 185
CPU 1: hi: 186, btch: 31 usd: 27
CPU 2: hi: 186, btch: 31 usd: 20
CPU 3: hi: 186, btch: 31 usd: 0
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 0
CPU 6: hi: 186, btch: 31 usd: 30
CPU 7: hi: 186, btch: 31 usd: 0
active_anon:7857379 inactive_anon:39 isolated_anon:0
active_file:0 inactive_file:0 isolated_file:32
unevictable:3776 dirty:5 writeback:0 unstable:0
free:49216 slab_reclaimable:13904 slab_unreclaimable:13600
mapped:173 shmem:101 pagetables:31758 bounce:0
free_cma:0
Node 0 DMA free:15908kB min:32kB low:40kB high:48kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
lowmem_reserve[]: 0 2976 31358 31358
Node 0 DMA32 free:119692kB min:6408kB low:8008kB high:9612kB active_anon:2899008kB inactive_anon:16kB active_file:44kB inactive_file:0kB unevictable:2700kB isolated(anon):0kB isolated(file):0kB present:3129316kB managed:3050168kB mlocked:2700kB dirty:0kB writeback:0kB mapped:268kB shmem:24kB slab_reclaimable:5984kB slab_unreclaimable:4884kB kernel_stack:584kB pagetables:12424kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:71 all_unreclaimable? yes
lowmem_reserve[]: 0 0 28382 28382
Node 0 Normal free:61264kB min:61136kB low:76420kB high:91704kB active_anon:28530508kB inactive_anon:140kB active_file:0kB inactive_file:0kB unevictable:12404kB isolated(anon):0kB isolated(file):128kB present:29577216kB managed:29063528kB mlocked:12404kB dirty:20kB writeback:0kB mapped:424kB shmem:380kB slab_reclaimable:49632kB slab_unreclaimable:49516kB kernel_stack:3816kB pagetables:114608kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:64 all_unreclaimable? yes
lowmem_reserve[]: 0 0 0 0
Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15908kB
Node 0 DMA32: 611*4kB (UEM) 290*8kB (UEM) 345*16kB (UEM) 314*32kB (UEM) 220*64kB (UEM) 105*128kB (UEM) 59*256kB (UEM) 28*512kB (UE) 4*1024kB (UE) 5*2048kB (UE) 7*4096kB (UEMR) = 120300kB
Node 0 Normal: 8892*4kB (UEM) 2730*8kB (UEM) 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 61504kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
456 total pagecache pages
0 pages in swap cache
Swap cache stats: add 0, delete 0, find 0/0
Free swap = 0kB
Total swap = 0kB
8180631 pages RAM
0 pages HighMem/MovableOnly
128422 pages reserved
[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
[ 478] 0 478 4869 74 13 0 0 upstart-udev-br
[ 482] 0 482 12392 245 27 0 -1000 systemd-udevd
[ 642] 0 642 12164 311 24 0 0 lldpd
[ 663] 0 663 3815 70 12 0 0 upstart-socket-
[ 698] 106 698 12164 147 24 0 0 lldpd
[ 736] 0 736 2555 617 8 0 0 dhclient
[ 986] 102 986 11999 201 26 0 0 dbus-daemon
[ 1026] 0 1026 8756 205 22 0 0 systemd-logind
[ 1033] 0 1033 3819 73 12 0 0 upstart-file-br
[ 1078] 0 1078 3634 164 12 0 0 getty
[ 1081] 0 1081 3634 165 12 0 0 getty
[ 1086] 0 1086 3634 166 12 0 0 getty
[ 1088] 0 1088 3634 166 11 0 0 getty
[ 1089] 0 1089 182070 2287 50 0 0 amazon-ssm-agen
[ 1091] 0 1091 3634 167 12 0 0 getty
[ 1126] 0 1126 1092 109 8 0 0 acpid
[ 1127] 0 1127 5913 189 17 0 0 cron
[ 1129] 0 1129 4785 40 12 0 0 atd
[ 1184] 0 1184 4799 148 15 0 0 irqbalance
[ 1302] 0 1302 2243 32 8 0 0 rngd
[ 1453] 0 1453 8487 156 15 0 0 memlockd
[ 1544] 1 1544 1600 112 9 0 0 uptimed
[ 1548] 0 1548 3634 174 12 0 0 getty
[ 1549] 0 1549 3196 168 12 0 0 getty
[ 7154] 0 7154 46 4 3 0 0 runsvdir
[ 7163] 0 7163 41 1 3 0 0 runsv
[ 7194] 0 7194 1085 127 8 0 0 logger
[ 7195] 0 7195 41 1 3 0 0 runsv
[ 7205] 0 7205 1085 147 8 0 0 logger
[ 9101] 998 9101 9764 1156 24 0 0 python
[ 9136] 109 9136 7478 176 18 0 0 dnsmasq
[17111] 0 17111 3211 77 11 0 0 mcelog
[20922] 0 20922 6289 214 16 0 0 master
[24780] 0 24780 5856 179 16 0 0 rpcbind
[26542] 0 26542 41 1 3 0 0 runsv
[26603] 0 26603 49 1 4 0 0 svlogd
[27249] 0 27249 41 1 3 0 0 runsv
[27398] 0 27398 49 1 4 0 0 svlogd
[29275] 0 29275 41 1 3 0 0 runsv
[29293] 0 29293 1085 79 7 0 0 logger
[ 467] 991 467 8792 1516 15 0 0 redis-server
[ 743] 0 743 41 1 3 0 0 runsv
[ 744] 0 744 49 6 4 0 0 svlogd
[ 1458] 0 1458 41 1 3 0 0 runsv
[ 1459] 0 1459 50 9 4 0 0 svlogd
[ 2423] 993 2423 6897 218 18 0 0 nrpe
[ 2427] 0 2427 19120 3350 34 0 0 ruby
[ 2683] 0 2683 10741 178 25 0 0 syslog-ng
[ 2684] 0 2684 28994 4246 62 0 0 syslog-ng
[ 2692] 0 2692 17384 2795 33 0 0 ruby
[ 2812] 0 2812 7324 353 17 0 0 nginx
[ 8224] 0 8224 41 1 3 0 0 runsv
[ 8269] 0 8269 49 1 4 0 0 svlogd
[12779] 0 12779 41 4 3 0 0 runsv
[12866] 0 12866 1085 146 8 0 0 logger
[12956] 0 12956 41 1 3 0 0 runsv
[13131] 0 13131 1085 146 8 0 0 logger
[13729] 65534 13729 84183 3755 30 0 0 memcached
[13793] 0 13793 4159 799 12 0 0 consul-template
[18314] 0 18314 29462 650 24 0 0 osqueryd
[18316] 0 18316 217702 5851 92 0 0 osqueryd
[21414] 0 21414 15344 298 34 0 -1000 sshd
[23885] 992 23885 167164 1237 39 0 0 raingutter
[23943] 992 23943 56394 22837 115 0 0 bundle
[13766] 0 13766 26264 3483 53 0 0 ruby
[29048] 999 29048 17319 5417 38 0 0 consul
[29867] 113 29867 26431 3767 54 0 0 supervisord
[29891] 113 29891 44805 8700 93 0 0 python
[29892] 113 29892 57087 9237 84 0 0 python
[29896] 113 29896 151800 9573 96 0 0 python
[26311] 33 26311 7846 440 19 0 0 nginx
[ 8234] 0 8234 39672 382 79 0 0 sssd
[ 8238] 0 8238 44386 747 90 0 0 sssd_be
[ 8258] 0 8258 37591 346 75 0 0 sssd_nss
[ 8259] 0 8259 34113 311 69 0 0 sssd_pam
[ 8260] 0 8260 34114 326 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 345 73 0 0 sssd_ssh
[11911] 0 11911 55169 12312 74 0 0 chef-client
[17952] 110 17952 7369 228 19 0 0 qmgr
[14580] 110 14580 10570 302 25 0 0 tlsmgr
[29141] 113 29141 219163 15829 92 0 0 trace-agent
[31674] 0 31674 8844 3727 20 0 0 atop
[19445] 110 19445 7326 206 18 0 0 pickup
[23650] 992 23650 192508 115159 382 0 0 ruby
[26952] 992 26952 270105 177287 529 0 0 ruby
[27739] 992 27739 271129 179517 532 0 0 ruby
[28539] 992 28539 276249 178848 542 0 0 ruby
[29063] 992 29063 279325 181603 541 0 0 ruby
[29458] 992 29458 269852 177324 526 0 0 ruby
[29620] 992 29620 273177 179833 536 0 0 ruby
[30048] 992 30048 273177 183644 536 0 0 ruby
[30527] 992 30527 269081 182127 528 0 0 ruby
[31149] 992 31149 280345 183769 550 0 0 ruby
[31346] 992 31346 273177 182558 536 0 0 ruby
[31429] 992 31429 571165 184603 1115 0 0 ruby
[31467] 992 31467 264985 175428 520 0 0 ruby
[31515] 992 31515 269085 170564 522 0 0 ruby
[13409] 992 13409 267033 183722 524 0 0 ruby
[14242] 992 14242 270105 178700 530 0 0 ruby
[15343] 992 15343 270105 180391 530 0 0 ruby
[27547] 992 27547 263961 175535 518 0 0 ruby
[30092] 992 30092 268057 175758 527 0 0 ruby
[30822] 992 30822 572185 185397 1121 0 0 ruby
[ 3328] 992 3328 2878490 1679653 4307 0 0 ruby
[ 4480] 992 4480 262937 174262 515 0 0 ruby
[11189] 992 11189 270105 178997 529 0 0 ruby
[14875] 992 14875 269081 175824 527 0 0 ruby
[ 9602] 992 9602 280345 177573 550 0 0 ruby
[24163] 108 24163 7872 276 19 0 0 ntpd
[27139] 992 27139 274205 181467 531 0 0 ruby
[17762] 992 17762 275225 177905 540 0 0 ruby
[21278] 992 21278 256536 168513 506 0 0 ruby
[23913] 0 23913 740714 186877 553 0 0 ruby
[23952] 992 23952 257560 169604 507 0 0 ruby
[25117] 992 25117 246296 160532 485 0 0 ruby
[25738] 992 25738 591641 197753 1157 0 0 ruby
[27011] 0 27011 11715 238 29 0 0 cron
[27023] 0 27023 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27034] 0 27034 2782 155 11 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[27061] 0 27061 1479 126 8 0 0 sleep
[28832] 992 28832 261913 178628 514 0 0 ruby
[ 389] 992 389 263961 175885 518 0 0 ruby
[ 1987] 992 1987 261913 171125 514 0 0 ruby
[ 3425] 992 3425 275225 178378 539 0 0 ruby
[ 4347] 992 4347 1133080 739556 1905 0 0 ruby
[ 6746] 992 6746 263961 171590 518 0 0 ruby
[10691] 992 10691 270105 169959 529 0 0 ruby
[14760] 110 14760 7326 171 17 0 0 showq
[14951] 992 14951 263704 175384 518 0 0 ruby
[17860] 992 17860 258841 164633 510 0 0 ruby
[20360] 0 20360 11715 238 29 0 0 cron
[20389] 0 20389 2782 155 10 0 0 bash
[20410] 0 20410 1479 126 7 0 0 sleep
[25556] 0 25556 11715 238 29 0 0 cron
[25559] 0 25559 11715 238 29 0 0 cron
[25560] 0 25560 11715 238 29 0 0 cron
[25562] 0 25562 11715 238 29 0 0 cron
[25563] 0 25563 11715 238 29 0 0 cron
[25564] 0 25564 11715 238 29 0 0 cron
[25570] 0 25570 11715 238 29 0 0 cron
[25571] 0 25571 11715 238 29 0 0 cron
[25572] 0 25572 11715 238 29 0 0 cron
[25573] 0 25573 11715 238 29 0 0 cron
[25574] 0 25574 11715 238 29 0 0 cron
[25577] 0 25577 2782 155 10 0 0 bash
[25579] 0 25579 2782 155 9 0 0 bash
[25587] 0 25587 2782 155 10 0 0 bash
[25588] 0 25588 1479 126 8 0 0 sleep
[25589] 0 25589 2782 154 9 0 0 bash
[25590] 0 25590 2782 155 9 0 0 bash
[25593] 0 25593 1479 127 8 0 0 sleep
[25594] 0 25594 2782 155 10 0 0 bash
[25595] 0 25595 2782 155 10 0 0 bash
[25596] 0 25596 2782 156 10 0 0 bash
[25599] 0 25599 1479 126 8 0 0 sleep
[25600] 0 25600 2782 155 9 0 0 bash
[25601] 0 25601 2782 155 10 0 0 bash
[25602] 0 25602 2782 155 10 0 0 bash
[25603] 0 25603 1479 127 8 0 0 sleep
[25605] 0 25605 1479 126 7 0 0 sleep
[25606] 0 25606 1479 127 8 0 0 sleep
[25607] 0 25607 1479 127 7 0 0 sleep
[25608] 0 25608 1479 127 7 0 0 sleep
[25611] 0 25611 1479 127 7 0 0 sleep
[25615] 0 25615 1479 126 8 0 0 sleep
[25616] 0 25616 1479 126 8 0 0 sleep
[26329] 992 26329 195580 117475 383 0 0 ruby
Out of memory: Kill process 3328 (ruby) score 209 or sacrifice child
Killed process 3328 (ruby) total-vm:11513960kB, anon-rss:6718204kB, file-rss:408kB
python invoked oom-killer: gfp_mask=0x201da, order=0, oom_score_adj=0
python cpuset=/ mems_allowed=0
CPU: 4 PID: 9101 Comm: python Not tainted 3.13.0-156-generic #206-Ubuntu
Hardware name: Amazon EC2 m5.2xlarge/, BIOS 1.0 10/16/2017
0000000000000000 ffff8807deeaf960 ffffffff8173654e ffff8807d713af20
0000000000000000 ffff8807deeaf9e8 ffffffff81730abd ffffffff8106d6c6
ffff8807deeaf9c0 ffffffff810cc81c 0000000001320122 0000000000000000
Call Trace:
[] dump_stack+0x64/0x80
[] dump_header+0x7f/0x1f1
[] ? put_online_cpus+0x56/0x80
[] ? rcu_oom_notify+0xcc/0xf0
[] oom_kill_process+0x201/0x360
[] ? security_capable_noaudit+0x1b/0x20
[] out_of_memory+0x471/0x4b0
[] __alloc_pages_nodemask+0xa6c/0xb90
[] alloc_pages_current+0xa3/0x160
[] __page_cache_alloc+0x97/0xc0
[] filemap_fault+0x185/0x400
[] __do_fault+0x75/0x560
[] ? do_wp_page+0x3a8/0x810
[] handle_mm_fault+0x234/0xeb0
[] ? core_sys_select+0x236/0x2e0
[] __do_page_fault+0x183/0x570
[] ? read_tsc+0x9/0x20
[] ? ktime_get_ts+0x4d/0xf0
[] ? poll_select_copy_remaining+0xdc/0x130
[] do_page_fault+0x1a/0x70
[] ? SyS_select+0xbd/0x100
[] do_async_page_fault+0x29/0xe0
[] async_page_fault+0x28/0x30
Mem-Info:
Node 0 DMA per-cpu:
CPU 0: hi: 0, btch: 1 usd: 0
CPU 1: hi: 0, btch: 1 usd: 0
CPU 2: hi: 0, btch: 1 usd: 0
CPU 3: hi: 0, btch: 1 usd: 0
CPU 4: hi: 0, btch: 1 usd: 0
CPU 5: hi: 0, btch: 1 usd: 0
CPU 6: hi: 0, btch: 1 usd: 0
CPU 7: hi: 0, btch: 1 usd: 0
Node 0 DMA32 per-cpu:
CPU 0: hi: 186, btch: 31 usd: 14
CPU 1: hi: 186, btch: 31 usd: 0
CPU 2: hi: 186, btch: 31 usd: 0
CPU 3: hi: 186, btch: 31 usd: 0
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 0
CPU 6: hi: 186, btch: 31 usd: 0
CPU 7: hi: 186, btch: 31 usd: 0
Node 0 Normal per-cpu:
CPU 0: hi: 186, btch: 31 usd: 173
CPU 1: hi: 186, btch: 31 usd: 0
CPU 2: hi: 186, btch: 31 usd: 30
CPU 3: hi: 186, btch: 31 usd: 15
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 0
CPU 6: hi: 186, btch: 31 usd: 0
CPU 7: hi: 186, btch: 31 usd: 0
active_anon:7856604 inactive_anon:39 isolated_anon:0
active_file:136 inactive_file:0 isolated_file:79
unevictable:3776 dirty:88 writeback:0 unstable:0
free:49186 slab_reclaimable:13971 slab_unreclaimable:13611
mapped:268 shmem:101 pagetables:32010 bounce:0
free_cma:0
Node 0 DMA free:15908kB min:32kB low:40kB high:48kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
lowmem_reserve[]: 0 2976 31358 31358
Node 0 DMA32 free:119752kB min:6408kB low:8008kB high:9612kB active_anon:2899648kB inactive_anon:16kB active_file:84kB inactive_file:280kB unevictable:2700kB isolated(anon):0kB isolated(file):0kB present:3129316kB managed:3050168kB mlocked:2700kB dirty:96kB writeback:0kB mapped:156kB shmem:24kB slab_reclaimable:5992kB slab_unreclaimable:4912kB kernel_stack:568kB pagetables:11924kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:566 all_unreclaimable? yes
lowmem_reserve[]: 0 0 28382 28382
Node 0 Normal free:61084kB min:61136kB low:76420kB high:91704kB active_anon:28526768kB inactive_anon:140kB active_file:460kB inactive_file:0kB unevictable:12404kB isolated(anon):0kB isolated(file):316kB present:29577216kB managed:29063528kB mlocked:12404kB dirty:256kB writeback:0kB mapped:916kB shmem:380kB slab_reclaimable:49892kB slab_unreclaimable:49532kB kernel_stack:4032kB pagetables:116116kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:811 all_unreclaimable? yes
lowmem_reserve[]: 0 0 0 0
Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15908kB
Node 0 DMA32: 904*4kB (UEM) 384*8kB (UEM) 340*16kB (UEM) 288*32kB (UEM) 199*64kB (UEM) 105*128kB (UEM) 59*256kB (UEM) 28*512kB (UE) 4*1024kB (UE) 5*2048kB (UE) 7*4096kB (UEMR) = 119968kB
Node 0 Normal: 5934*4kB (UE) 4202*8kB (U) 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 61448kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
372 total pagecache pages
0 pages in swap cache
Swap cache stats: add 0, delete 0, find 0/0
Free swap = 0kB
Total swap = 0kB
8180631 pages RAM
0 pages HighMem/MovableOnly
128422 pages reserved
[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
[ 478] 0 478 4869 74 13 0 0 upstart-udev-br
[ 482] 0 482 12392 245 27 0 -1000 systemd-udevd
[ 642] 0 642 12164 311 24 0 0 lldpd
[ 663] 0 663 3815 70 12 0 0 upstart-socket-
[ 698] 106 698 12164 125 24 0 0 lldpd
[ 736] 0 736 2555 617 8 0 0 dhclient
[ 986] 102 986 11999 201 26 0 0 dbus-daemon
[ 1026] 0 1026 8756 205 22 0 0 systemd-logind
[ 1033] 0 1033 3819 73 12 0 0 upstart-file-br
[ 1078] 0 1078 3634 164 12 0 0 getty
[ 1081] 0 1081 3634 165 12 0 0 getty
[ 1086] 0 1086 3634 166 12 0 0 getty
[ 1088] 0 1088 3634 166 11 0 0 getty
[ 1089] 0 1089 182070 2287 50 0 0 amazon-ssm-agen
[ 1091] 0 1091 3634 167 12 0 0 getty
[ 1126] 0 1126 1092 109 8 0 0 acpid
[ 1127] 0 1127 5913 190 17 0 0 cron
[ 1129] 0 1129 4785 40 12 0 0 atd
[ 1184] 0 1184 4799 148 15 0 0 irqbalance
[ 1302] 0 1302 2243 74 8 0 0 rngd
[ 1453] 0 1453 8487 156 15 0 0 memlockd
[ 1544] 1 1544 1600 112 9 0 0 uptimed
[ 1548] 0 1548 3634 174 12 0 0 getty
[ 1549] 0 1549 3196 168 12 0 0 getty
[ 7154] 0 7154 46 4 3 0 0 runsvdir
[ 7163] 0 7163 41 1 3 0 0 runsv
[ 7194] 0 7194 1085 127 8 0 0 logger
[ 7195] 0 7195 41 1 3 0 0 runsv
[ 7205] 0 7205 1085 147 8 0 0 logger
[ 9101] 998 9101 9764 1156 24 0 0 python
[ 9136] 109 9136 7478 203 18 0 0 dnsmasq
[17111] 0 17111 3211 77 11 0 0 mcelog
[20922] 0 20922 6289 193 16 0 0 master
[24780] 0 24780 5856 171 16 0 0 rpcbind
[26542] 0 26542 41 1 3 0 0 runsv
[26603] 0 26603 49 1 4 0 0 svlogd
[27249] 0 27249 41 1 3 0 0 runsv
[27398] 0 27398 49 1 4 0 0 svlogd
[29275] 0 29275 41 1 3 0 0 runsv
[29293] 0 29293 1085 79 7 0 0 logger
[ 467] 991 467 8792 1535 15 0 0 redis-server
[ 743] 0 743 41 1 3 0 0 runsv
[ 744] 0 744 49 6 4 0 0 svlogd
[ 1458] 0 1458 41 1 3 0 0 runsv
[ 1459] 0 1459 50 9 4 0 0 svlogd
[ 2423] 993 2423 6897 215 18 0 0 nrpe
[ 2427] 0 2427 19120 3333 34 0 0 ruby
[ 2683] 0 2683 10741 178 25 0 0 syslog-ng
[ 2684] 0 2684 28994 4272 62 0 0 syslog-ng
[ 2692] 0 2692 17384 2809 33 0 0 ruby
[ 2812] 0 2812 7324 353 17 0 0 nginx
[ 8224] 0 8224 41 1 3 0 0 runsv
[ 8269] 0 8269 49 1 4 0 0 svlogd
[12779] 0 12779 41 4 3 0 0 runsv
[12866] 0 12866 1085 146 8 0 0 logger
[12956] 0 12956 41 1 3 0 0 runsv
[13131] 0 13131 1085 146 8 0 0 logger
[13729] 65534 13729 84183 3755 30 0 0 memcached
[13793] 0 13793 4159 799 12 0 0 consul-template
[18314] 0 18314 29462 629 24 0 0 osqueryd
[18316] 0 18316 217702 5867 92 0 0 osqueryd
[21414] 0 21414 15344 309 34 0 -1000 sshd
[23885] 992 23885 167164 1237 39 0 0 raingutter
[23943] 992 23943 56394 22807 115 0 0 bundle
[13766] 0 13766 26264 3483 53 0 0 ruby
[29048] 999 29048 17319 5417 38 0 0 consul
[29867] 113 29867 26431 3790 54 0 0 supervisord
[29891] 113 29891 44805 8700 93 0 0 python
[29892] 113 29892 57087 9235 84 0 0 python
[29896] 113 29896 151800 9573 96 0 0 python
[26311] 33 26311 7846 471 19 0 0 nginx
[ 8234] 0 8234 39672 381 79 0 0 sssd
[ 8238] 0 8238 44386 748 90 0 0 sssd_be
[ 8258] 0 8258 37591 363 75 0 0 sssd_nss
[ 8259] 0 8259 34113 374 69 0 0 sssd_pam
[ 8260] 0 8260 34114 328 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 384 73 0 0 sssd_ssh
[11911] 0 11911 55169 12312 74 0 0 chef-client
[17952] 110 17952 7369 208 19 0 0 qmgr
[14580] 110 14580 10570 302 25 0 0 tlsmgr
[29141] 113 29141 219163 15829 92 0 0 trace-agent
[31674] 0 31674 8844 3727 20 0 0 atop
[19445] 110 19445 7326 193 18 0 0 pickup
[23650] 992 23650 192508 115162 382 0 0 ruby
[26952] 992 26952 270105 176145 529 0 0 ruby
[27739] 992 27739 271129 183083 532 0 0 ruby
[28539] 992 28539 276249 179323 542 0 0 ruby
[29063] 992 29063 279325 180854 541 0 0 ruby
[29458] 992 29458 269852 177502 526 0 0 ruby
[29620] 992 29620 273177 180470 536 0 0 ruby
[30048] 992 30048 273177 181969 536 0 0 ruby
[30527] 992 30527 269081 175903 528 0 0 ruby
[31149] 992 31149 280345 186260 550 0 0 ruby
[31346] 992 31346 273177 178382 536 0 0 ruby
[31429] 992 31429 571165 185338 1115 0 0 ruby
[31467] 992 31467 264985 178003 520 0 0 ruby
[31515] 992 31515 269085 170911 522 0 0 ruby
[13409] 992 13409 267033 176312 524 0 0 ruby
[14242] 992 14242 270105 179916 530 0 0 ruby
[15343] 992 15343 270105 176648 530 0 0 ruby
[27547] 992 27547 263961 172690 518 0 0 ruby
[30092] 992 30092 268057 176121 527 0 0 ruby
[30822] 992 30822 572185 187429 1121 0 0 ruby
[ 4480] 992 4480 262937 175799 515 0 0 ruby
[11189] 992 11189 270105 181460 529 0 0 ruby
[14875] 992 14875 269081 176513 527 0 0 ruby
[ 9602] 992 9602 280345 181964 550 0 0 ruby
[24163] 108 24163 7872 274 19 0 0 ntpd
[27139] 992 27139 274205 178060 531 0 0 ruby
[17762] 992 17762 275225 172804 540 0 0 ruby
[21278] 992 21278 256536 169079 506 0 0 ruby
[23913] 0 23913 757098 188275 556 0 0 ruby
[23952] 992 23952 257560 169922 507 0 0 ruby
[25117] 992 25117 246296 163255 485 0 0 ruby
[25738] 992 25738 591641 194308 1157 0 0 ruby
[27011] 0 27011 11715 238 29 0 0 cron
[27023] 0 27023 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27034] 0 27034 2782 155 11 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[27061] 0 27061 1479 126 8 0 0 sleep
[28832] 992 28832 261913 178676 514 0 0 ruby
[ 389] 992 389 263961 173393 518 0 0 ruby
[ 1987] 992 1987 261913 172302 514 0 0 ruby
[ 3425] 992 3425 275225 181263 539 0 0 ruby
[ 4347] 992 4347 3610136 2316209 5702 0 0 ruby
[ 6746] 992 6746 263961 171817 518 0 0 ruby
[10691] 992 10691 270105 171133 529 0 0 ruby
[14760] 110 14760 7326 204 17 0 0 showq
[14951] 992 14951 263704 172551 518 0 0 ruby
[17860] 992 17860 258841 165190 510 0 0 ruby
[20360] 0 20360 11715 238 29 0 0 cron
[20389] 0 20389 2782 155 10 0 0 bash
[20410] 0 20410 1479 126 7 0 0 sleep
[26329] 992 26329 207641 126457 407 0 0 ruby
[26921] 992 26921 195580 116722 382 0 0 ruby
[27989] 0 27989 11715 238 29 0 0 cron
[27990] 0 27990 11715 238 29 0 0 cron
[27991] 0 27991 11715 238 29 0 0 cron
[27992] 0 27992 11715 238 29 0 0 cron
[27993] 0 27993 11715 238 29 0 0 cron
[27994] 0 27994 11715 238 29 0 0 cron
[27995] 0 27995 11715 238 29 0 0 cron
[27996] 0 27996 11715 238 29 0 0 cron
[27997] 0 27997 11715 238 29 0 0 cron
[27998] 0 27998 11715 238 29 0 0 cron
[28000] 0 28000 11715 238 29 0 0 cron
[28003] 0 28003 11715 238 29 0 0 cron
[28004] 0 28004 11715 238 29 0 0 cron
[28005] 0 28005 11715 238 29 0 0 cron
[28006] 0 28006 11715 238 29 0 0 cron
[28007] 0 28007 11715 238 29 0 0 cron
[28008] 0 28008 2782 155 9 0 0 bash
[28009] 0 28009 2782 154 10 0 0 bash
[28010] 0 28010 2782 155 10 0 0 bash
[28011] 0 28011 2782 156 9 0 0 bash
[28014] 0 28014 2782 155 9 0 0 bash
[28015] 0 28015 2782 155 9 0 0 bash
[28016] 0 28016 2782 156 10 0 0 bash
[28017] 0 28017 1479 126 8 0 0 sleep
[28018] 0 28018 2782 155 9 0 0 bash
[28019] 0 28019 1479 127 8 0 0 sleep
[28021] 0 28021 2782 154 10 0 0 bash
[28022] 0 28022 1479 126 8 0 0 sleep
[28023] 0 28023 2782 154 10 0 0 bash
[28024] 0 28024 1479 127 8 0 0 sleep
[28025] 0 28025 2782 155 10 0 0 bash
[28026] 0 28026 2782 155 9 0 0 bash
[28028] 0 28028 2782 156 10 0 0 bash
[28029] 0 28029 1479 127 8 0 0 sleep
[28030] 0 28030 2782 156 10 0 0 bash
[28031] 0 28031 1479 126 8 0 0 sleep
[28032] 0 28032 1479 126 7 0 0 sleep
[28033] 0 28033 1479 126 8 0 0 sleep
[28034] 0 28034 2782 155 10 0 0 bash
[28035] 0 28035 1479 126 7 0 0 sleep
[28036] 0 28036 1479 127 8 0 0 sleep
[28037] 0 28037 1479 126 7 0 0 sleep
[28038] 0 28038 1479 127 8 0 0 sleep
[28039] 0 28039 2782 155 9 0 0 bash
[28040] 0 28040 1479 127 8 0 0 sleep
[28041] 0 28041 1479 126 8 0 0 sleep
[28043] 0 28043 1479 127 8 0 0 sleep
[28044] 0 28044 1479 127 8 0 0 sleep
Out of memory: Kill process 4347 (ruby) score 289 or sacrifice child
Killed process 4347 (ruby) total-vm:14440544kB, anon-rss:9264464kB, file-rss:372kB
ruby invoked oom-killer: gfp_mask=0x280da, order=0, oom_score_adj=0
ruby cpuset=/ mems_allowed=0
CPU: 2 PID: 9602 Comm: ruby Not tainted 3.13.0-156-generic #206-Ubuntu
Hardware name: Amazon EC2 m5.2xlarge/, BIOS 1.0 10/16/2017
0000000000000000 ffff8800bbb75a60 ffffffff8173654e ffff88018c005e40
0000000000000000 ffff8800bbb75ae8 ffffffff81730abd 0000000000000000
0000000000000015 ffffffff81c43580 ffff8807a90fb0c0 ffffffffffffff6a
Call Trace:
[] dump_stack+0x64/0x80
[] dump_header+0x7f/0x1f1
[] ? cap_capable+0xd/0x80
[] oom_kill_process+0x201/0x360
[] ? security_capable_noaudit+0x1b/0x20
[] out_of_memory+0x471/0x4b0
[] __alloc_pages_nodemask+0xa6c/0xb90
[] alloc_pages_vma+0x9a/0x140
[] handle_mm_fault+0xaaf/0xeb0
[] __do_page_fault+0x183/0x570
[] ? do_mmap_pgoff+0x381/0x400
[] ? set_next_entity+0x95/0xb0
[] ? __switch_to+0x350/0x530
[] do_page_fault+0x1a/0x70
[] do_async_page_fault+0x29/0xe0
[] async_page_fault+0x28/0x30
Mem-Info:
Node 0 DMA per-cpu:
CPU 0: hi: 0, btch: 1 usd: 0
CPU 1: hi: 0, btch: 1 usd: 0
CPU 2: hi: 0, btch: 1 usd: 0
CPU 3: hi: 0, btch: 1 usd: 0
CPU 4: hi: 0, btch: 1 usd: 0
CPU 5: hi: 0, btch: 1 usd: 0
CPU 6: hi: 0, btch: 1 usd: 0
CPU 7: hi: 0, btch: 1 usd: 0
Node 0 DMA32 per-cpu:
CPU 0: hi: 186, btch: 31 usd: 0
CPU 1: hi: 186, btch: 31 usd: 25
CPU 2: hi: 186, btch: 31 usd: 30
CPU 3: hi: 186, btch: 31 usd: 8
CPU 4: hi: 186, btch: 31 usd: 0
CPU 5: hi: 186, btch: 31 usd: 29
CPU 6: hi: 186, btch: 31 usd: 18
CPU 7: hi: 186, btch: 31 usd: 168
Node 0 Normal per-cpu:
CPU 0: hi: 186, btch: 31 usd: 16
CPU 1: hi: 186, btch: 31 usd: 84
CPU 2: hi: 186, btch: 31 usd: 39
CPU 3: hi: 186, btch: 31 usd: 71
CPU 4: hi: 186, btch: 31 usd: 7
CPU 5: hi: 186, btch: 31 usd: 42
CPU 6: hi: 186, btch: 31 usd: 23
CPU 7: hi: 186, btch: 31 usd: 167
active_anon:7856838 inactive_anon:39 isolated_anon:0
active_file:10 inactive_file:0 isolated_file:0
unevictable:3776 dirty:0 writeback:0 unstable:0
free:49193 slab_reclaimable:14064 slab_unreclaimable:13652
mapped:168 shmem:101 pagetables:31755 bounce:0
free_cma:0
Node 0 DMA free:15908kB min:32kB low:40kB high:48kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
lowmem_reserve[]: 0 2976 31358 31358
Node 0 DMA32 free:119824kB min:6408kB low:8008kB high:9612kB active_anon:2897076kB inactive_anon:16kB active_file:84kB inactive_file:0kB unevictable:2700kB isolated(anon):0kB isolated(file):0kB present:3129316kB managed:3050168kB mlocked:2700kB dirty:72kB writeback:0kB mapped:0kB shmem:24kB slab_reclaimable:6396kB slab_unreclaimable:4908kB kernel_stack:600kB pagetables:12412kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:3385 all_unreclaimable? yes
lowmem_reserve[]: 0 0 28382 28382
Node 0 Normal free:61040kB min:61136kB low:76420kB high:91704kB active_anon:28530276kB inactive_anon:140kB active_file:0kB inactive_file:0kB unevictable:12404kB isolated(anon):0kB isolated(file):0kB present:29577216kB managed:29063528kB mlocked:12404kB dirty:0kB writeback:0kB mapped:964kB shmem:380kB slab_reclaimable:49860kB slab_unreclaimable:49700kB kernel_stack:3824kB pagetables:114608kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:32 all_unreclaimable? yes
lowmem_reserve[]: 0 0 0 0
Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15908kB
Node 0 DMA32: 434*4kB (UE) 339*8kB (UEM) 351*16kB (UEM) 288*32kB (UEM) 226*64kB (UEM) 106*128kB (UEM) 59*256kB (UEM) 28*512kB (UE) 4*1024kB (UE) 5*2048kB (UE) 7*4096kB (UEMR) = 119760kB
Node 0 Normal: 8099*4kB (UEM) 2999*8kB (UEM) 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 60484kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
272 total pagecache pages
0 pages in swap cache
Swap cache stats: add 0, delete 0, find 0/0
Free swap = 0kB
Total swap = 0kB
8180631 pages RAM
0 pages HighMem/MovableOnly
128422 pages reserved
[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
[ 478] 0 478 4869 74 13 0 0 upstart-udev-br
[ 482] 0 482 12392 245 27 0 -1000 systemd-udevd
[ 642] 0 642 12164 311 24 0 0 lldpd
[ 663] 0 663 3815 70 12 0 0 upstart-socket-
[ 698] 106 698 12164 157 24 0 0 lldpd
[ 736] 0 736 2555 617 8 0 0 dhclient
[ 986] 102 986 11999 201 26 0 0 dbus-daemon
[ 1026] 0 1026 8756 205 22 0 0 systemd-logind
[ 1033] 0 1033 3819 73 12 0 0 upstart-file-br
[ 1078] 0 1078 3634 164 12 0 0 getty
[ 1081] 0 1081 3634 165 12 0 0 getty
[ 1086] 0 1086 3634 166 12 0 0 getty
[ 1088] 0 1088 3634 166 11 0 0 getty
[ 1089] 0 1089 182070 2279 50 0 0 amazon-ssm-agen
[ 1091] 0 1091 3634 167 12 0 0 getty
[ 1126] 0 1126 1092 109 8 0 0 acpid
[ 1127] 0 1127 5913 191 17 0 0 cron
[ 1129] 0 1129 4785 40 12 0 0 atd
[ 1184] 0 1184 4799 148 15 0 0 irqbalance
[ 1302] 0 1302 2243 68 8 0 0 rngd
[ 1453] 0 1453 8487 156 15 0 0 memlockd
[ 1544] 1 1544 1600 112 9 0 0 uptimed
[ 1548] 0 1548 3634 174 12 0 0 getty
[ 1549] 0 1549 3196 168 12 0 0 getty
[ 7154] 0 7154 46 4 3 0 0 runsvdir
[ 7163] 0 7163 41 1 3 0 0 runsv
[ 7194] 0 7194 1085 127 8 0 0 logger
[ 7195] 0 7195 41 1 3 0 0 runsv
[ 7205] 0 7205 1085 147 8 0 0 logger
[ 9101] 998 9101 9764 1156 24 0 0 python
[ 9136] 109 9136 7478 192 18 0 0 dnsmasq
[17111] 0 17111 3211 77 11 0 0 mcelog
[20922] 0 20922 6289 232 16 0 0 master
[24780] 0 24780 5856 163 16 0 0 rpcbind
[26542] 0 26542 41 1 3 0 0 runsv
[26603] 0 26603 49 1 4 0 0 svlogd
[27249] 0 27249 41 1 3 0 0 runsv
[27398] 0 27398 49 1 4 0 0 svlogd
[29275] 0 29275 41 1 3 0 0 runsv
[29293] 0 29293 1085 79 7 0 0 logger
[ 467] 991 467 8792 1533 15 0 0 redis-server
[ 743] 0 743 41 1 3 0 0 runsv
[ 744] 0 744 49 6 4 0 0 svlogd
[ 1458] 0 1458 41 1 3 0 0 runsv
[ 1459] 0 1459 50 9 4 0 0 svlogd
[ 2423] 993 2423 6897 221 18 0 0 nrpe
[ 2427] 0 2427 19120 3381 34 0 0 ruby
[ 2683] 0 2683 10741 178 25 0 0 syslog-ng
[ 2684] 0 2684 28994 4257 62 0 0 syslog-ng
[ 2692] 0 2692 17384 2771 33 0 0 ruby
[ 2812] 0 2812 7324 353 17 0 0 nginx
[ 8224] 0 8224 41 1 3 0 0 runsv
[ 8269] 0 8269 49 1 4 0 0 svlogd
[12779] 0 12779 41 4 3 0 0 runsv
[12866] 0 12866 1085 146 8 0 0 logger
[12956] 0 12956 41 1 3 0 0 runsv
[13131] 0 13131 1085 146 8 0 0 logger
[13729] 65534 13729 84183 3755 30 0 0 memcached
[13793] 0 13793 4159 799 12 0 0 consul-template
[18314] 0 18314 29462 612 24 0 0 osqueryd
[18316] 0 18316 217702 5867 92 0 0 osqueryd
[21414] 0 21414 15344 319 34 0 -1000 sshd
[23885] 992 23885 167164 1239 39 0 0 raingutter
[23943] 992 23943 56394 22832 115 0 0 bundle
[13766] 0 13766 26264 3488 53 0 0 ruby
[29048] 999 29048 17319 5417 38 0 0 consul
[29867] 113 29867 26431 3745 54 0 0 supervisord
[29891] 113 29891 44805 8700 93 0 0 python
[29892] 113 29892 57087 9213 84 0 0 python
[29896] 113 29896 151802 9573 96 0 0 python
[26311] 33 26311 7846 443 19 0 0 nginx
[ 8234] 0 8234 39672 382 79 0 0 sssd
[ 8238] 0 8238 44386 700 90 0 0 sssd_be
[ 8258] 0 8258 37591 380 75 0 0 sssd_nss
[ 8259] 0 8259 34113 370 69 0 0 sssd_pam
[ 8260] 0 8260 34114 355 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 402 73 0 0 sssd_ssh
[11911] 0 11911 55169 12312 74 0 0 chef-client
[17952] 110 17952 7369 208 19 0 0 qmgr
[14580] 110 14580 10570 302 25 0 0 tlsmgr
[29141] 113 29141 219163 15247 92 0 0 trace-agent
[31674] 0 31674 8844 3727 20 0 0 atop
[23650] 992 23650 192508 115154 382 0 0 ruby
[26952] 992 26952 270105 177168 529 0 0 ruby
[27739] 992 27739 271129 182889 532 0 0 ruby
[28539] 992 28539 276249 179342 542 0 0 ruby
[29063] 992 29063 279325 183793 541 0 0 ruby
[29458] 992 29458 269852 177376 526 0 0 ruby
[29620] 992 29620 273177 181110 536 0 0 ruby
[30048] 992 30048 273177 183802 536 0 0 ruby
[30527] 992 30527 269081 179688 528 0 0 ruby
[31149] 992 31149 280345 182201 550 0 0 ruby
[31346] 992 31346 273177 178462 536 0 0 ruby
[31429] 992 31429 571165 183369 1115 0 0 ruby
[31467] 992 31467 264985 175284 520 0 0 ruby
[31515] 992 31515 269085 173063 522 0 0 ruby
[13409] 992 13409 267033 179100 524 0 0 ruby
[14242] 992 14242 270105 183923 530 0 0 ruby
[15343] 992 15343 270105 178768 530 0 0 ruby
[27547] 992 27547 263961 177998 518 0 0 ruby
[30092] 992 30092 268057 178683 527 0 0 ruby
[30822] 992 30822 572185 184620 1121 0 0 ruby
[ 4480] 992 4480 262937 170750 515 0 0 ruby
[11189] 992 11189 270105 185100 529 0 0 ruby
[14875] 992 14875 269081 179413 527 0 0 ruby
[ 9602] 992 9602 3821337 2247667 5676 0 0 ruby
[24163] 108 24163 7872 267 19 0 0 ntpd
[27139] 992 27139 274205 179465 531 0 0 ruby
[17762] 992 17762 275225 176041 540 0 0 ruby
[21278] 992 21278 256536 167848 506 0 0 ruby
[23913] 0 23913 773482 193327 569 0 0 ruby
[23952] 992 23952 258841 168721 508 0 0 ruby
[25117] 992 25117 246296 161729 486 0 0 ruby
[25738] 992 25738 591641 196059 1157 0 0 ruby
[27011] 0 27011 11715 238 29 0 0 cron
[27023] 0 27023 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27034] 0 27034 2782 155 11 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[27061] 0 27061 1479 126 8 0 0 sleep
[28832] 992 28832 261913 171429 514 0 0 ruby
[ 389] 992 389 263961 175738 518 0 0 ruby
[ 1987] 992 1987 261913 175128 514 0 0 ruby
[ 3425] 992 3425 275225 179419 539 0 0 ruby
[ 6746] 992 6746 263961 174748 518 0 0 ruby
[10691] 992 10691 270105 172603 529 0 0 ruby
[14760] 110 14760 7326 175 17 0 0 showq
[14951] 992 14951 263704 175512 518 0 0 ruby
[17860] 992 17860 258841 171630 510 0 0 ruby
[26329] 992 26329 240409 159233 473 0 0 ruby
[26921] 992 26921 210456 130640 414 0 0 ruby
[28360] 992 28360 226840 148227 448 0 0 ruby
[31937] 0 31937 23656 395 50 0 0 sshd
[32063] 992 32063 23656 314 49 0 0 sshd
[ 1056] 0 1056 11715 238 29 0 0 cron
[ 1059] 0 1059 11715 238 29 0 0 cron
[ 1060] 0 1060 11715 238 29 0 0 cron
[ 1061] 0 1061 11715 238 29 0 0 cron
[ 1062] 0 1062 11715 238 29 0 0 cron
[ 1063] 0 1063 11715 238 29 0 0 cron
[ 1064] 0 1064 11715 238 29 0 0 cron
[ 1071] 0 1071 11715 238 29 0 0 cron
[ 1072] 0 1072 11715 238 29 0 0 cron
[ 1073] 0 1073 11715 238 29 0 0 cron
[ 1074] 0 1074 11715 238 29 0 0 cron
[ 1075] 0 1075 11715 238 29 0 0 cron
[ 1079] 0 1079 2782 156 9 0 0 bash
[ 1083] 0 1083 2782 156 10 0 0 bash
[ 1084] 0 1084 2782 155 10 0 0 bash
[ 1087] 0 1087 2782 155 10 0 0 bash
[ 1090] 0 1090 2782 156 10 0 0 bash
[ 1093] 0 1093 1479 126 8 0 0 sleep
[ 1094] 0 1094 1479 127 8 0 0 sleep
[ 1095] 0 1095 1479 127 8 0 0 sleep
[ 1096] 0 1096 1479 126 8 0 0 sleep
[ 1098] 0 1098 2782 155 9 0 0 bash
[ 1102] 0 1102 1479 126 8 0 0 sleep
[ 1103] 0 1103 1479 127 7 0 0 sleep
[ 1104] 0 1104 2782 156 10 0 0 bash
[ 1105] 0 1105 2782 156 10 0 0 bash
[ 1106] 0 1106 2782 155 10 0 0 bash
[ 1107] 0 1107 2782 154 10 0 0 bash
[ 1109] 0 1109 2782 156 10 0 0 bash
[ 1111] 0 1111 2782 155 9 0 0 bash
[ 1112] 0 1112 1479 127 7 0 0 sleep
[ 1114] 0 1114 1479 126 8 0 0 sleep
[ 1118] 0 1118 1479 127 8 0 0 sleep
[ 1119] 0 1119 1479 127 8 0 0 sleep
[ 1120] 0 1120 1479 126 8 0 0 sleep
[ 1122] 0 1122 1479 126 8 0 0 sleep
[ 1160] 110 1160 7326 204 19 0 0 pickup
[ 1300] 992 1300 41486 390 51 0 0 curl
[ 2268] 113 2268 1088 167 7 0 0 mpstat
[ 2283] 0 2283 17384 2666 28 0 0 ruby
Out of memory: Kill process 9602 (ruby) score 280 or sacrifice child
Killed process 9602 (ruby) total-vm:15285348kB, anon-rss:8990024kB, file-rss:644kB
ruby invoked oom-killer: gfp_mask=0x280da, order=0, oom_score_adj=0
ruby cpuset=/ mems_allowed=0
CPU: 2 PID: 25738 Comm: ruby Not tainted 3.13.0-156-generic #206-Ubuntu
Hardware name: Amazon EC2 m5.2xlarge/, BIOS 1.0 10/16/2017
0000000000000000 ffff880697015a60 ffffffff8173654e ffff880305182f20
0000000000000000 ffff880697015ae8 ffffffff81730abd ffffffff8106d6c6
ffff880697015ac0 ffffffff810cc81c 0000000001320122 0000000000000000
Call Trace:
[] dump_stack+0x64/0x80
[] dump_header+0x7f/0x1f1
[] ? put_online_cpus+0x56/0x80
[] ? rcu_oom_notify+0xcc/0xf0
[] oom_kill_process+0x201/0x360
[] ? security_capable_noaudit+0x1b/0x20
[] out_of_memory+0x471/0x4b0
[] __alloc_pages_nodemask+0xa6c/0xb90
[] alloc_pages_vma+0x9a/0x140
[] handle_mm_fault+0xaaf/0xeb0
[] __do_page_fault+0x183/0x570
[] ? do_mmap_pgoff+0x381/0x400
[] ? vm_mmap_pgoff+0x99/0xc0
[] do_page_fault+0x1a/0x70
[] ? SyS_mmap_pgoff+0x116/0x270
[] do_async_page_fault+0x29/0xe0
[] async_page_fault+0x28/0x30
Mem-Info:
Node 0 DMA per-cpu:
CPU 0: hi: 0, btch: 1 usd: 0
CPU 1: hi: 0, btch: 1 usd: 0
CPU 2: hi: 0, btch: 1 usd: 0
CPU 3: hi: 0, btch: 1 usd: 0
CPU 4: hi: 0, btch: 1 usd: 0
CPU 5: hi: 0, btch: 1 usd: 0
CPU 6: hi: 0, btch: 1 usd: 0
CPU 7: hi: 0, btch: 1 usd: 0
Node 0 DMA32 per-cpu:
CPU 0: hi: 186, btch: 31 usd: 54
CPU 1: hi: 186, btch: 31 usd: 11
CPU 2: hi: 186, btch: 31 usd: 46
CPU 3: hi: 186, btch: 31 usd: 30
CPU 4: hi: 186, btch: 31 usd: 81
CPU 5: hi: 186, btch: 31 usd: 15
CPU 6: hi: 186, btch: 31 usd: 0
CPU 7: hi: 186, btch: 31 usd: 0
Node 0 Normal per-cpu:
CPU 0: hi: 186, btch: 31 usd: 105
CPU 1: hi: 186, btch: 31 usd: 48
CPU 2: hi: 186, btch: 31 usd: 168
CPU 3: hi: 186, btch: 31 usd: 89
CPU 4: hi: 186, btch: 31 usd: 169
CPU 5: hi: 186, btch: 31 usd: 13
CPU 6: hi: 186, btch: 31 usd: 178
CPU 7: hi: 186, btch: 31 usd: 23
active_anon:7857291 inactive_anon:41 isolated_anon:0
active_file:0 inactive_file:13 isolated_file:0
unevictable:1945 dirty:40 writeback:0 unstable:0
free:49204 slab_reclaimable:15406 slab_unreclaimable:13663
mapped:396 shmem:104 pagetables:31372 bounce:0
free_cma:0
Node 0 DMA free:15908kB min:32kB low:40kB high:48kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB isolated(anon):0kB isolated(file):0kB present:15992kB managed:15908kB mlocked:0kB dirty:0kB writeback:0kB mapped:0kB shmem:0kB slab_reclaimable:0kB slab_unreclaimable:0kB kernel_stack:0kB pagetables:0kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:0 all_unreclaimable? yes
lowmem_reserve[]: 0 2976 31358 31358
Node 0 DMA32 free:119792kB min:6408kB low:8008kB high:9612kB active_anon:2896816kB inactive_anon:20kB active_file:0kB inactive_file:112kB unevictable:2232kB isolated(anon):0kB isolated(file):0kB present:3129316kB managed:3050168kB mlocked:2232kB dirty:32kB writeback:0kB mapped:0kB shmem:32kB slab_reclaimable:7132kB slab_unreclaimable:4968kB kernel_stack:568kB pagetables:13972kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:823 all_unreclaimable? yes
lowmem_reserve[]: 0 0 28382 28382
Node 0 Normal free:61116kB min:61136kB low:76420kB high:91704kB active_anon:28532348kB inactive_anon:144kB active_file:0kB inactive_file:0kB unevictable:5548kB isolated(anon):0kB isolated(file):0kB present:29577216kB managed:29063528kB mlocked:5548kB dirty:128kB writeback:0kB mapped:1792kB shmem:384kB slab_reclaimable:54492kB slab_unreclaimable:49684kB kernel_stack:3960kB pagetables:111516kB unstable:0kB bounce:0kB free_cma:0kB writeback_tmp:0kB pages_scanned:32 all_unreclaimable? yes
lowmem_reserve[]: 0 0 0 0
Node 0 DMA: 1*4kB (U) 0*8kB 0*16kB 1*32kB (U) 2*64kB (U) 1*128kB (U) 1*256kB (U) 0*512kB 1*1024kB (U) 1*2048kB (R) 3*4096kB (M) = 15908kB
Node 0 DMA32: 1214*4kB (UE) 493*8kB (UE) 203*16kB (UE) 200*32kB (UE) 238*64kB (UEM) 103*128kB (UEM) 57*256kB (UEM) 31*512kB (UE) 4*1024kB (UE) 5*2048kB (UE) 7*4096kB (UEMR) = 120336kB
Node 0 Normal: 12587*4kB (UEM) 885*8kB (UEM) 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 1*4096kB (R) = 61524kB
Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB
688 total pagecache pages
0 pages in swap cache
Swap cache stats: add 0, delete 0, find 0/0
Free swap = 0kB
Total swap = 0kB
8180631 pages RAM
0 pages HighMem/MovableOnly
128422 pages reserved
[ pid ] uid tgid total_vm rss nr_ptes swapents oom_score_adj name
[ 478] 0 478 4869 74 13 0 0 upstart-udev-br
[ 482] 0 482 12392 245 27 0 -1000 systemd-udevd
[ 642] 0 642 12164 311 24 0 0 lldpd
[ 663] 0 663 3815 70 12 0 0 upstart-socket-
[ 698] 106 698 12164 159 24 0 0 lldpd
[ 736] 0 736 2555 617 8 0 0 dhclient
[ 986] 102 986 11999 201 26 0 0 dbus-daemon
[ 1026] 0 1026 8756 205 22 0 0 systemd-logind
[ 1033] 0 1033 3819 73 12 0 0 upstart-file-br
[ 1078] 0 1078 3634 164 12 0 0 getty
[ 1081] 0 1081 3634 165 12 0 0 getty
[ 1086] 0 1086 3634 166 12 0 0 getty
[ 1088] 0 1088 3634 166 11 0 0 getty
[ 1089] 0 1089 182070 2284 50 0 0 amazon-ssm-agen
[ 1091] 0 1091 3634 167 12 0 0 getty
[ 1126] 0 1126 1092 109 8 0 0 acpid
[ 1127] 0 1127 5913 191 17 0 0 cron
[ 1129] 0 1129 4785 40 12 0 0 atd
[ 1184] 0 1184 4799 148 15 0 0 irqbalance
[ 1302] 0 1302 2243 62 8 0 0 rngd
[ 1453] 0 1453 8487 156 15 0 0 memlockd
[ 1544] 1 1544 1600 112 9 0 0 uptimed
[ 1548] 0 1548 3634 174 12 0 0 getty
[ 1549] 0 1549 3196 168 12 0 0 getty
[ 7154] 0 7154 46 4 3 0 0 runsvdir
[ 7163] 0 7163 41 1 3 0 0 runsv
[ 7194] 0 7194 1085 125 8 0 0 logger
[ 7195] 0 7195 41 1 3 0 0 runsv
[ 7205] 0 7205 1085 147 8 0 0 logger
[ 9101] 998 9101 9764 1156 24 0 0 python
[ 9136] 109 9136 7478 206 18 0 0 dnsmasq
[17111] 0 17111 3211 77 11 0 0 mcelog
[20922] 0 20922 6289 231 16 0 0 master
[24780] 0 24780 5856 155 16 0 0 rpcbind
[26542] 0 26542 41 1 3 0 0 runsv
[26603] 0 26603 49 1 4 0 0 svlogd
[27249] 0 27249 41 1 3 0 0 runsv
[27398] 0 27398 49 1 4 0 0 svlogd
[29275] 0 29275 41 1 3 0 0 runsv
[29293] 0 29293 1085 79 7 0 0 logger
[ 467] 991 467 8792 1505 15 0 0 redis-server
[ 743] 0 743 41 1 3 0 0 runsv
[ 744] 0 744 49 6 4 0 0 svlogd
[ 1458] 0 1458 41 1 3 0 0 runsv
[ 1459] 0 1459 50 9 4 0 0 svlogd
[ 2423] 993 2423 6897 221 18 0 0 nrpe
[ 2427] 0 2427 19120 3393 34 0 0 ruby
[ 2683] 0 2683 10741 178 25 0 0 syslog-ng
[ 2684] 0 2684 28994 4269 62 0 0 syslog-ng
[ 2692] 0 2692 17384 2794 33 0 0 ruby
[ 2812] 0 2812 7324 353 17 0 0 nginx
[ 8224] 0 8224 41 1 3 0 0 runsv
[ 8269] 0 8269 49 1 4 0 0 svlogd
[12779] 0 12779 41 4 3 0 0 runsv
[12866] 0 12866 1085 146 8 0 0 logger
[12956] 0 12956 41 1 3 0 0 runsv
[13131] 0 13131 1085 146 8 0 0 logger
[13729] 65534 13729 84183 3774 30 0 0 memcached
[13793] 0 13793 4159 804 12 0 0 consul-template
[18314] 0 18314 29462 643 24 0 0 osqueryd
[18316] 0 18316 217702 6408 92 0 0 osqueryd
[21414] 0 21414 15344 321 34 0 -1000 sshd
[23885] 992 23885 167164 1297 39 0 0 raingutter
[23943] 992 23943 56394 22822 115 0 0 bundle
[13766] 0 13766 26264 3484 53 0 0 ruby
[29048] 999 29048 17319 5472 38 0 0 consul
[29867] 113 29867 26431 3761 54 0 0 supervisord
[29891] 113 29891 44805 8700 93 0 0 python
[29892] 113 29892 57087 9172 84 0 0 python
[29896] 113 29896 151802 9573 96 0 0 python
[26311] 33 26311 7846 484 19 0 0 nginx
[ 8234] 0 8234 39672 384 79 0 0 sssd
[ 8238] 0 8238 44386 736 90 0 0 sssd_be
[ 8258] 0 8258 37591 355 75 0 0 sssd_nss
[ 8259] 0 8259 34113 371 69 0 0 sssd_pam
[ 8260] 0 8260 34114 384 70 0 0 sssd_sudo
[ 8261] 0 8261 35330 404 73 0 0 sssd_ssh
[11911] 0 11911 55169 12327 74 0 0 chef-client
[17952] 110 17952 7369 199 19 0 0 qmgr
[14580] 110 14580 10570 282 25 0 0 tlsmgr
[29141] 113 29141 219163 14535 92 0 0 trace-agent
[31674] 0 31674 5662 1898 16 0 0 atop
[23650] 992 23650 192508 115198 382 0 0 ruby
[23913] 0 23913 840046 199015 584 0 0 ruby
[25738] 992 25738 7057177 4385314 10835 0 0 ruby
[27011] 0 27011 11715 238 29 0 0 cron
[27031] 0 27031 2782 155 10 0 0 bash
[27054] 0 27054 1479 127 7 0 0 sleep
[14760] 110 14760 7326 208 17 0 0 showq
[31937] 0 31937 23656 379 50 0 0 sshd
[32063] 992 32063 23694 334 49 0 0 sshd
[ 1160] 110 1160 7326 227 19 0 0 pickup
[12469] 108 12469 7872 293 20 0 0 ntpd
[19061] 0 19061 11715 238 29 0 0 cron
[19083] 0 19083 2782 156 10 0 0 bash
[19107] 0 19107 1479 126 8 0 0 sleep
[20221] 992 20221 190460 112701 375 0 0 ruby
[21671] 0 21671 13738 251 32 0 0 sudo
[21676] 0 21676 2788 203 9 0 0 check_services
[21677] 0 21677 16444 2393 32 0 0 ruby
[21704] 992 21704 208408 128438 408 0 0 ruby
[21942] 992 21942 210713 130397 412 0 0 ruby
[21960] 992 21960 209432 130049 411 0 0 ruby
[21989] 992 21989 211480 131695 414 0 0 ruby
[22014] 992 22014 207641 127281 407 0 0 ruby
[22034] 992 22034 209432 129779 410 0 0 ruby
[22056] 992 22056 208408 129273 409 0 0 ruby
[22079] 992 22079 209432 129423 410 0 0 ruby
[22150] 992 22150 202264 123574 398 0 0 ruby
[22167] 992 22167 201240 122644 396 0 0 ruby
[22241] 992 22241 209432 130706 411 0 0 ruby
[22349] 992 22349 211480 132099 415 0 0 ruby
[22404] 992 22404 210456 130789 412 0 0 ruby
[22429] 992 22429 202264 123439 397 0 0 ruby
[22448] 992 22448 210713 129677 412 0 0 ruby
[22497] 992 22497 201240 122570 396 0 0 ruby
[22551] 992 22551 200216 120254 392 0 0 ruby
[22600] 992 22600 209432 129738 410 0 0 ruby
[22654] 992 22654 194072 115076 381 0 0 ruby
[22682] 992 22682 202748 125036 396 0 0 ruby
[22754] 992 22754 209432 129541 410 0 0 ruby
[22795] 992 22795 207384 128861 408 0 0 ruby
[22815] 992 22815 209689 129409 410 0 0 ruby
[22834] 992 22834 205336 126131 403 0 0 ruby
[22851] 992 22851 211994 130982 414 0 0 ruby
[22950] 992 22950 197144 118076 387 0 0 ruby
[23032] 992 23032 208408 128614 408 0 0 ruby
[23087] 992 23087 208408 128185 408 0 0 ruby
[23226] 992 23226 209432 129681 410 0 0 ruby
[23245] 992 23245 205336 126109 403 0 0 ruby
[23294] 992 23294 205336 125373 402 0 0 ruby
[23317] 992 23317 203545 122730 398 0 0 ruby
[23342] 992 23342 208408 128594 408 0 0 ruby
[23367] 992 23367 205820 127194 401 0 0 ruby
[23421] 992 23421 192508 113577 374 0 0 ruby
[23525] 992 23525 209432 129659 410 0 0 ruby
[23573] 992 23573 201240 121574 395 0 0 ruby
[23600] 992 23600 200700 122451 391 0 0 ruby
[23613] 992 23613 205336 126051 403 0 0 ruby
[23636] 992 23636 200216 121064 393 0 0 ruby
[24035] 0 24035 11715 238 29 0 0 cron
[24038] 0 24038 11715 238 29 0 0 cron
[24039] 0 24039 11715 238 29 0 0 cron
[24041] 0 24041 11715 238 29 0 0 cron
[24042] 0 24042 11715 238 29 0 0 cron
[24043] 0 24043 11715 238 29 0 0 cron
[24049] 0 24049 11715 238 29 0 0 cron
[24050] 0 24050 11715 238 29 0 0 cron
[24051] 0 24051 11715 238 29 0 0 cron
[24052] 0 24052 11715 238 29 0 0 cron
[24053] 0 24053 11715 238 29 0 0 cron
[24054] 0 24054 2782 155 11 0 0 bash
[24056] 0 24056 2782 156 11 0 0 bash
[24060] 0 24060 2782 155 11 0 0 bash
[24062] 0 24062 2782 155 10 0 0 bash
[24064] 0 24064 2782 154 11 0 0 bash
[24066] 0 24066 2782 156 9 0 0 bash
[24068] 0 24068 1479 127 8 0 0 sleep
[24069] 0 24069 2782 156 10 0 0 bash
[24070] 0 24070 2782 154 10 0 0 bash
[24071] 0 24071 2782 155 9 0 0 bash
[24073] 0 24073 2782 155 9 0 0 bash
[24074] 0 24074 1479 126 8 0 0 sleep
[24075] 0 24075 1479 127 8 0 0 sleep
[24076] 0 24076 1479 127 8 0 0 sleep
[24077] 0 24077 1479 127 8 0 0 sleep
[24078] 0 24078 2782 156 10 0 0 bash
[24079] 0 24079 1479 127 8 0 0 sleep
[24080] 0 24080 1479 126 7 0 0 sleep
[24085] 0 24085 1479 127 8 0 0 sleep
[24087] 0 24087 1479 127 8 0 0 sleep
[24090] 0 24090 1479 126 8 0 0 sleep
[24092] 0 24092 1479 126 8 0 0 sleep
[25116] 113 25116 1088 169 9 0 0 mpstat
[25160] 999 25160 4492 208 13 0 0 memcached
[25161] 999 25161 4492 79 12 0 0 memcached
[25163] 999 25163 2805 162 11 0 0 nc
Out of memory: Kill process 25738 (ruby) score 547 or sacrifice child
Killed process 25738 (ruby) total-vm:28228708kB, anon-rss:17540680kB, file-rss:576kB