Доброго времени суток уважаемые форумчане. Прошу помощи у Вас, так как уже долгое время ломаю голову над одной проблемой - нехваткой mbuf. Оговорюсь заранее, что во FeeBSD новичок, поэтому сильно не пинайте, а по возможности направьте на правильное решение.
Сервер - все в одном, 1500 Мбит трафика, 2200 абонентов всего, ну примерно те-же 2000 онлайна в час пик. Что касается работы инета в целом, то видимых проблем не наблюдается, т.е. скорость и пинг у абонов более менее. Бывают падения когда нагрузка переваливает за 1.5 гигабита, но они кратковременные. Тарифы у нас в сети: 25, 50 и 100 Мбит/с.
Инфо по железу:
Процессор: Core i7-3770K, 3500 MHz
Мат. Плата: Asus Maximus V Gene
Память: Kingston 2x4 8GB
Сетевой адаптер: 4-х портовый серверный адаптер Intel E1G44ET2
[root@nodeny ~]# uname -a
FreeBSD nodeny 10.3-RELEASE FreeBSD 10.3-RELEASE #0: Sun Apr 23 09:47:58 MSK 2017 admin_nodeny@nodeny:/usr/obj/usr/src/sys/GENERIC amd64
[root@nodeny ~]# uptime
5:37AM up 3 days, 22:51, 2 users, load averages: 1.06, 1.09, 1.00
[root@nodeny ~]# netstat -m
261824/7126/268950 mbufs in use (current/cache/total)
261817/3593/265410/524288 mbuf clusters in use (current/cache/total/max)
261817/3580 mbuf+clusters out of packet secondary zone in use (current/cache)
2/162/164/252584 4k (page size) jumbo clusters in use (current/cache/total/max)
0/0/0/74839 9k jumbo clusters in use (current/cache/total/max)
0/0/0/42097 16k jumbo clusters in use (current/cache/total/max)
589103K/9615K/598719K bytes allocated to network (current/cache/total)
0/0/0 requests for mbufs denied (mbufs/clusters/mbuf+clusters)
0/0/0 requests for mbufs delayed (mbufs/clusters/mbuf+clusters)
0/0/0 requests for jumbo clusters delayed (4k/9k/16k)
0/0/0 requests for jumbo clusters denied (4k/9k/16k)
0 requests for sfbufs denied
0 requests for sfbufs delayed
0 requests for I/O initiated by sendfile
[root@nodeny ~]# cat /etc/rc.conf
hostname="nodeny"
cloned_interfaces="lagg0 lagg1 vlan2001 vlan2002 vlan2003 vlan2004 vlan2005 vlan2006 vlan2007 vlan2008 vlan2009 vlan2010 vlan2011 vlan2012 vlan2013 vlan2014 vlan2015 vlan2016 vlan2017 vlan2018 vlan2019 vlan2020 vlan2021 vlan2022 vlan2023 vlan2024 vlan2025 vlan2026 vlan2027 vlan2028 vlan2029 vlan2030 vlan2031 vlan2032 vlan2033 vlan2034 vlan2035 vlan2036 vlan2037 vlan2038 vlan2039 vlan2040 vlan2041 vlan2042 vlan2043 vlan2044 vlan2045 vlan2046 vlan2047 vlan2048 vlan3001 vlan3002 vlan3003 vlan3004 vlan3005 vlan3006 vlan3007 vlan3008 vlan3009 vlan3010 vlan3011 vlan3012 vlan3013 vlan3014 vlan3015 vlan3016 vlan3017 vlan3018 vlan3019 vlan3020 vlan3021 vlan3022 vlan3023 vlan3024 vlan3025 vlan3026 vlan3027 vlan3028 vlan3029 vlan3030 vlan3031 vlan3032 vlan3033 vlan3034 vlan3035 vlan3036 vlan3037 vlan3038 vlan3039 vlan3040 vlan3041 vlan3042 vlan3043 vlan3044 vlan3045 vlan3046 vlan3047 vlan3048"
ifconfig_lo0_alias0="inet 1.1.1.1 netmask 255.255.255.255"
ifconfig_igb0="-rxcsum -txcsum -lro -tso up"
ifconfig_igb1="-rxcsum -txcsum -lro -tso up"
ifconfig_igb2="-rxcsum -txcsum -lro -tso up"
ifconfig_igb3="-rxcsum -txcsum -lro -tso up"
ifconfig_lagg1="laggproto lacp laggport igb0 laggport igb1 91.ххх.ххх.ххх netmask 255.255.255.240"
ifconfig_lagg1_alias0="inet 91.ххх.ххх.ххх netmask 255.255.255.255"
ifconfig_lagg1_alias1="inet 91.ххх.ххх.ххх netmask 255.255.255.255"
ifconfig_lagg1_alias2="inet 91.ххх.ххх.ххх netmask 255.255.255.255"
ifconfig_lagg1_alias3="inet 91.ххх.ххх.ххх netmask 255.255.255.255"
ifconfig_lagg1_alias4="inet 91.ххх.ххх.ххх netmask 255.255.255.255"
ifconfig_lagg0="laggproto lacp laggport igb2 laggport igb3 10.0.0.1 netmask 255.255.255.252"
ifconfig_vlan2001="vlan 2001 vlandev lagg0 172.18.7.1 netmask 255.255.255.0"
ifconfig_vlan2002="vlan 2002 vlandev lagg0 172.18.8.1 netmask 255.255.255.0"
ifconfig_vlan2003="vlan 2003 vlandev lagg0 172.18.9.1 netmask 255.255.255.0"
ifconfig_vlan2004="vlan 2004 vlandev lagg0 172.18.10.1 netmask 255.255.255.0"
ifconfig_vlan2005="vlan 2005 vlandev lagg0 172.18.11.1 netmask 255.255.255.0"
ifconfig_vlan2006="vlan 2006 vlandev lagg0 172.18.12.1 netmask 255.255.255.0"
ifconfig_vlan2007="vlan 2007 vlandev lagg0 172.18.13.1 netmask 255.255.255.0"
ifconfig_vlan2008="vlan 2008 vlandev lagg0 172.18.14.1 netmask 255.255.255.0"
ifconfig_vlan2009="vlan 2009 vlandev lagg0 172.18.15.1 netmask 255.255.255.0"
ifconfig_vlan2010="vlan 2010 vlandev lagg0 172.18.16.1 netmask 255.255.255.0"
ifconfig_vlan2011="vlan 2011 vlandev lagg0 172.18.17.1 netmask 255.255.255.0"
ifconfig_vlan2012="vlan 2012 vlandev lagg0 172.18.18.1 netmask 255.255.255.0"
ifconfig_vlan2013="vlan 2013 vlandev lagg0 172.18.19.1 netmask 255.255.255.0"
ifconfig_vlan2014="vlan 2014 vlandev lagg0 172.18.20.1 netmask 255.255.255.0"
ifconfig_vlan2015="vlan 2015 vlandev lagg0 172.18.21.1 netmask 255.255.255.0"
ifconfig_vlan2016="vlan 2016 vlandev lagg0 172.18.22.1 netmask 255.255.255.0"
ifconfig_vlan2017="vlan 2017 vlandev lagg0 172.18.23.1 netmask 255.255.255.0"
ifconfig_vlan2018="vlan 2018 vlandev lagg0 172.18.24.1 netmask 255.255.255.0"
ifconfig_vlan2019="vlan 2019 vlandev lagg0 172.18.25.1 netmask 255.255.255.0"
ifconfig_vlan2020="vlan 2020 vlandev lagg0 172.18.26.1 netmask 255.255.255.0"
ifconfig_vlan2021="vlan 2021 vlandev lagg0 172.18.27.1 netmask 255.255.255.0"
ifconfig_vlan2022="vlan 2022 vlandev lagg0 172.18.28.1 netmask 255.255.255.0"
ifconfig_vlan2023="vlan 2023 vlandev lagg0 172.18.29.1 netmask 255.255.255.0"
ifconfig_vlan2024="vlan 2024 vlandev lagg0 172.18.30.1 netmask 255.255.255.0"
ifconfig_vlan2025="vlan 2025 vlandev lagg0 172.18.31.1 netmask 255.255.255.0"
ifconfig_vlan2026="vlan 2026 vlandev lagg0 172.18.32.1 netmask 255.255.255.0"
ifconfig_vlan2027="vlan 2027 vlandev lagg0 172.18.33.1 netmask 255.255.255.0"
ifconfig_vlan2028="vlan 2028 vlandev lagg0 172.18.34.1 netmask 255.255.255.0"
ifconfig_vlan2029="vlan 2029 vlandev lagg0 172.18.35.1 netmask 255.255.255.0"
ifconfig_vlan2030="vlan 2030 vlandev lagg0 172.18.36.1 netmask 255.255.255.0"
ifconfig_vlan2031="vlan 2031 vlandev lagg0 172.18.37.1 netmask 255.255.255.0"
ifconfig_vlan2032="vlan 2032 vlandev lagg0 172.18.38.1 netmask 255.255.255.0"
ifconfig_vlan2033="vlan 2033 vlandev lagg0 172.18.39.1 netmask 255.255.255.0"
ifconfig_vlan2034="vlan 2034 vlandev lagg0 172.18.40.1 netmask 255.255.255.0"
ifconfig_vlan2035="vlan 2035 vlandev lagg0 172.18.41.1 netmask 255.255.255.0"
ifconfig_vlan2036="vlan 2036 vlandev lagg0 172.18.42.1 netmask 255.255.255.0"
ifconfig_vlan2037="vlan 2037 vlandev lagg0 172.18.43.1 netmask 255.255.255.0"
ifconfig_vlan2038="vlan 2038 vlandev lagg0 172.18.44.1 netmask 255.255.255.0"
ifconfig_vlan2039="vlan 2039 vlandev lagg0 172.18.45.1 netmask 255.255.255.0"
ifconfig_vlan2040="vlan 2040 vlandev lagg0 172.18.46.1 netmask 255.255.255.0"
ifconfig_vlan2041="vlan 2041 vlandev lagg0 172.18.47.1 netmask 255.255.255.0"
ifconfig_vlan2042="vlan 2042 vlandev lagg0 172.18.48.1 netmask 255.255.255.0"
ifconfig_vlan2043="vlan 2043 vlandev lagg0 172.18.49.1 netmask 255.255.255.0"
ifconfig_vlan2044="vlan 2044 vlandev lagg0 172.18.50.1 netmask 255.255.255.0"
ifconfig_vlan2045="vlan 2045 vlandev lagg0 172.18.51.1 netmask 255.255.255.0"
ifconfig_vlan2046="vlan 2046 vlandev lagg0 172.18.52.1 netmask 255.255.255.0"
ifconfig_vlan2047="vlan 2047 vlandev lagg0 172.18.53.1 netmask 255.255.255.0"
ifconfig_vlan2048="vlan 2048 vlandev lagg0 172.18.54.1 netmask 255.255.255.0"
ifconfig_vlan3001="vlan 3001 vlandev lagg0 172.18.55.1 netmask 255.255.255.0"
ifconfig_vlan3002="vlan 3002 vlandev lagg0 172.18.56.1 netmask 255.255.255.0"
ifconfig_vlan3003="vlan 3003 vlandev lagg0 172.18.57.1 netmask 255.255.255.0"
ifconfig_vlan3004="vlan 3004 vlandev lagg0 172.18.58.1 netmask 255.255.255.0"
ifconfig_vlan3005="vlan 3005 vlandev lagg0 172.18.59.1 netmask 255.255.255.0"
ifconfig_vlan3006="vlan 3006 vlandev lagg0 172.18.60.1 netmask 255.255.255.0"
ifconfig_vlan3007="vlan 3007 vlandev lagg0 172.18.61.1 netmask 255.255.255.0"
ifconfig_vlan3008="vlan 3008 vlandev lagg0 172.18.62.1 netmask 255.255.255.0"
ifconfig_vlan3009="vlan 3009 vlandev lagg0 172.18.63.1 netmask 255.255.255.0"
ifconfig_vlan3010="vlan 3010 vlandev lagg0 172.18.64.1 netmask 255.255.255.0"
ifconfig_vlan3011="vlan 3011 vlandev lagg0 172.18.65.1 netmask 255.255.255.0"
ifconfig_vlan3012="vlan 3012 vlandev lagg0 172.18.66.1 netmask 255.255.255.0"
ifconfig_vlan3013="vlan 3013 vlandev lagg0 172.18.67.1 netmask 255.255.255.0"
ifconfig_vlan3014="vlan 3014 vlandev lagg0 172.18.68.1 netmask 255.255.255.0"
ifconfig_vlan3015="vlan 3015 vlandev lagg0 172.18.69.1 netmask 255.255.255.0"
ifconfig_vlan3016="vlan 3016 vlandev lagg0 172.18.70.1 netmask 255.255.255.0"
ifconfig_vlan3017="vlan 3017 vlandev lagg0 172.18.71.1 netmask 255.255.255.0"
ifconfig_vlan3018="vlan 3018 vlandev lagg0 172.18.72.1 netmask 255.255.255.0"
ifconfig_vlan3019="vlan 3019 vlandev lagg0 172.18.73.1 netmask 255.255.255.0"
ifconfig_vlan3020="vlan 3020 vlandev lagg0 172.18.74.1 netmask 255.255.255.0"
ifconfig_vlan3021="vlan 3021 vlandev lagg0 172.18.75.1 netmask 255.255.255.0"
ifconfig_vlan3022="vlan 3022 vlandev lagg0 172.18.76.1 netmask 255.255.255.0"
ifconfig_vlan3023="vlan 3023 vlandev lagg0 172.18.77.1 netmask 255.255.255.0"
ifconfig_vlan3024="vlan 3024 vlandev lagg0 172.18.78.1 netmask 255.255.255.0"
ifconfig_vlan3025="vlan 3025 vlandev lagg0 172.18.79.1 netmask 255.255.255.0"
ifconfig_vlan3026="vlan 3026 vlandev lagg0 172.18.80.1 netmask 255.255.255.0"
ifconfig_vlan3027="vlan 3027 vlandev lagg0 172.18.81.1 netmask 255.255.255.0"
ifconfig_vlan3028="vlan 3028 vlandev lagg0 172.18.82.1 netmask 255.255.255.0"
ifconfig_vlan3029="vlan 3029 vlandev lagg0 172.18.83.1 netmask 255.255.255.0"
ifconfig_vlan3030="vlan 3030 vlandev lagg0 172.18.84.1 netmask 255.255.255.0"
ifconfig_vlan3031="vlan 3031 vlandev lagg0 172.18.85.1 netmask 255.255.255.0"
ifconfig_vlan3032="vlan 3032 vlandev lagg0 172.18.86.1 netmask 255.255.255.0"
ifconfig_vlan3033="vlan 3033 vlandev lagg0 172.18.87.1 netmask 255.255.255.0"
ifconfig_vlan3034="vlan 3034 vlandev lagg0 172.18.88.1 netmask 255.255.255.0"
ifconfig_vlan3035="vlan 3035 vlandev lagg0 172.18.89.1 netmask 255.255.255.0"
ifconfig_vlan3036="vlan 3036 vlandev lagg0 172.18.90.1 netmask 255.255.255.0"
ifconfig_vlan3037="vlan 3037 vlandev lagg0 172.18.91.1 netmask 255.255.255.0"
ifconfig_vlan3038="vlan 3038 vlandev lagg0 172.18.92.1 netmask 255.255.255.0"
ifconfig_vlan3039="vlan 3039 vlandev lagg0 172.18.93.1 netmask 255.255.255.0"
ifconfig_vlan3040="vlan 3040 vlandev lagg0 172.18.94.1 netmask 255.255.255.0"
ifconfig_vlan3041="vlan 3041 vlandev lagg0 172.18.95.1 netmask 255.255.255.0"
ifconfig_vlan3042="vlan 3042 vlandev lagg0 172.18.96.1 netmask 255.255.255.0"
ifconfig_vlan3043="vlan 3043 vlandev lagg0 172.18.97.1 netmask 255.255.255.0"
ifconfig_vlan3044="vlan 3044 vlandev lagg0 172.18.98.1 netmask 255.255.255.0"
ifconfig_vlan3045="vlan 3045 vlandev lagg0 172.18.99.1 netmask 255.255.255.0"
ifconfig_vlan3046="vlan 3046 vlandev lagg0 172.18.100.1 netmask 255.255.255.0"
ifconfig_vlan3047="vlan 3047 vlandev lagg0 172.18.101.1 netmask 255.255.255.0"
ifconfig_vlan3048="vlan 3048 vlandev lagg0 172.18.102.1 netmask 255.255.255.0"
defaultrouter="91.ххх.ххх.ххх"
sshd_enable="YES"
# Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable
dumpdev="AUTO"
firewall_enable="YES"
atop_enable="YES"
fsck_y_enable="YES"
background_fsck="NO"
sendmail_enable="NO"
sendmail_submit_enable="NO"
sendmail_outbound_enable="NO"
sendmail_msp_queue_enable="NO"
gateway_enable="YES"
local_unbound_enable="YES"
pf_enable="YES"
pf_rules="/etc/pf.conf"
mysql_enable="YES"
apache24_enable="YES"
ipcad_enable="YES"
dhcpd_enable="YES"
dhcpd_flags="-q"
dhcpd_withuser=root
static_routes="vlan"
route_vlan="-net 172.16.0.0/16 10.0.0.2"
[root@nodeny ~]# cat /etc/pf.conf
set limit states 16000000
set optimization aggressive
set limit frags 1000000
set limit src-nodes 200000
set limit table-entries 1000000
ext_if = "lagg1"
ext_gw = "91.ххх.ххх.ххх"
set skip on lo0
scrub in all
nat pass on $ext_if from 172.16.0.0/16 to any -> 91.ххх.ххх.ххх
nat pass on $ext_if from 172.18.0.0/20 to any -> 91.ххх.ххх.ххх
nat pass on $ext_if from 172.18.16.0/20 to any -> 91.ххх.ххх.ххх
nat pass on $ext_if from 172.18.32.0/20 to any -> 91.ххх.ххх.ххх
nat pass on $ext_if from 172.18.48.0/20 to any -> 91.ххх.ххх.ххх
nat pass on $ext_if from 172.18.64.0/20 to any -> 91.ххх.ххх.ххх
nat pass on $ext_if from 172.18.80.0/20 to any -> 91.ххх.ххх.ххх
nat pass on $ext_if from 172.18.96.0/20 to any -> 91.ххх.ххх.ххх
pass in quick on lo0 proto tcp from any to any port 80 keep state (source-track rule, max-src-states 2, max-src-nodes 100)
[root@nodeny ~]# cat /etc/sysctl.conf
#net.link.ether.inet.log_arp_wrong_iface=0
#dev.igb.0.rx_processing_limit=4096
#dev.igb.1.rx_processing_limit=4096
#dev.igb.2.rx_processing_limit=4096
#dev.igb.3.rx_processing_limit=4096
dev.igb.0.enable_aim=0
dev.igb.1.enable_aim=0
dev.igb.2.enable_aim=0
dev.igb.3.enable_aim=0
net.inet.ip.forwarding=1
net.inet.ip.fastforwarding=1
net.inet.ip.dummynet.io_fast=1
net.inet.ip.dummynet.hash_size=65535
net.inet.ip.dummynet.pipe_slot_limit=2048
net.inet.ip.fw.one_pass=1
net.inet.ip.maxfragpackets=1024
net.inet.tcp.blackhole=2
net.inet.udp.blackhole=1
net.inet.icmp.drop_redirect=1
net.inet.icmp.log_redirect=0
net.inet.icmp.maskrepl=0
net.inet.icmp.icmplim=200
net.inet.icmp.bmcastecho=0
net.inet.ip.redirect=0
net.inet.tcp.drop_synfin=1
net.inet.tcp.syncookies=1
net.inet.ip.ttl=226
net.inet.ip.intr_queue_maxlen=8192
kern.ipc.somaxconn=32768
kern.maxfiles=204800
kern.maxfilesperproc=200000
kern.ipc.nmbclusters=524288
kern.ipc.maxsockbuf=83886080
kern.random.sys.harvest.ethernet=0
kern.random.sys.harvest.interrupt=0
net.inet.ip.intr_queue_maxlen=10240
net.inet.ip.sourceroute=0
net.inet.tcp.maxtcptw=40960
net.inet.tcp.nolocaltimewait=1
net.inet.tcp.fast_finwait2_recycle=1
net.inet.tcp.finwait2_timeout=3000
net.inet.tcp.hostcache.expire=1200
net.inet.ip.fw.dyn_max=24576
net.inet.tcp.sendspace=131072
net.inet.tcp.recvspace=65536
net.inet.udp.recvspace=32768
net.inet.ip.maxfragpackets=1024
net.inet.tcp.sendbuf_auto=0
net.inet.tcp.recvbuf_auto=0
net.inet.ip.portrange.first=1024
net.inet.ip.portrange.last=65535
net.inet.ip.portrange.randomized=0
Здесь беспорядок, так как пробовал добавлять-удалять-редактировать опции.
С loader-а правила убраны.
Грешил на то, что много пользователей, мало трафика но даже при минимальной нагрузке счетчики mbuf продолжают расти. Имеется также резервный сервер, там железо по мощнее Двухпортовый серверный адаптер Intel(R) Gigabit ET Установлены 2 штуки, памяти 16 гиг. и такая же проблема.
