linux resource limits
TRANSCRIPT
Linux Resource Linux Resource ManagementManagement
Marian HackMan MarinovChief System [email protected]
Who am I?Who am I?● Chief System Architect - SiteGroundChief System Architect - SiteGround● Linux System Administrator since 1996Linux System Administrator since 1996● Teaching LSA and NetSec at FMI SofiaTeaching LSA and NetSec at FMI Sofia● Organizing OpenFest and othersOrganizing OpenFest and others
● ulimitulimit● quotaquota● CPU affinity per-device and per-processCPU affinity per-device and per-process● cGroupscGroups
cpu time (seconds, -t) unlimitedcpu time (seconds, -t) unlimited
scheduling priority (-e) 0scheduling priority (-e) 0
real-time priority (-r) 0real-time priority (-r) 0
file size (blocks, -f) unlimitedfile size (blocks, -f) unlimited
pending signals (-i) 96832pending signals (-i) 96832
open files (-n) 1024open files (-n) 1024
file locks (-x) unlimitedfile locks (-x) unlimited
pipe size (512 bytes, -p) 8pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200POSIX message queues (bytes, -q) 819200
max user processes (-u) 200max user processes (-u) 200
max locked memory (kbytes, -l) 64max locked memory (kbytes, -l) 64
max memory size (kbytes, -m) unlimitedmax memory size (kbytes, -m) unlimited
virtual memory (kbytes, -v) unlimitedvirtual memory (kbytes, -v) unlimited
core file size (blocks, -c) 0core file size (blocks, -c) 0
data seg size (kbytes, -d) unlimiteddata seg size (kbytes, -d) unlimited
stack size (kbytes, -s) 8192stack size (kbytes, -s) 8192
ulimitsulimits
app1
userXuserX
user procsuser procsuserX 1userX 1
tty:tty:
core file size (blocks, -c) 0core file size (blocks, -c) 0
data seg size (kbytes, -d) unlimiteddata seg size (kbytes, -d) unlimited
scheduling priority (-e) 0scheduling priority (-e) 0
file size (blocks, -f) unlimitedfile size (blocks, -f) unlimited
pending signals (-i) 96832pending signals (-i) 96832
max locked memory (kbytes, -l) 64max locked memory (kbytes, -l) 64
max memory size (kbytes, -m) unlimitedmax memory size (kbytes, -m) unlimited
open files (-n) 1024open files (-n) 1024
pipe size (512 bytes, -p) 8pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200POSIX message queues (bytes, -q) 819200
real-time priority (-r) 0real-time priority (-r) 0
stack size (kbytes, -s) 8192stack size (kbytes, -s) 8192
cpu time (seconds, -t) unlimitedcpu time (seconds, -t) unlimited
max user processes (-u) 200max user processes (-u) 200
virtual memory (kbytes, -v) unlimitedvirtual memory (kbytes, -v) unlimited
file locks (-x) unlimitedfile locks (-x) unlimited
ulimitsulimits
app2
app1
userXuserX
userXuserX
user procsuser procsuserX 2userX 2
tty:tty:
core file size (blocks, -c) 0core file size (blocks, -c) 0
data seg size (kbytes, -d) unlimiteddata seg size (kbytes, -d) unlimited
scheduling priority (-e) 0scheduling priority (-e) 0
file size (blocks, -f) unlimitedfile size (blocks, -f) unlimited
pending signals (-i) 96832pending signals (-i) 96832
max locked memory (kbytes, -l) 64max locked memory (kbytes, -l) 64
max memory size (kbytes, -m) unlimitedmax memory size (kbytes, -m) unlimited
open files (-n) 1024open files (-n) 1024
pipe size (512 bytes, -p) 8pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200POSIX message queues (bytes, -q) 819200
real-time priority (-r) 0real-time priority (-r) 0
stack size (kbytes, -s) 8192stack size (kbytes, -s) 8192
cpu time (seconds, -t) unlimitedcpu time (seconds, -t) unlimited
max user processes (-u) 200max user processes (-u) 200
virtual memory (kbytes, -v) unlimitedvirtual memory (kbytes, -v) unlimited
file locks (-x) unlimitedfile locks (-x) unlimited
ulimitsulimits
app2
app1
app3
userXuserX
userXuserX
userXuserX
user procsuser procsuserX 3userX 3
tty:tty:
core file size (blocks, -c) 0core file size (blocks, -c) 0
data seg size (kbytes, -d) unlimiteddata seg size (kbytes, -d) unlimited
scheduling priority (-e) 0scheduling priority (-e) 0
file size (blocks, -f) unlimitedfile size (blocks, -f) unlimited
pending signals (-i) 96832pending signals (-i) 96832
max locked memory (kbytes, -l) 64max locked memory (kbytes, -l) 64
max memory size (kbytes, -m) unlimitedmax memory size (kbytes, -m) unlimited
open files (-n) 1024open files (-n) 1024
pipe size (512 bytes, -p) 8pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200POSIX message queues (bytes, -q) 819200
real-time priority (-r) 0real-time priority (-r) 0
stack size (kbytes, -s) 8192stack size (kbytes, -s) 8192
cpu time (seconds, -t) unlimitedcpu time (seconds, -t) unlimited
max user processes (-u) 200max user processes (-u) 200
virtual memory (kbytes, -v) unlimitedvirtual memory (kbytes, -v) unlimited
file locks (-x) unlimitedfile locks (-x) unlimited
ulimitsulimits
core file size (blocks, -c) 0core file size (blocks, -c) 0
data seg size (kbytes, -d) unlimiteddata seg size (kbytes, -d) unlimited
scheduling priority (-e) 0scheduling priority (-e) 0
file size (blocks, -f) unlimitedfile size (blocks, -f) unlimited
pending signals (-i) 96832pending signals (-i) 96832
max locked memory (kbytes, -l) 64max locked memory (kbytes, -l) 64
max memory size (kbytes, -m) unlimitedmax memory size (kbytes, -m) unlimited
open files (-n) 1024open files (-n) 1024
pipe size (512 bytes, -p) 8pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200POSIX message queues (bytes, -q) 819200
real-time priority (-r) 0real-time priority (-r) 0
stack size (kbytes, -s) 8192stack size (kbytes, -s) 8192
cpu time (seconds, -t) unlimitedcpu time (seconds, -t) unlimited
max user processes (-u) 200max user processes (-u) 200
virtual memory (kbytes, -v) unlimitedvirtual memory (kbytes, -v) unlimited
file locks (-x) unlimitedfile locks (-x) unlimitedapp2
app1
app3
userXuserX
userXuserX
userXuserX
user procsuser procsuserX 4userX 4
app4
userXuserXssh:ssh:
tty:tty:
ulimitsulimits
● login (on tty, via PAM)● KDM, GDM, XDM & etc. (locally via PAM)● ssh (remotely, via PAM and shell)
● pam_limits– /etc/security/limits.conf
– /etc/security/limits.d/
● shell (sh, bash, zsh, csh, tcsh)– /etc/profile.d/limits.[tcz]sh
ulimitsulimits how-tohow-to
$ cat /proc/self/limits
Limit Soft Limit Hard Limit Units
Max cpu time unlimited unlimited seconds
Max file size unlimited unlimited bytes
Max data size unlimited unlimited bytes
Max stack size 8388608 unlimited bytes
Max core file size 0 unlimited bytes
Max resident set unlimited unlimited bytes
Max processes 200 200 processes
Max open files 1024 4096 files
Max locked memory 65536 65536 bytes
Max address space unlimited unlimited bytes
Max file locks unlimited unlimited locks
Max pending signals 200 200 signals
Max msgqueue size 819200 819200 bytes
Max nice priority 0 0
Max realtime priority 0 0
Max realtime timeout unlimited unlimited us
ulimitsulimits how-tohow-to
$ cat /proc/self/limits
on older kernels:
$ echo -n "Max open files=2000:6000" > /proc/self/limits
$ prlimit
ulimitsulimits how-tohow-to
Other kernel limits
● fs.file-max - max fd for the machine● fs.nr_open - max fd per process● fs.mount-max - max mounted filesystems● kernel.threads-max
● Dedicate a CPU to HW device● Dedicate a CPU to a process
● taskset mask cmd● /proc/interrupts
– /proc/irq/NUM/smp_affinity
– /proc/irq/NUM/smp_affinity_list
– /proc/irq/NUM/affinity_hint
CPU AffinityCPU Affinity
● Dedicate a CPU to HW device● Dedicate a CPU to a process
core0 core1
core2 core3
eth0 1Gbpseth4 10Gbpsmegaraid 6Gbps
CPU AffinityCPU Affinity
● Dedicate a CPU to HW device● Dedicate a CPU to a process
core0 core1
core2 core3
eth0 1Gbpseth1 10Gbpseth2 10Gbpsmegaraid 6Gbps
core0 - eth1 10Gbpscore1 - eth2 10Gbpscore3 - megaraid 6Gbpscore4 - eth0 & processes
CPU AffinityCPU Affinity
taskset example
root@terion:~# taskset -p 2727
pid 2727's current affinity mask: ff
root@terion:~# taskset -pc 3 2727
pid 2727's current affinity list: 0-7
pid 2727's new affinity list: 3
root@terion:~# taskset -p 2727
pid 2727's current affinity mask: 8
root@terion:~# ps axf|grep 2727
2727 ? Ss 2:06 /usr/sbin/acpid
root@terion:~#
irq affinity example
root@terion:~# cat /proc/interrupts
CPU0 CPU1
16: 3567385 0 IO-APIC 16-fasteoi ehci_hcd:usb1
17: 4567 0 IO-APIC 17-fasteoi snd_hda_intel:
23: 50797 0 IO-APIC 23-fasteoi ehci_hcd:usb2
25: 78045696 0 PCI-MSI 512000-edge ahci
36: 12 0 PCI-MSI 409600-edge eth0
37: 169256226 0 PCI-MSI 1572864-edge iwlwifi
38: 3515939 0 PCI-MSI 524288-edge nvidia
irq affinity example
root@terion:~# cd /proc/irq/37
root@terion:/proc/irq/37# cat smp_affinity
ff
root@terion:/proc/irq/37# cat smp_affinity_list
0-7
root@terion:/proc/irq/37# echo 3 > smp_affinity_list
root@terion:/proc/irq/37# cat smp_affinity
08
root@terion:/proc/irq/37# cat smp_affinity_list
3
root@terion:/proc/irq/37#
Other resource limitations can be enforced using virtualization
technologies like KVM, Xen, etc.
What if you want to set a limit to a group of processes?
● CPUSET● CPU● CPUACCT● MEMORY● BLKIO● DEVICES
● freezer● net_cls● net_prio● perf_event● hudgetlb
cGroupscGroups
cGroupscGroups
● freezer● net_cls● net_prio● perf_event● hudgetlb
● CPUSET● CPU● CPUACCT● MEMORY● BLKIO● DEVICES
● cGroups have hierarchy
//
/user1/user1
/user2/user2
/user1/user3/user1/user3
cGroupscGroups
root@goblin:/cgroup# ls -1 cpuset*
cpuset.cpus
cpuset.mems
cpuset.cpu_exclusive
cpuset.mem_exclusive
cpuset.effective_cpus
cpuset.effective_mems
...
cGroupscGroups CPUSET CPUSET
root@goblin:/cgroup# ls -1 cpu.*
cpu.cfs_period_us cpu.cfs_quota_us cpu.rt_period_us cpu.rt_runtime_us cpu.shares cpu.stat
cGroupscGroups CPUCPU
root@goblin:/cgroup# ls -1 cpuacct.*
cpuacct.stat
cpuacct.usage
cpuacct.usage_percpu
cpuacct.usage_all
cpuacct.usage_percpu_sys
cpuacct.usage_percpu_user
cpuacct.usage_sys
cpuacct.usage_user
CPUACCTCPUACCTcGroupscGroups
memory.memsw.failcnt
memory.memsw.limit_in_bytes
memory.memsw.max_usage_in_bytes
memory.memsw.usage_in_bytes
memory.limit_in_bytes memory.usage_in_bytes
memory.soft_limit_in_bytes
memory.max_usage_in_bytes
memory.move_charge_at_immigrate memory.failcnt
memory.numa_stat memory.stat
memory.oom_control memory.pressure_level
memory.swappiness memory.use_hierarchy
cGroups cGroups MEMORYMEMORY
blkio.throttle.io_service_bytes
blkio.throttle.io_serviced
blkio.throttle.read_bps_device
blkio.throttle.read_iops_device
blkio.throttle.write_bps_device
blkio.throttle.write_iops_device
cGroupscGroups BLKIOBLKIO
blkio.weight
blkio.weight_device
blkio.leaf_weight
blkio.leaf_weight_device
BLKIOBLKIO cGroupscGroups
cGroupscGroups
root@goblin:/cgroup# ls -1 devices.*
devices.allow
devices.deny
devices.list
DEVICESDEVICES
Marian HackMan MarinovChief System [email protected]
QuestionsQuestions