linux resource limits

Linux Resource Linux Resource ManagementManagement

Marian HackMan MarinovChief System Architectmm@siteground.com

Who am I?Who am I?● Chief System Architect - SiteGroundChief System Architect - SiteGround● Linux System Administrator since 1996Linux System Administrator since 1996● Teaching LSA and NetSec at FMI SofiaTeaching LSA and NetSec at FMI Sofia● Organizing OpenFest and othersOrganizing OpenFest and others

● ulimitulimit● quotaquota● CPU affinity per-device and per-processCPU affinity per-device and per-process● cGroupscGroups

cpu time (seconds, -t) unlimitedcpu time (seconds, -t) unlimited

scheduling priority (-e) 0scheduling priority (-e) 0

real-time priority (-r) 0real-time priority (-r) 0

file size (blocks, -f) unlimitedfile size (blocks, -f) unlimited

pending signals (-i) 96832pending signals (-i) 96832

open files (-n) 1024open files (-n) 1024

file locks (-x) unlimitedfile locks (-x) unlimited

pipe size (512 bytes, -p) 8pipe size (512 bytes, -p) 8

POSIX message queues (bytes, -q) 819200POSIX message queues (bytes, -q) 819200

max user processes (-u) 200max user processes (-u) 200

max locked memory (kbytes, -l) 64max locked memory (kbytes, -l) 64

max memory size (kbytes, -m) unlimitedmax memory size (kbytes, -m) unlimited

virtual memory (kbytes, -v) unlimitedvirtual memory (kbytes, -v) unlimited

core file size (blocks, -c) 0core file size (blocks, -c) 0

data seg size (kbytes, -d) unlimiteddata seg size (kbytes, -d) unlimited

stack size (kbytes, -s) 8192stack size (kbytes, -s) 8192

ulimitsulimits

app1

userXuserX

  user  procsuser  procsuserX    1userX    1

tty:tty:

core file size (blocks, -c) 0core file size (blocks, -c) 0

data seg size (kbytes, -d) unlimiteddata seg size (kbytes, -d) unlimited

scheduling priority (-e) 0scheduling priority (-e) 0

file size (blocks, -f) unlimitedfile size (blocks, -f) unlimited

pending signals (-i) 96832pending signals (-i) 96832

max locked memory (kbytes, -l) 64max locked memory (kbytes, -l) 64

max memory size (kbytes, -m) unlimitedmax memory size (kbytes, -m) unlimited

open files (-n) 1024open files (-n) 1024

pipe size (512 bytes, -p) 8pipe size (512 bytes, -p) 8

POSIX message queues (bytes, -q) 819200POSIX message queues (bytes, -q) 819200

real-time priority (-r) 0real-time priority (-r) 0

stack size (kbytes, -s) 8192stack size (kbytes, -s) 8192

cpu time (seconds, -t) unlimitedcpu time (seconds, -t) unlimited

max user processes (-u) 200max user processes (-u) 200

virtual memory (kbytes, -v) unlimitedvirtual memory (kbytes, -v) unlimited

file locks (-x) unlimitedfile locks (-x) unlimited

ulimitsulimits

app2

app1

userXuserX

userXuserX

  user  procsuser  procsuserX    2userX    2

tty:tty:

core file size (blocks, -c) 0core file size (blocks, -c) 0

data seg size (kbytes, -d) unlimiteddata seg size (kbytes, -d) unlimited

scheduling priority (-e) 0scheduling priority (-e) 0

file size (blocks, -f) unlimitedfile size (blocks, -f) unlimited

pending signals (-i) 96832pending signals (-i) 96832

max locked memory (kbytes, -l) 64max locked memory (kbytes, -l) 64

max memory size (kbytes, -m) unlimitedmax memory size (kbytes, -m) unlimited

open files (-n) 1024open files (-n) 1024

pipe size (512 bytes, -p) 8pipe size (512 bytes, -p) 8

POSIX message queues (bytes, -q) 819200POSIX message queues (bytes, -q) 819200

real-time priority (-r) 0real-time priority (-r) 0

stack size (kbytes, -s) 8192stack size (kbytes, -s) 8192

cpu time (seconds, -t) unlimitedcpu time (seconds, -t) unlimited

max user processes (-u) 200max user processes (-u) 200

virtual memory (kbytes, -v) unlimitedvirtual memory (kbytes, -v) unlimited

file locks (-x) unlimitedfile locks (-x) unlimited

ulimitsulimits

app2

app1

app3

userXuserX

userXuserX

userXuserX

  user  procsuser  procsuserX    3userX    3

tty:tty:

core file size (blocks, -c) 0core file size (blocks, -c) 0

data seg size (kbytes, -d) unlimiteddata seg size (kbytes, -d) unlimited

scheduling priority (-e) 0scheduling priority (-e) 0

file size (blocks, -f) unlimitedfile size (blocks, -f) unlimited

pending signals (-i) 96832pending signals (-i) 96832

max locked memory (kbytes, -l) 64max locked memory (kbytes, -l) 64

max memory size (kbytes, -m) unlimitedmax memory size (kbytes, -m) unlimited

open files (-n) 1024open files (-n) 1024

pipe size (512 bytes, -p) 8pipe size (512 bytes, -p) 8

POSIX message queues (bytes, -q) 819200POSIX message queues (bytes, -q) 819200

real-time priority (-r) 0real-time priority (-r) 0

stack size (kbytes, -s) 8192stack size (kbytes, -s) 8192

cpu time (seconds, -t) unlimitedcpu time (seconds, -t) unlimited

max user processes (-u) 200max user processes (-u) 200

virtual memory (kbytes, -v) unlimitedvirtual memory (kbytes, -v) unlimited

file locks (-x) unlimitedfile locks (-x) unlimited

ulimitsulimits

core file size (blocks, -c) 0core file size (blocks, -c) 0

data seg size (kbytes, -d) unlimiteddata seg size (kbytes, -d) unlimited

scheduling priority (-e) 0scheduling priority (-e) 0

file size (blocks, -f) unlimitedfile size (blocks, -f) unlimited

pending signals (-i) 96832pending signals (-i) 96832

max locked memory (kbytes, -l) 64max locked memory (kbytes, -l) 64

max memory size (kbytes, -m) unlimitedmax memory size (kbytes, -m) unlimited

open files (-n) 1024open files (-n) 1024

pipe size (512 bytes, -p) 8pipe size (512 bytes, -p) 8

POSIX message queues (bytes, -q) 819200POSIX message queues (bytes, -q) 819200

real-time priority (-r) 0real-time priority (-r) 0

stack size (kbytes, -s) 8192stack size (kbytes, -s) 8192

cpu time (seconds, -t) unlimitedcpu time (seconds, -t) unlimited

max user processes (-u) 200max user processes (-u) 200

virtual memory (kbytes, -v) unlimitedvirtual memory (kbytes, -v) unlimited

file locks (-x) unlimitedfile locks (-x) unlimitedapp2

app1

app3

userXuserX

userXuserX

userXuserX

  user  procsuser  procsuserX    4userX    4

app4

userXuserXssh:ssh:

tty:tty:

ulimitsulimits

● login (on tty, via PAM)● KDM, GDM, XDM & etc. (locally via PAM)● ssh (remotely, via PAM and shell)

● pam_limits– /etc/security/limits.conf

– /etc/security/limits.d/

● shell (sh, bash, zsh, csh, tcsh)– /etc/profile.d/limits.[tcz]sh

ulimitsulimits how-tohow-to

$ cat /proc/self/limits

Limit Soft Limit Hard Limit Units

Max cpu time unlimited unlimited seconds

Max file size unlimited unlimited bytes

Max data size unlimited unlimited bytes

Max stack size 8388608 unlimited bytes

Max core file size 0 unlimited bytes

Max resident set unlimited unlimited bytes

Max processes 200 200 processes

Max open files 1024 4096 files

Max locked memory 65536 65536 bytes

Max address space unlimited unlimited bytes

Max file locks unlimited unlimited locks

Max pending signals 200 200 signals

Max msgqueue size 819200 819200 bytes

Max nice priority 0 0

Max realtime priority 0 0

Max realtime timeout unlimited unlimited us

ulimitsulimits how-tohow-to

$ cat /proc/self/limits

on older kernels:

$ echo -n "Max open files=2000:6000" > /proc/self/limits

$ prlimit

ulimitsulimits how-tohow-to

Other kernel limits

● fs.file-max - max fd for the machine● fs.nr_open - max fd per process● fs.mount-max - max mounted filesystems● kernel.threads-max

● Dedicate a CPU to HW device● Dedicate a CPU to a process

● taskset mask cmd● /proc/interrupts

– /proc/irq/NUM/smp_affinity

– /proc/irq/NUM/smp_affinity_list

– /proc/irq/NUM/affinity_hint

CPU AffinityCPU Affinity

● Dedicate a CPU to HW device● Dedicate a CPU to a process

core0 core1

core2 core3

eth0 1Gbpseth4 10Gbpsmegaraid 6Gbps

CPU AffinityCPU Affinity

● Dedicate a CPU to HW device● Dedicate a CPU to a process

core0 core1

core2 core3

eth0 1Gbpseth1 10Gbpseth2 10Gbpsmegaraid 6Gbps

core0 - eth1 10Gbpscore1 - eth2 10Gbpscore3 - megaraid 6Gbpscore4 - eth0 & processes

CPU AffinityCPU Affinity

taskset example

root@terion:~# taskset -p 2727

pid 2727's current affinity mask: ff

root@terion:~# taskset -pc 3 2727

pid 2727's current affinity list: 0-7

pid 2727's new affinity list: 3

root@terion:~# taskset -p 2727

pid 2727's current affinity mask: 8

root@terion:~# ps axf|grep 2727

2727 ? Ss 2:06 /usr/sbin/acpid

root@terion:~#

irq affinity example

root@terion:~# cat /proc/interrupts

CPU0 CPU1

16: 3567385 0 IO-APIC 16-fasteoi ehci_hcd:usb1

17: 4567 0 IO-APIC 17-fasteoi snd_hda_intel:

23: 50797 0 IO-APIC 23-fasteoi ehci_hcd:usb2

25: 78045696 0 PCI-MSI 512000-edge ahci

36: 12 0 PCI-MSI 409600-edge eth0

37: 169256226 0 PCI-MSI 1572864-edge iwlwifi

38: 3515939 0 PCI-MSI 524288-edge nvidia

irq affinity example

root@terion:~# cd /proc/irq/37

root@terion:/proc/irq/37# cat smp_affinity

ff

root@terion:/proc/irq/37# cat smp_affinity_list

0-7

root@terion:/proc/irq/37# echo 3 > smp_affinity_list

root@terion:/proc/irq/37# cat smp_affinity

08

root@terion:/proc/irq/37# cat smp_affinity_list

3

root@terion:/proc/irq/37#

Other resource limitations can be enforced using virtualization

technologies like KVM, Xen, etc.

What if you want to set a limit to a group of processes?

● CPUSET● CPU● CPUACCT● MEMORY● BLKIO● DEVICES

● freezer● net_cls● net_prio● perf_event● hudgetlb

cGroupscGroups

cGroupscGroups

● freezer● net_cls● net_prio● perf_event● hudgetlb

● CPUSET● CPU● CPUACCT● MEMORY● BLKIO● DEVICES

● cGroups have hierarchy

//

/user1/user1

/user2/user2

/user1/user3/user1/user3

cGroupscGroups

root@goblin:/cgroup# ls -1 cpuset*

cpuset.cpus

cpuset.mems

cpuset.cpu_exclusive

cpuset.mem_exclusive

cpuset.effective_cpus

cpuset.effective_mems

...

cGroupscGroups CPUSET CPUSET

root@goblin:/cgroup# ls -1 cpu.*

cpu.cfs_period_us cpu.cfs_quota_us cpu.rt_period_us cpu.rt_runtime_us cpu.shares cpu.stat

cGroupscGroups CPUCPU

root@goblin:/cgroup# ls -1 cpuacct.*

cpuacct.stat

cpuacct.usage

cpuacct.usage_percpu

cpuacct.usage_all

cpuacct.usage_percpu_sys

cpuacct.usage_percpu_user

cpuacct.usage_sys

cpuacct.usage_user

CPUACCTCPUACCTcGroupscGroups

memory.memsw.failcnt

memory.memsw.limit_in_bytes

memory.memsw.max_usage_in_bytes

memory.memsw.usage_in_bytes

memory.limit_in_bytes memory.usage_in_bytes

memory.soft_limit_in_bytes

memory.max_usage_in_bytes

memory.move_charge_at_immigrate memory.failcnt

memory.numa_stat memory.stat

memory.oom_control memory.pressure_level

memory.swappiness memory.use_hierarchy

cGroups cGroups MEMORYMEMORY

blkio.throttle.io_service_bytes

blkio.throttle.io_serviced

blkio.throttle.read_bps_device

blkio.throttle.read_iops_device

blkio.throttle.write_bps_device

blkio.throttle.write_iops_device

cGroupscGroups BLKIOBLKIO

blkio.weight

blkio.weight_device

blkio.leaf_weight

blkio.leaf_weight_device

BLKIOBLKIO cGroupscGroups

cGroupscGroups

root@goblin:/cgroup# ls -1 devices.*

devices.allow

devices.deny

devices.list

DEVICESDEVICES

Marian HackMan MarinovChief System Architectmm@siteground.com

QuestionsQuestions