Can't get the correct information from /proc/stat

[hejiahang929]

I can get correct information from /proc/cpuinfo and /sys/devices/system/cpu/online, but can't get the correct information from /proc/stat, The information from /proc/stat is all cpus on hosts. Is there any missing configuration?

/proc/cpuinfo

cat /proc/cpuinfo |grep processor
processor	: 0
processor	: 1

/sys/devices/system/cpu/online

cat /sys/devices/system/cpu/online
0-1

/proc/stat

cat /proc/stat
cpu  53578 146 33020 23066313 1873 4506 3573 0 0 0
cpu0 1119 3 801 573933 52 1889 1263 0 0 0
cpu1 1095 4 732 577008 39 65 174 0 0 0
cpu2 1058 2 597 577284 38 58 96 0 0 0
cpu3 1086 4 590 577321 30 56 61 0 0 0
cpu4 1246 5 610 577143 47 56 35 0 0 0
cpu5 1278 4 580 577134 31 58 38 0 0 0
cpu6 1077 3 652 577253 40 59 40 0 0 0
cpu7 1295 3 752 576737 55 89 142 0 0 0
cpu8 1053 4 596 577173 92 64 45 0 0 0
cpu9 1123 4 611 577205 55 58 39 0 0 0
cpu10 1714 3 1062 575911 68 76 46 0 0 0
cpu11 2592 4 1339 574465 26 104 406 0 0 0
cpu12 1499 3 1114 576183 57 112 42 0 0 0
cpu13 1508 4 1008 576388 49 65 44 0 0 0
cpu14 1417 4 962 576506 62 64 39 0 0 0
cpu15 1438 5 958 576483 68 67 48 0 0 0
cpu16 1397 3 942 576568 24 66 47 0 0 0
cpu17 1462 5 982 576460 79 62 44 0 0 0
cpu18 1362 3 1035 576539 33 64 45 0 0 0
cpu19 1340 6 954 576536 73 64 60 0 0 0
cpu20 1669 3 720 576590 29 65 40 0 0 0
cpu21 1065 4 628 577142 33 58 37 0 0 0
cpu22 1064 3 606 577350 16 58 32 0 0 0
cpu23 1002 4 576 577403 33 57 35 0 0 0
cpu24 996 4 582 577371 77 58 34 0 0 0
cpu25 962 3 558 577472 42 55 35 0 0 0
cpu26 1024 4 616 577343 42 57 49 0 0 0
cpu27 965 3 609 577404 57 56 30 0 0 0
cpu28 1556 2 610 576842 40 56 29 0 0 0
cpu29 1113 3 657 577196 56 60 41 0 0 0
cpu30 1628 4 1018 576272 29 66 45 0 0 0
cpu31 1579 6 1092 576176 68 68 45 0 0 0
cpu32 1532 3 1046 576352 36 68 41 0 0 0
cpu33 1419 2 976 576420 30 134 54 0 0 0
cpu34 1543 3 1048 576279 87 70 44 0 0 0
cpu35 1859 4 961 576101 34 67 39 0 0 0
cpu36 1419 4 966 576527 35 65 55 0 0 0
cpu37 1357 4 956 576612 48 64 38 0 0 0
cpu38 1371 4 967 576573 31 63 49 0 0 0
cpu39 1296 3 951 576658 32 65 47 0 0 0
...

lxcfs --version

"5.0.4"

systemctl status lxcfs
● lxcfs.service - FUSE filesystem for LXC
Loaded: loaded (/usr/lib/systemd/system/lxcfs.service; enabled; vendor preset: disabled)
Active: active (running) since Thu 2024-03-14 19:38:44 CST; 1h 18min ago
Docs: man:lxcfs(1)
Main PID: 1296 (lxcfs)
Tasks: 6
Memory: 31.6M
CGroup: /system.slice/lxcfs.service
└─1296 /usr/bin/lxcfs -l --enable-cfs --enable-pidfd /var/lib/lxc/lxcfs/

mount info

k8s version 1.27

    volumeMounts:
    - mountPath: /proc/meminfo
      name: lxcfs-proc-meminfo
    - mountPath: /proc/cpuinfo
      name: lxcfs-proc-cpuinfo
    - mountPath: /proc/diskstats
      name: lxcfs-proc-diskstats
    - mountPath: /proc/uptime
      name: lxcfs-proc-uptime
    - mountPath: /proc/stat
      name: lxcfs-proc-stat
    - mountPath: /proc/swaps
      name: lxcfs-proc-swaps
    - mountPath: /proc/loadavg
      name: lxcfs-proc-loadavg
    - mountPath: /sys/devices/system/cpu/online
      name: lxcfs-cpu-online

  volumes:
  - hostPath:
      path: /var/lib/lxc/lxcfs/proc/meminfo
      type: ""
    name: lxcfs-proc-meminfo
  - hostPath:
      path: /var/lib/lxc/lxcfs/proc/cpuinfo
      type: ""
    name: lxcfs-proc-cpuinfo
  - hostPath:
      path: /var/lib/lxc/lxcfs/proc/diskstats
      type: ""
    name: lxcfs-proc-diskstats
  - hostPath:
      path: /var/lib/lxc/lxcfs/proc/uptime
      type: ""
    name: lxcfs-proc-uptime
  - hostPath:
      path: /var/lib/lxc/lxcfs/proc/stat
      type: ""
    name: lxcfs-proc-stat
  - hostPath:
      path: /var/lib/lxc/lxcfs/proc/swaps
      type: ""
    name: lxcfs-proc-swaps
  - hostPath:
      path: /var/lib/lxc/lxcfs/proc/loadavg
      type: ""
    name: lxcfs-proc-loadavg
  - hostPath:
      path: /var/lib/lxc/lxcfs/sys/devices/system/cpu/online
      type: ""
    name: lxcfs-cpu-online

[mihalicyn]

Hi @hejiahang929

Thank you for your report!

I have tried to reproduce this on my environment with LXC and found no issues.
But I have an idea what can be different in your case.

Please, provide the following information.

(from the host)

ls -la /proc/self/ns
cat /sys/fs/cgroup/cgroup.controllers
mount | grep cgroup
uname -a

(from the container/k8s pod)

ls -la /proc/self/ns
cat /sys/fs/cgroup/cgroup.controllers

[hejiahang929]

Hi @mihalicyn
Thank you for your reply！

follow is the information：

(from the host)

ls -la /proc/self/ns

lrwxrwxrwx 1 root root 0 Mar 27 17:32 cgroup -> cgroup:[4026531835]
lrwxrwxrwx 1 root root 0 Mar 27 17:32 ipc -> ipc:[4026531839]
lrwxrwxrwx 1 root root 0 Mar 27 17:32 mnt -> mnt:[4026531840]
lrwxrwxrwx 1 root root 0 Mar 27 17:32 net -> net:[4026532008]
lrwxrwxrwx 1 root root 0 Mar 27 17:32 pid -> pid:[4026531836]
lrwxrwxrwx 1 root root 0 Mar 27 17:32 pid_for_children -> pid:[4026531836]
lrwxrwxrwx 1 root root 0 Mar 27 17:32 user -> user:[4026531837]
lrwxrwxrwx 1 root root 0 Mar 27 17:32 uts -> uts:[4026531838]

cat /sys/fs/cgroup/cgroup.controllers

my cgroup version is v1 ,so i provides information about v1 version

cat /proc/cgroups
#subsys_name hierarchy num_cgroups enabled
cpuset 5 66 1
cpu 8 215 1
cpuacct 8 215 1
blkio 9 215 1
memory 12 351 1
devices 7 200 1
freezer 2 66 1
net_cls 10 66 1
perf_event 3 66 1
net_prio 10 66 1
hugetlb 4 66 1
pids 6 215 1
rdma 11 66 1

mount | grep cgroup

tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/rdma type cgroup (rw,nosuid,nodev,noexec,relatime,rdma)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)

uname -a

Linux cgpcc01n010237206001.sncloud.com 4.18.0-80.11.1.el7.centos.sn11.x86_64 #1 SMP Mon Feb 24 20:34:03 CST 2020 x86_64 x86_64 x86_64 GNU/Linux

(from the container/k8s pod)

ls -la /proc/self/ns

lrwxrwxrwx 1 root root 0 Mar 27 17:37 cgroup -> cgroup:[4026531835]
lrwxrwxrwx 1 root root 0 Mar 27 17:37 ipc -> ipc:[4026533334]
lrwxrwxrwx 1 root root 0 Mar 27 17:37 mnt -> mnt:[4026533336]
lrwxrwxrwx 1 root root 0 Mar 27 17:37 net -> net:[4026532008]
lrwxrwxrwx 1 root root 0 Mar 27 17:37 pid -> pid:[4026533337]
lrwxrwxrwx 1 root root 0 Mar 27 17:37 pid_for_children -> pid:[4026533337]
lrwxrwxrwx 1 root root 0 Mar 27 17:37 user -> user:[4026531837]
lrwxrwxrwx 1 root root 0 Mar 27 17:37 uts -> uts:[4026531838]

cat /sys/fs/cgroup/cgroup.controllers

my cgroup version is v1 ,so i provides information about v1 version

cat /proc/cgroups
#subsys_name hierarchy num_cgroups enabled
cpuset 4 45 1
cpu 5 174 1
cpuacct 5 174 1
blkio 3 174 1
memory 8 1109 1
devices 7 160 1
freezer 11 45 1
net_cls 2 45 1
perf_event 10 45 1
net_prio 2 45 1
hugetlb 9 45 1
pids 12 174 1
rdma 6 45 1

[gecon]

I'am also dealing with the same problem/bug.

Container shows correct /proc/cpuinfo, but wrong /proc/stat.

The container uses 5 CPUs.
The host has 32 CPUs.

Below is the discussed info:

From the container:

# cat /proc/cpuinfo |grep processor|wc -l
5

# cat /proc/stat |grep -e 'cpu[[:digit:]]'|wc -l
32

#  cat /sys/devices/system/cpu/online
0-4


# ls -la /proc/self/ns
total 0
dr-x--x--x 2 root root 0 Apr 26 07:17 .
dr-xr-xr-x 9 root root 0 Apr 26 07:17 ..
lrwxrwxrwx 1 root root 0 Apr 26 07:17 cgroup -> 'cgroup:[4026533046]'
lrwxrwxrwx 1 root root 0 Apr 26 07:17 ipc -> 'ipc:[4026533044]'
lrwxrwxrwx 1 root root 0 Apr 26 07:17 mnt -> 'mnt:[4026533042]'
lrwxrwxrwx 1 root root 0 Apr 26 07:17 net -> 'net:[4026533048]'
lrwxrwxrwx 1 root root 0 Apr 26 07:17 pid -> 'pid:[4026533045]'
lrwxrwxrwx 1 root root 0 Apr 26 07:17 pid_for_children -> 'pid:[4026533045]'
lrwxrwxrwx 1 root root 0 Apr 26 07:17 time -> 'time:[4026531834]'
lrwxrwxrwx 1 root root 0 Apr 26 07:17 time_for_children -> 'time:[4026531834]'
lrwxrwxrwx 1 root root 0 Apr 26 07:17 user -> 'user:[4026533041]'
lrwxrwxrwx 1 root root 0 Apr 26 07:17 uts -> 'uts:[4026533043]'

# cat /sys/fs/cgroup/cgroup.controllers
cpuset cpu io memory hugetlb pids rdma misc

From the host:

# lxcfs --version
"6.0.0"

# ls -la /proc/self/ns
total 0
dr-x--x--x 2 root root 0 Apr 26 10:17 .
dr-xr-xr-x 9 root root 0 Apr 26 10:17 ..
lrwxrwxrwx 1 root root 0 Apr 26 10:17 cgroup -> 'cgroup:[4026531835]'
lrwxrwxrwx 1 root root 0 Apr 26 10:17 ipc -> 'ipc:[4026531839]'
lrwxrwxrwx 1 root root 0 Apr 26 10:17 mnt -> 'mnt:[4026531841]'
lrwxrwxrwx 1 root root 0 Apr 26 10:17 net -> 'net:[4026531840]'
lrwxrwxrwx 1 root root 0 Apr 26 10:17 pid -> 'pid:[4026531836]'
lrwxrwxrwx 1 root root 0 Apr 26 10:17 pid_for_children -> 'pid:[4026531836]'
lrwxrwxrwx 1 root root 0 Apr 26 10:17 time -> 'time:[4026531834]'
lrwxrwxrwx 1 root root 0 Apr 26 10:17 time_for_children -> 'time:[4026531834]'
lrwxrwxrwx 1 root root 0 Apr 26 10:17 user -> 'user:[4026531837]'
lrwxrwxrwx 1 root root 0 Apr 26 10:17 uts -> 'uts:[4026531838]'

# cat /sys/fs/cgroup/cgroup.controllers
cpuset cpu io memory hugetlb pids rdma misc

# mount | grep cgroup
cgroup2 on /sys/fs/cgroup type cgroup2 (rw,nosuid,nodev,noexec,relatime)

# uname -a
Linux x.y.gr 6.8.7-300.fc40.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Apr 17 19:21:08 UTC 2024 x86_64 GNU/Linux

# cat /proc/cpuinfo |grep processor|wc -l
32

# cat /proc/stat |grep -e 'cpu[[:digit:]]'|wc -l
32

# cat /sys/devices/system/cpu/online
0-31

# journalctl -u lxcfs|tail -25
-- Boot 06a3613662a147e9acfd441611416f34 --
Apr 26 02:19:30 x.y.gr systemd[1]: Starting lxcfs.service - FUSE filesystem for LXC...
Apr 26 02:19:30 x.y.gr systemd[1]: Started lxcfs.service - FUSE filesystem for LXC.
Apr 26 02:19:30 x.y.gr lxcfs[2028]: Starting LXCFS at /usr/bin/lxcfs
Apr 26 02:19:30 x.y.gr lxcfs[2028]: Running constructor lxcfs_init to reload liblxcfs
Apr 26 02:19:30 x.y.gr lxcfs[2028]: mount namespace: 5
Apr 26 02:19:30 x.y.gr lxcfs[2028]: hierarchies:
Apr 26 02:19:30 x.y.gr lxcfs[2028]:   0: fd:   6: cpuset,cpu,io,memory,hugetlb,pids,rdma,misc
Apr 26 02:19:30 x.y.gr lxcfs[2028]: Kernel supports pidfds
Apr 26 02:19:30 x.y.gr lxcfs[2028]: Kernel supports swap accounting
Apr 26 02:19:30 x.y.gr lxcfs[2028]: api_extensions:
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - cgroups
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - sys_cpu_online
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - proc_cpuinfo
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - proc_diskstats
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - proc_loadavg
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - proc_meminfo
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - proc_stat
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - proc_swaps
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - proc_uptime
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - proc_slabinfo
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - shared_pidns
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - cpuview_daemon
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - loadavg_daemon
Apr 26 02:19:30 x.y.gr lxcfs[2028]: - pidfds

# dnf list installed|grep lxc
incus.x86_64                            6.0.0-0.1.fc40                  @copr:copr.fedorainfracloud.org:ganto:lxc4
incus-agent.x86_64                      6.0.0-0.1.fc40                  @copr:copr.fedorainfracloud.org:ganto:lxc4
incus-client.x86_64                     6.0.0-0.1.fc40                  @copr:copr.fedorainfracloud.org:ganto:lxc4
incus-selinux.noarch                    6.0.0-0.1.fc40                  @copr:copr.fedorainfracloud.org:ganto:lxc4
lxc-libs.x86_64                         6.0.0-0.1.fc40                  @copr:copr.fedorainfracloud.org:ganto:lxc4
lxc-templates.noarch                    6.0.0-0.1.fc40                  @copr:copr.fedorainfracloud.org:ganto:lxc4
lxcfs.x86_64                            6.0.0-0.1.fc40                  @copr:copr.fedorainfracloud.org:ganto:lxc4