Hi,
I got an issue with iscsiadm and podman containers. I’m thinking about an issue with selinux and podman.
I’m trying to run democratic CSI, with the synology driver in order to have some iSCSI drive dynamically attach to node. This use the iscsciadm command to mount the scsi drive in the node.
When I disable selinux, everything works well. I’m able to mount my iSCSI inside my podman container.
Without touching to my podman config, I try to run the same container with selinux enforced but iscsciadm return an exit 6.
Here is my podman inspect:
Podman inspect
# podman inspect 30d342f6bbae
[
{
"Id": "30d342f6bbae53fb986fd4d5dbac1ce7c96e9c7d5eb2e781c43885501db033fb",
"Created": "2023-01-16T03:00:34.318906293Z",
"Path": "bin/democratic-csi",
"Args": [
"--csi-version=1.5.0",
"--csi-name=org.democratic-csi.iscsi",
"--driver-config-file=/data/driver-config-file.yaml",
"--log-level=info",
"--csi-mode=node",
"--server-socket=/csi/csi.sock"
],
"State": {
"OciVersion": "1.0.2-dev",
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 11790,
"ConmonPid": 11788,
"ExitCode": 0,
"Error": "",
"StartedAt": "2023-01-16T03:00:34.717470055Z",
"FinishedAt": "0001-01-01T00:00:00Z",
"Health": {
"Status": "",
"FailingStreak": 0,
"Log": null
},
"CgroupPath": "/machine.slice/libpod-30d342f6bbae53fb986fd4d5dbac1ce7c96e9c7d5eb2e781c43885501db033fb.scope",
"CheckpointedAt": "0001-01-01T00:00:00Z",
"RestoredAt": "0001-01-01T00:00:00Z"
},
"Image": "6f5e1aad67f210ee2880e63ec709e3f8502ed1e59f7a4a35ccd586ea9ca1e138",
"ImageDigest": "sha256:9633b08bf21d93dec186e8c4b7a39177fb6d59fd4371c88700097b9cc0aa4712",
"ImageName": "docker.io/democraticcsi/democratic-csi:latest",
"Rootfs": "",
"Pod": "",
"ResolvConfPath": "/run/containers/storage/overlay-containers/30d342f6bbae53fb986fd4d5dbac1ce7c96e9c7d5eb2e781c43885501db033fb/userdata/resolv.conf",
"HostnamePath": "/run/containers/storage/overlay-containers/30d342f6bbae53fb986fd4d5dbac1ce7c96e9c7d5eb2e781c43885501db033fb/userdata/hostname",
"HostsPath": "/run/containers/storage/overlay-containers/30d342f6bbae53fb986fd4d5dbac1ce7c96e9c7d5eb2e781c43885501db033fb/userdata/hosts",
"StaticDir": "/var/lib/containers/storage/overlay-containers/30d342f6bbae53fb986fd4d5dbac1ce7c96e9c7d5eb2e781c43885501db033fb/userdata",
"OCIConfigPath": "/var/lib/containers/storage/overlay-containers/30d342f6bbae53fb986fd4d5dbac1ce7c96e9c7d5eb2e781c43885501db033fb/userdata/config.json",
"OCIRuntime": "crun",
"ConmonPidFile": "/run/containers/storage/overlay-containers/30d342f6bbae53fb986fd4d5dbac1ce7c96e9c7d5eb2e781c43885501db033fb/userdata/conmon.pid",
"PidFile": "/run/containers/storage/overlay-containers/30d342f6bbae53fb986fd4d5dbac1ce7c96e9c7d5eb2e781c43885501db033fb/userdata/pidfile",
"Name": "plugin-7c7d33cb-eb89-dcb6-df52-a9972d05c46a",
"RestartCount": 0,
"Driver": "overlay",
"MountLabel": "system_u:object_r:container_file_t:s0:c1022,c1023",
"ProcessLabel": "",
"AppArmorProfile": "",
"EffectiveCaps": [
"CAP_AUDIT_CONTROL",
"CAP_AUDIT_READ",
"CAP_AUDIT_WRITE",
"CAP_BLOCK_SUSPEND",
"CAP_BPF",
"CAP_CHECKPOINT_RESTORE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_DAC_READ_SEARCH",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_IPC_LOCK",
"CAP_IPC_OWNER",
"CAP_KILL",
"CAP_LEASE",
"CAP_LINUX_IMMUTABLE",
"CAP_MAC_ADMIN",
"CAP_MAC_OVERRIDE",
"CAP_MKNOD",
"CAP_NET_ADMIN",
"CAP_NET_BIND_SERVICE",
"CAP_NET_BROADCAST",
"CAP_NET_RAW",
"CAP_PERFMON",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYSLOG",
"CAP_SYS_ADMIN",
"CAP_SYS_BOOT",
"CAP_SYS_CHROOT",
"CAP_SYS_MODULE",
"CAP_SYS_NICE",
"CAP_SYS_PACCT",
"CAP_SYS_PTRACE",
"CAP_SYS_RAWIO",
"CAP_SYS_RESOURCE",
"CAP_SYS_TIME",
"CAP_SYS_TTY_CONFIG",
"CAP_WAKE_ALARM"
],
"BoundingCaps": [
"CAP_AUDIT_CONTROL",
"CAP_AUDIT_READ",
"CAP_AUDIT_WRITE",
"CAP_BLOCK_SUSPEND",
"CAP_BPF",
"CAP_CHECKPOINT_RESTORE",
"CAP_CHOWN",
"CAP_DAC_OVERRIDE",
"CAP_DAC_READ_SEARCH",
"CAP_FOWNER",
"CAP_FSETID",
"CAP_IPC_LOCK",
"CAP_IPC_OWNER",
"CAP_KILL",
"CAP_LEASE",
"CAP_LINUX_IMMUTABLE",
"CAP_MAC_ADMIN",
"CAP_MAC_OVERRIDE",
"CAP_MKNOD",
"CAP_NET_ADMIN",
"CAP_NET_BIND_SERVICE",
"CAP_NET_BROADCAST",
"CAP_NET_RAW",
"CAP_PERFMON",
"CAP_SETFCAP",
"CAP_SETGID",
"CAP_SETPCAP",
"CAP_SETUID",
"CAP_SYSLOG",
"CAP_SYS_ADMIN",
"CAP_SYS_BOOT",
"CAP_SYS_CHROOT",
"CAP_SYS_MODULE",
"CAP_SYS_NICE",
"CAP_SYS_PACCT",
"CAP_SYS_PTRACE",
"CAP_SYS_RAWIO",
"CAP_SYS_RESOURCE",
"CAP_SYS_TIME",
"CAP_SYS_TTY_CONFIG",
"CAP_WAKE_ALARM"
],
"ExecIDs": [
"d47c7d349db827eea487c56e3ec476158f1f4f293df6d9849d7f7636c726b45d"
],
"GraphDriver": {
"Name": "overlay",
"Data": {
"LowerDir": "/var/lib/containers/storage/overlay/5a3f84c756ba2361b51803f1232b35f1dcacd6e9a5a4d61a303f40167823e621/diff:/var/lib/containers/storage/overlay/b92fcb00d82b8735b9bad29d79270be7445bd85de54f7d9c508b6c742dba39a8/diff:/var/lib/containers/storage/overlay/fc127f6f76588fa46cdc5fbfe3bd40d9a7b871c2fc107311ac4e095618904d79/diff:/var/lib/containers/storage/overlay/6805eeb90409aa4ca7248b26cb7f3440ab156e54b38f7624fcf9972ce878475f/diff:/var/lib/containers/storage/overlay/69ced6f975e6eb5d2e7bce2b1c4bd90feea1086227f31eb80ad47dab8b8d6efd/diff:/var/lib/containers/storage/overlay/a6d55c8eab6dba9d1b91370a711b2176e6ad9c583b92a7b71a34e70261a40af9/diff:/var/lib/containers/storage/overlay/e65b574a332993669d34030cd895edb913b54a92034c07af4ecd585b4e3494bd/diff:/var/lib/containers/storage/overlay/47aed5226aaf4ff5121279c18d6975022182d6b37ac791db9a2e9fcf93916dc3/diff:/var/lib/containers/storage/overlay/54ef73c895b462fb0ce57506b2404e631a2337d9b299a47e4e7b9f346324e20b/diff:/var/lib/containers/storage/overlay/d4a6c19cfe4dafed082c37b8646e37e02efda742a7e5b817174ff587674eac95/diff:/var/lib/containers/storage/overlay/8e1760a4f2f34088de3d13a2b56f94d17e4d88dea15b27354c594f2caaea3320/diff:/var/lib/containers/storage/overlay/b5d883d3d541f59f51c8736ceca394f00880d3a0ba422b05f373a52a27c2ab62/diff:/var/lib/containers/storage/overlay/a8a8eee11799a5ea87eba36bf61f34d233c848a7a1066405bba8ebcd96b89eb3/diff:/var/lib/containers/storage/overlay/26ee9cc64c650fe0d880c6ae0055cfb58598e60609ddb74092f604ab44a52c7d/diff:/var/lib/containers/storage/overlay/a69c3f2dde415ec61e9901cc43016d86cf4054c92604ca8578ca4392e6b41667/diff:/var/lib/containers/storage/overlay/8d2ef22b8edf699d8f545e567b963a47535c5e7f2bb3c41ab8c742c65731948f/diff:/var/lib/containers/storage/overlay/936fecc5c6b9ae9ff7a94f4aa71894e886bc6568fd88364678c3e997ff878d63/diff:/var/lib/containers/storage/overlay/7ee0f188807856959389f326189af4a984b953f0906970790a60452932b7d6a9/diff:/var/lib/containers/storage/overlay/3a2828c7ead5fd693945c0215b0e2395c2884dfb582cd564005c5efdeb7cdb32/diff:/var/lib/containers/storage/overlay/8450b35421cbaab8929ac39fcf94241585b70829891b112a70a9e4af86ffa701/diff:/var/lib/containers/storage/overlay/ef439403c8cf539028a90bc6e04ca2e8b620a20702d950e2433828a4231b8fe2/diff:/var/lib/containers/storage/overlay/29bd8c17e1d41a1dadc706f3ff5170b345df93726464f8efc49e3b4829169a89/diff:/var/lib/containers/storage/overlay/c75eaa0eefd3c60b86bed7b8e234f032b2234382d113a6125c40c553146271fe/diff",
"MergedDir": "/var/lib/containers/storage/overlay/0b155b8191b69f595cb213aa6c1ead9e2c6eb7763741523ad76822f1db7750d9/merged",
"UpperDir": "/var/lib/containers/storage/overlay/0b155b8191b69f595cb213aa6c1ead9e2c6eb7763741523ad76822f1db7750d9/diff",
"WorkDir": "/var/lib/containers/storage/overlay/0b155b8191b69f595cb213aa6c1ead9e2c6eb7763741523ad76822f1db7750d9/work"
}
},
"Mounts": [
{
"Type": "bind",
"Source": "/opt/nomad/alloc/7c7d33cb-eb89-dcb6-df52-a9972d05c46a/alloc",
"Destination": "/alloc",
"Driver": "",
"Mode": "",
"Options": [
"rbind"
],
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/opt/nomad/alloc/7c7d33cb-eb89-dcb6-df52-a9972d05c46a/plugin/local",
"Destination": "/local",
"Driver": "",
"Mode": "",
"Options": [
"rbind"
],
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/opt/nomad/alloc/7c7d33cb-eb89-dcb6-df52-a9972d05c46a/plugin/secrets",
"Destination": "/secrets",
"Driver": "",
"Mode": "",
"Options": [
"noexec",
"rbind"
],
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/opt/nomad/alloc/7c7d33cb-eb89-dcb6-df52-a9972d05c46a/plugin/local/driver-config-file.yaml",
"Destination": "/data/driver-config-file.yaml",
"Driver": "",
"Mode": "",
"Options": [
"rbind"
],
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/",
"Destination": "/host",
"Driver": "",
"Mode": "",
"Options": [
"rbind"
],
"RW": true,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/run/udev",
"Destination": "/run/udev",
"Driver": "",
"Mode": "",
"Options": [
"nosuid",
"nodev",
"rbind"
],
"RW": false,
"Propagation": "rprivate"
},
{
"Type": "bind",
"Source": "/opt/nomad/client/csi/plugins/7c7d33cb-eb89-dcb6-df52-a9972d05c46a",
"Destination": "/csi",
"Driver": "",
"Mode": "",
"Options": [
"rbind"
],
"RW": true,
"Propagation": "rshared"
},
{
"Type": "bind",
"Source": "/opt/nomad/client/csi/node/org.democratic-csi.iscsi",
"Destination": "/local/csi",
"Driver": "",
"Mode": "",
"Options": [
"rbind"
],
"RW": true,
"Propagation": "rshared"
},
{
"Type": "bind",
"Source": "/dev",
"Destination": "/dev",
"Driver": "",
"Mode": "",
"Options": [
"nosuid",
"rbind"
],
"RW": true,
"Propagation": "rprivate"
}
],
"Dependencies": [],
"NetworkSettings": {
"EndpointID": "",
"Gateway": "",
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"MacAddress": "",
"Bridge": "",
"SandboxID": "",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": {},
"SandboxKey": ""
},
"Namespace": "",
"IsInfra": falseThis text will be hidden,
"IsService": false,
"Config": {
"Hostname": "nomad01",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"NOMAD_ALLOC_INDEX=0",
"NOMAD_ALLOC_NAME=democratic-csi-iscsi-node.nodes[0]",
"CSI_ENDPOINT=unix:///csi/csi.sock",This text will be hidden
"CSI_NODE_ID=nomad01",
"container=podman",
"DEBIAN_FRONTEND=noninteractive",
"NOMAD_JOB_NAME=democratic-csi-iscsi-node",
"NOMAD_SECRETS_DIR=/secrets",
"NOMAD_SHORT_ALLOC_ID=7c7d33cb",
"NOMAD_CPU_LIMIT=500",
"NOMAD_JOB_ID=democratic-csi-iscsi-node",
"NOMAD_REGION=global",
"NOMAD_TASK_NAME=plugin",
"NOMAD_GROUP_NAME=nodes",
"TERM=xterm",
"NODE_ENV=production",
"LANG=en_US.utf8",
"NOMAD_TASK_DIR=/local",
"NOMAD_DC=home",
"NOMAD_PARENT_CGROUP=nomad.slice",
"NOMAD_ALLOC_ID=7c7d33cb-eb89-dcb6-df52-a9972d05c46a",
"NOMAD_MEMORY_LIMIT=256",
"NOMAD_NAMESPACE=default",
"NOMAD_ALLOC_DIR=/alloc",
"HOME=/root",
"HOSTNAME=nomad01"
],
"Cmd": [
"--csi-version=1.5.0",
"--csi-name=org.democratic-csi.iscsi",
"--driver-config-file=/data/driver-config-file.yaml",
"--log-level=info",
"--csi-mode=node",
"--server-socket=/csi/csi.sock"
],
"Image": "docker.io/democraticcsi/democratic-csi:latest",
"Volumes": null,
"WorkingDir": "/home/csi/app",
"Entrypoint": "bin/democratic-csi",
"OnBuild": null,
"Labels": {
"org.opencontainers.image.created": "2022-10-18T06:05:20+00:00",
"org.opencontainers.image.licenses": "MIT",
"org.opencontainers.image.revision": "c8b13450d2fa8432daac7df09fa5e32f50aa0b74",
"org.opencontainers.image.source": "https://github.com/democratic-csi/democratic-csi",
"org.opencontainers.image.url": "https://github.com/democratic-csi/democratic-csi"
},
"Annotations": {
"io.container.manager": "libpod",
"io.kubernetes.cri-o.Created": "2023-01-16T03:00:34.318906293Z",
"io.podman.annotations.autoremove": "FALSE",
"io.podman.annotations.init": "FALSE",
"io.podman.annotations.privileged": "TRUE",
"io.podman.annotations.publish-all": "FALSE",
"org.opencontainers.image.stopSignal": "15"
},
"StopSignal": 15,
"HealthcheckOnFailureAction": "none",
"Umask": "0022",
"Timeout": 0,
"StopTimeout": 10,
"Passwd": true
},
"HostConfig": {
"Binds": [
"/opt/nomad/alloc/7c7d33cb-eb89-dcb6-df52-a9972d05c46a/alloc:/alloc:rw,rprivate,rbind",
"/opt/nomad/alloc/7c7d33cb-eb89-dcb6-df52-a9972d05c46a/plugin/local:/local:rw,rprivate,rbind",
"/opt/nomad/alloc/7c7d33cb-eb89-dcb6-df52-a9972d05c46a/plugin/secrets:/secrets:rw,rprivate,noexec,rbind",
"/opt/nomad/alloc/7c7d33cb-eb89-dcb6-df52-a9972d05c46a/plugin/local/driver-config-file.yaml:/data/driver-config-file.yaml:rw,rprivate,rbind",
"/:/host:rw,rprivate,rbind",
"/run/udev:/run/udev:ro,rprivate,nosuid,nodev,rbind",
"/opt/nomad/client/csi/plugins/7c7d33cb-eb89-dcb6-df52-a9972d05c46a:/csi:rshared,rw,rbind",
"/opt/nomad/client/csi/node/org.democratic-csi.iscsi:/local/csi:rshared,rw,rbind",
"/dev:/dev:rw,rprivate,nosuid,rbind"
],
"CgroupManager": "systemd",
"CgroupMode": "private",
"ContainerIDFile": "",
"LogConfig": {
"Type": "k8s-file",
"Config": null,
"Path": "/opt/nomad/alloc/7c7d33cb-eb89-dcb6-df52-a9972d05c46a/alloc/logs/.plugin.stdout.fifo",
"Tag": "",
"Size": "0B"
},
"NetworkMode": "host",
"PortBindings": {},
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": [],
"CapDrop": [],
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": [],
"GroupAdd": [],
"IpcMode": "shareable",
"Cgroup": "",
"Cgroups": "default",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "private",
"Privileged": true,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [],
"Tmpfs": {},
"UTSMode": "private",
"UsernsMode": "",
"ShmSize": 65536000,
"Runtime": "oci",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 500,
"Memory": 268435456,
"NanoCpus": 0,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 536870912,
"MemorySwappiness": -1,
"OomKillDisable": false,
"PidsLimit": 2048,
"Ulimits": [
{
"Name": "RLIMIT_NOFILE",
"Soft": 1048576,
"Hard": 1048576
},
{
"Name": "RLIMIT_NPROC",
"Soft": 4194304,
"Hard": 4194304
}
],
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"CgroupConf": null
}
}
]
Here is the line I got in the journal when the container try to run iscsciadm:
AVC avc: denied { dac_override } for pid=54661 comm="iscsiadm" capability=1 scontext=system_u:system_r:iscsid_t:s0 tcontext=system_u:system_r:iscsid_t:s0 tclass=capability permissive=0
Jan 13 04:27:27 nomad01 audit[54661]: SYSCALL arch=c00000b7 syscall=56 success=no exit=-13 a0=ffffffffffffff9c a1=aaaae4916bb0 a2=241 a3=1b6 items=0 ppid=54660 pid=54661 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm="iscsiadm" exe="/usr/sbin/iscsiadm" subj=system_u:system_r:iscsid_t:s0 key=(null)
Jan 13 04:27:27 nomad01 audit: PROCTITLE proctitle=697363736961646D002D6D006E6F6465002D540069716E2E323030302D30312E636F6D2E73796E6F6C6F67793A6373692E746573742D6E6F6D61642D7465737432002D70003139322E3136382E31382E31350033323630002D6F006E6577
Inside the container, the iscsiadm
command is a fake binary with this content:
# cat /usr/local/sbin/iscsiadm
#!/bin/bash
: "${ISCSIADM_HOST_STRATEGY:=chroot}"
: "${ISCSIADM_HOST_PATH:=iscsiadm}"
echoerr() { printf "%s\n" "$*" >&2; }
case ${ISCSIADM_HOST_STRATEGY} in
chroot)
# https://engineering.docker.com/2019/07/road-to-containing-iscsi/
chroot /host /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" ${ISCSIADM_HOST_PATH} "${@:1}"
;;
nsenter)
# https://github.com/siderolabs/extensions/issues/38#issuecomment-1125403043
iscsid_pid=$(pgrep iscsid)
if [[ "${iscsid_pid}x" == "x" ]]; then
echoerr "failed to find iscsid pid for nsenter"
exit 1
fi
nsenter --mount="/proc/${iscsid_pid}/ns/mnt" --net="/proc/${iscsid_pid}/ns/net" -- ${ISCSIADM_HOST_PATH} "${@:1}"
;;
*)
echoerr "invalid ISCSIADM_HOST_STRATEGY: ${ISCSIADM_HOST_STRATEGY}"
exit 1
;;
esac
My coreos version is Fedora CoreOS 37.20221225.3.0.
Quentin.