Open Stack

[AMD Official Use Only]

Hi All,

(1) Using kata-containers-2.2.0-alpha1 -  configuration.toml --

# Copyright (c) 2017-2019 Intel Corporation # # SPDX-License-Identifier: Apache-2.0 #

# XXX: WARNING: this file is auto-generated.
# XXX:
# XXX: Source file: "cli/config/configuration-qemu.toml.in"
# XXX: Project:
# XXX:   Name: Kata Containers
# XXX:   Type: kata

[hypervisor.qemu]
path = "/usr/local/bin/qemu-system-x86_64"
#kernel = "/usr/share/kata-containers/vmlinux.container"
kernel = "/usr/share/kata-containers/vmlinux-sev-es-5.9-v1.container"
#image = "/usr/share/kata-containers/kata-containers.img"
initrd = "/usr/share/kata-containers/kata-containers-initrd.img"
machine_type = "q35"

# Enable confidential guest support.
# Toggling that setting may trigger different hardware features, ranging # from memory encryption to both memory and CPU-state encryption and integrity.
# The Kata Containers runtime dynamically detects the available feature set and # aims at enabling the largest possible one.
# Default false
confidential_guest = true

# List of valid annotation names for the hypervisor # Each member of the list is a regular expression, which is the base name # of the annotation, e.g. "path" for io.katacontainers.config.hypervisor.path"
enable_annotations = []

# List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3).
# The default if not set is empty (all annotations rejected.) # Your distribution recommends: ["/usr/bin/qemu-system-x86_64"] valid_hypervisor_paths = ["/usr/local/bin/qemu-system-x86_64"]

# Optional space-separated list of options to pass to the guest kernel.
# For example, use `kernel_params = "vsyscall=emulate"` if you are having # trouble running pre-2.15 glibc.
#
# WARNING: - any parameter specified here will take priority over the default # parameter value of the same name used to start the virtual machine.
# Do not set values here unless you understand the impact of doing so as you # may stop the virtual machine from booting.
# To see the list of default parameters, enable hypervisor debug, create a # container and look for 'default-kernel-parameters' log entries.
#kernel_params = ""
kernel_params = " root=/dev/vda1 rootflags=data=ordered,errors=remount-ro agent.log=debug initcall_debug"

# Path to the firmware.
# If you want that qemu uses the default firmware leave this option empty #firmware = ""
firmware = "/usr/local/share/qemu/OVMF_CODE.fd.kata"
...........
..............
# Shared file system type:
#   - virtio-fs (default)
#   - virtio-9p
shared_fs = "virtio-9p"
..............
................


(2) containerd version 1.5.2
/usr/local/bin/containerd --version
containerd github.com/containerd/containerd 1.5.2-0ubuntu1~20.04.2

/usr/local/bin/containerd --config /etc/containerd/config.toml

# cat /etc/containerd/config.toml
#   Copyright 2018-2020 Docker Inc.

#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at

#       http://www.apache.org/licenses/LICENSE-2.0

#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
[debug]
        level = "debug"

[plugins.cri.containerd]
      no_pivot = false
    [plugins.cri.containerd.runtimes]
      [plugins.cri.containerd.runtimes.runc]
         runtime_type = "io.containerd.runc.v1"
         [plugins.cri.containerd.runtimes.runc.options]
           NoPivotRoot = false
           NoNewKeyring = false
           ShimCgroup = ""
           IoUid = 0
           IoGid = 0
           BinaryName = "runc"
           Root = ""
           CriuPath = ""
           SystemdCgroup = false
      [plugins.cri.containerd.runtimes.kata]
         runtime_type = "io.containerd.kata.v2"
          [plugins.cri.containerd.runtimes.kata.options]
           ConfigPath = "/etc/kata-containers/configuration.toml"
           NoPivotRoot = false
           NoNewKeyring = false
           ShimCgroup = ""
           IoUid = 0
           IoGid = 0
           BinaryName = "/usr/local/bin/kata-runtime"
           Root = ""
           CriuPath = ""
           SystemdCgroup = false
      [plugins.cri.containerd.runtimes.katacli]
         runtime_type = "io.containerd.runc.v1"
         [plugins.cri.containerd.runtimes.katacli.options]
           NoPivotRoot = false
           NoNewKeyring = false
           ShimCgroup = ""
           IoUid = 0
           IoGid = 0
           BinaryName = "/usr/local/bin/kata-runtime"
           Root = ""
           CriuPath = ""
           SystemdCgroup = false


(3) # ctr run --runtime io.containerd.kata.v2 -t --rm docker.io/library/alpine_linux:latest bash
ctr: failed to create shim: Failed to check if grpc server is working: rpc error: code = DeadlineExceeded desc = timed out connecting to vsock 1025935578:1024: unknown


(4) containerd LOG:

DEBU[2021-07-30T07:53:27.915940335Z] stat snapshot                                 key="sha256:7444ea29e45e927abea1f923bf24cac20deaddea603c4bb1c7f2f5819773d453"
DEBU[2021-07-30T07:53:27.921636473Z] prepare snapshot                              key=bash parent="sha256:7444ea29e45e927abea1f923bf24cac20deaddea603c4bb1c7f2f5819773d453"
DEBU[2021-07-30T07:53:27.930411184Z] event published                               ns=default topic=/snapshot/prepare type=containerd.events.SnapshotPrepare
DEBU[2021-07-30T07:53:27.939032985Z] get snapshot mounts                           key=bash
DEBU[2021-07-30T07:53:27.955103540Z] event published                               ns=default topic=/containers/create type=containerd.events.ContainerCreate
DEBU[2021-07-30T07:53:27.962015595Z] get snapshot mounts                           key=bash
DEBU[2021-07-30T07:53:28.039779478Z] garbage collected                             d=2.588852ms
time="2021-07-30T07:53:28.043806590Z" level=debug msg="registering ttrpc server"
time="2021-07-30T07:53:28.043934212Z" level=debug msg="serving api on socket" socket="[inherited from parent]"
time="2021-07-30T07:53:28.043956084Z" level=info msg="starting signal loop" namespace=default path=/run/containerd/io.containerd.runtime.v2.task/default/bash pid=1590317 time="2021-07-30T07:53:28.044534952Z" level=debug msg="converting /run/containerd/io.containerd.runtime.v2.task/default/bash/config.json" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=compatoci time="2021-07-30T07:53:28.049788100Z" level=info msg="loaded configuration" file=/etc/kata-containers/configuration.toml format=TOML name=containerd-shim-v2 pid=1590317 sandbox=bash source=katautils time="2021-07-30T07:53:28.049991937Z" level=info msg="IOMMUPlatform is enabled by default." name=containerd-shim-v2 pid=1590317 sandbox=bash source=katautils time="2021-07-30T07:53:28.051163250Z" level=info msg="shm-size detected: 67108864" source=virtcontainers subsystem=oci time="2021-07-30T07:53:28.054669132Z" level=info msg="create netns" name=containerd-shim-v2 netns=/var/run/netns/cnitest-6747183b-4b3c-3ec6-81c4-d517d32b120b pid=1590317 sandbox=bash source=katautils time="2021-07-30T07:53:28.071513285Z" level=info msg="Enabling SEV guest protection" kernel-params="[{tsc reliable} {no_timer_check } {rcupdate.rcu_expedited 1} {i8042.direct 1} {i8042.dumbkbd 1} {i8042.nopnp 1} {i8042.noaux 1} {noreplace-smp } {reboot k} {console hvc0} {console hvc1} {cryptomgr.notests } {net.ifnames 0} {pci lastbus=0}]" kernel-params-debug="[{debug }]" kernel-params-non-debug="[{quiet }]" machine="{q35  accel=kvm,kernel_irqchip=split}" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=qemuAmd64 time="2021-07-30T07:53:28.092119026Z" level=info msg="adding volume" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=qemu volume-type=virtio-9p time="2021-07-30T07:53:28.093015598Z" level=info msg="Endpoints found after scan" endpoints="[]" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=network time="2021-07-30T07:53:28.093157978Z" level=info msg="Starting VM" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=sandbox time="2021-07-30T07:53:28.093562065Z" level=info msg="Adding extra file [0xc000010058]" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=qmp time="2021-07-30T07:53:28.093652406Z" level=info msg="launching /usr/local/bin/qemu-system-x86_64 with: [-name sandbox-bash -uuid b1ff0af6-de06-4025-ab63-9aad959900d5 -machine q35,accel=kvm,kernel_irqchip=split,confidential-guest-support=sev -cpu host,pmu=off -qmp unix:/run/vc/vm/bash/qmp.sock,server=on,wait=off -m 2048M,slots=10,maxmem=129746M -device pci-bridge,bus=pcie.0,id=pci-bridge-0,chassis_nr=1,shpc=on,addr=2 -device virtio-serial-pci,disable-modern=false,id=serial0 -device virtconsole,chardev=charconsole0,id=console0 -chardev socket,id=charconsole0,path=/run/vc/vm/bash/console.sock,server=on,wait=off -device virtio-scsi-pci,id=scsi0,disable-modern=false -object sev-guest,id=sev,cbitpos=51,reduced-phys-bits=5 -drive if=pflash,format=raw,readonly=on,file=/usr/local/share/qemu/OVMF_CODE.fd.kata -object rng-random,id=rng0,filename=/dev/urandom -device virtio-rng-pci,rng=rng0 -device vhost-vsock-pci,disable-modern=false,vhostfd=3,id=vsock-1025935578,guest-cid=1025935578 -device virtio-9p-pci,disable-modern=false,fsdev=extra-9p-kataShared,mount_tag=kataShared -fsdev local,id=extra-9p-kataShared,path=/run/kata-containers/shared/sandboxes/bash/shared,security_model=none,multidevs=remap -rtc base=utc,driftfix=slew,clock=host -global kvm-pit.lost_tick_policy=discard -vga none -no-user-config -nodefaults -nographic --no-reboot -daemonize -object memory-backend-ram,id=dimm1,size=2048M -numa node,memdev=dimm1 -kernel /usr/share/kata-containers/vmlinux-sev-es-5.9-v1.container -initrd /usr/share/kata-containers/kata-containers-initrd.img -append tsc=reliable no_timer_check rcupdate.rcu_expedited=1 i8042.direct=1 i8042.dumbkbd=1 i8042.nopnp=1 i8042.noaux=1 noreplace-smp reboot=k console=hvc0 console=hvc1 cryptomgr.notests net.ifnames=0 pci=lastbus=0 debug panic=1 nr_cpus=240 scsi_mod.scan=none root=/dev/vda1 rootflags=data=ordered,errors=remount-ro agent.log=debug initcall_debug -pidfile /run/vc/vm/bash/pid -D /run/vc/vm/bash/qemu.log -smp 1,cores=1,threads=1,sockets=240,maxcpus=240]" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=qmp time="2021-07-30T07:53:29.231939187Z" level=info msg="QMP details" name=containerd-shim-v2 pid=1590317 qmp-capabilities=oob qmp-major-version=6 qmp-micro-version=91 qmp-minor-version=0 sandbox=bash source=virtcontainers subsystem=qemu time="2021-07-30T07:53:29.232726561Z" level=info msg="scanner return error: read unix @->/run/vc/vm/bash/qmp.sock: use of closed network connection" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=qmp time="2021-07-30T07:53:29.232914117Z" level=info msg="VM started" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=sandbox time="2021-07-30T07:53:29.233044113Z" level=info msg="New client" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=kata_agent url="vsock://1025935578:1024"
time="2021-07-30T07:53:59.475362081Z" level=info msg="Stopping Sandbox" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=qemu time="2021-07-30T07:53:59.477063059Z" level=warning msg="The virtiofsd had stopped" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=qemu time="2021-07-30T07:53:59.477246988Z" level=info msg="cleanup vm path" dir=/run/vc/vm/bash link=/run/vc/vm/bash name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=qemu time="2021-07-30T07:53:59.477447608Z" level=info msg="Network namespace \"/var/run/netns/cnitest-6747183b-4b3c-3ec6-81c4-d517d32b120b\" deleted" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=network time="2021-07-30T07:53:59.478504534Z" level=warning msg="sandbox cgroups path is empty" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=sandbox time="2021-07-30T07:53:59.478574457Z" level=info msg="cleanup agent" name=containerd-shim-v2 path=/run/kata-containers/shared/sandboxes/bash/shared pid=1590317 sandbox=bash source=virtcontainers subsystem=kata_agent time="2021-07-30T07:53:59.479262623Z" level=warning msg="failed to cleanup netns" error="failed to get netns /var/run/netns/cnitest-6747183b-4b3c-3ec6-81c4-d517d32b120b: failed to Statfs \"/var/run/netns/cnitest-6747183b-4b3c-3ec6-81c4-d517d32b120b\": no such file or directory" name=containerd-shim-v2 path=/var/run/netns/cnitest-6747183b-4b3c-3ec6-81c4-d517d32b120b pid=1590317 sandbox=bash source=katautils time="2021-07-30T07:53:59.479464936Z" level=info msg="scanner return error: <nil>" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=qmp time="2021-07-30T07:53:59.479683231Z" level=info msg="QMP event channel closed" name=containerd-shim-v2 pid=1590317 sandbox=bash source=virtcontainers subsystem=qemu
DEBU[2021-07-30T07:53:59.480447321Z] failed to delete task                         error="rpc error: code = NotFound desc = container does not exist bash: not found" id=bash
INFO[2021-07-30T07:53:59.484752059Z] shim disconnected                             id=bash
WARN[2021-07-30T07:53:59.484860645Z] cleaning up after shim disconnected           id=bash namespace=default
INFO[2021-07-30T07:53:59.484882296Z] cleaning up dead shim
ERRO[2021-07-30T07:53:59.545695272Z] failed to delete                              cmd="/usr/local/bin/containerd-shim-kata-v2 -namespace default -address /run/containerd/containerd.sock -publish-binary /usr/local/bin/containerd -id bash -bundle /run/containerd/io.containerd.runtime.v2.task/default/bash delete" error="exit status 1"
WARN[2021-07-30T07:53:59.545862980Z] failed to clean up after shim disconnected    error="time=\"2021-07-30T07:53:59Z\" level=warning msg=\"failed to cleanup container\" container=bash error=\"open /run/vc/sbs/bash: no such file or directory\" name=containerd-shim-v2 pid=1590410 sandbox=bash source=containerd-kata-shim-v2\nio.containerd.kata.v2: open /run/vc/sbs/bash: no such file or directory\n: exit status 1" id=bash namespace=default
ERRO[2021-07-30T07:53:59.545951698Z] copy shim log                                 error="read /proc/self/fd/15: file already closed"
DEBU[2021-07-30T07:53:59.557980504Z] remove snapshot                               key=bash
DEBU[2021-07-30T07:53:59.560442896Z] event published                               ns=default topic=/snapshot/remove type=containerd.events.SnapshotRemove
DEBU[2021-07-30T07:53:59.563483766Z] event published                               ns=default topic=/containers/delete type=containerd.events.ContainerDelete
DEBU[2021-07-30T07:53:59.610327214Z] schedule snapshotter cleanup                  snapshotter=overlayfs
DEBU[2021-07-30T07:53:59.612570831Z] removed snapshot                              key=default/23/bash snapshotter=overlayfs
DEBU[2021-07-30T07:53:59.613058306Z] snapshot garbage collected                    d=2.633938ms snapshotter=overlayfs
DEBU[2021-07-30T07:53:59.613107950Z] garbage collected                             d=2.265409ms


Any suggestions please.


--Thanking You
--Sandeep