User guide OpenShift sandboxed containers 1.8 Deploying sandboxed containers in OpenShift Container Platform Red Hat Customer Content Services

Commands:
apiVersion: "local.storage.openshift.io/v1" kind: "LocalVolume" metadata: name: "local-disks" namespace: "openshift-local-storage" spec: nodeSelector: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - worker-0 storageClassDevices: - storageClassName: "local-sc" forceWipeDevicesAndDestroyAllData: false volumeMode: Block 1 devicePaths: - /path/to/device 2
oc describe runtimeclass kata
kind: RuntimeClass apiVersion: node.k8s.io/v1 metadata: name: kata overhead: podFixed: memory: "500Mi" cpu: "500m"
apiVersion: v1 kind: <object> spec: runtimeClassName: kata
apiVersion: v1 kind: Namespace metadata: name: openshift-sandboxed-containers-operator
oc apply -f osc-namespace.yaml
apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: sandboxed-containers-operator-group namespace: openshift-sandboxed-containers-operator spec: targetNamespaces: - openshift-sandboxed-containers-operator
oc apply -f osc-operatorgroup.yaml
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: sandboxed-containers-operator namespace: openshift-sandboxed-containers-operator spec: channel: stable installPlanApproval: Automatic name: sandboxed-containers-operator source: redhat-operators sourceNamespace: openshift-marketplace startingCSV: sandboxed-containers-operator.v1.8.1
oc apply -f osc-subscription.yaml
oc get csv -n openshift-sandboxed-containers-operator
watch oc get csv -n openshift-sandboxed-containers-operator
NAME DISPLAY VERSION REPLACES PHASE openshift-sandboxed-containers openshift-sandboxed-containers-operator 1.8.1 1.7.0 Succeeded
apiVersion: "local.storage.openshift.io/v1" kind: "LocalVolume" metadata: name: "local-disks" namespace: "openshift-local-storage" 1 spec: nodeSelector: 2 nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname operator: In values: - ip-10-0-136-143 - ip-10-0-140-255 - ip-10-0-144-180 storageClassDevices: - storageClassName: "local-sc" 3 forceWipeDevicesAndDestroyAllData: false 4 volumeMode: Block devicePaths: 5 - /path/to/device 6
oc apply -f <local-volume>.yaml
oc get all -n openshift-local-storage
NAME READY STATUS RESTARTS AGE pod/diskmaker-manager-9wzms 1/1 Running 0 5m43s pod/diskmaker-manager-jgvjp 1/1 Running 0 5m43s pod/diskmaker-manager-tbdsj 1/1 Running 0 5m43s pod/local-storage-operator-7db4bd9f79-t6k87 1/1 Running 0 14m NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/local-storage-operator-metrics ClusterIP 172.30.135.36 <none> 8383/TCP,8686/TCP 14m NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE daemonset.apps/diskmaker-manager 3 3 3 3 3 <none> 5m43s NAME READY UP-TO-DATE AVAILABLE AGE deployment.apps/local-storage-operator 1/1 1 1 14m NAME DESIRED CURRENT READY AGE replicaset.apps/local-storage-operator-7db4bd9f79 1 1 1 14m
oc get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE local-pv-1cec77cf 100Gi RWO Delete Available local-sc 88m local-pv-2ef7cd2a 100Gi RWO Delete Available local-sc 82m local-pv-3fa1c73 100Gi RWO Delete Available local-sc 48m
oc debug node/worker-0 -- chcon -vt container_file_t /host/path/to/device
system_u:object_r:container_file_t:s0 /host/path/to/device
apiVersion: nfd.openshift.io/v1 kind: NodeFeatureDiscovery metadata: name: nfd-kata namespace: openshift-nfd spec: workerConfig: configData: | sources: custom: - name: "feature.node.kubernetes.io/runtime.kata" matchOn: - cpuId: ["SSE4", "VMX"] loadedKMod: ["kvm", "kvm_intel"] - cpuId: ["SSE4", "SVM"] loadedKMod: ["kvm", "kvm_amd"]
oc create -f nfd.yaml
apiVersion: kataconfiguration.openshift.io/v1 kind: KataConfig metadata: name: example-kataconfig spec: checkNodeEligibility: true
oc create -f kata-config.yaml
oc get nodes --selector='feature.node.kubernetes.io/runtime.kata=true'
NAME STATUS ROLES AGE VERSION compute-3.example.com Ready worker 4h38m v1.25.0 compute-2.example.com Ready worker 4h35m v1.25.0
apiVersion: kataconfiguration.openshift.io/v1 kind: KataConfig metadata: name: example-kataconfig spec: checkNodeEligibility: false 1 logLevel: info kataConfigPoolSelector: matchLabels: <label_key>: '<label_value>' 2
oc apply -f example-kataconfig.yaml
watch "oc describe kataconfig | sed -n /^Status:/,/^Events/p"
oc describe runtimeclass kata
kind: RuntimeClass apiVersion: node.k8s.io/v1 metadata: name: kata overhead: podFixed: memory: "500Mi" cpu: "500m"
oc apply -f runtimeclass.yaml
apiVersion: v1 kind: <object> spec: runtimeClassName: kata
apiVersion: confidentialcontainers.org/v1alpha1 kind: PeerPodConfig metadata: name: peerpodconfig-openshift namespace: openshift-sandboxed-containers-operator spec: cloudSecretName: peer-pods-secret configMapName: peer-pods-cm limit: "10" 1 nodeSelector: node-role.kubernetes.io/kata-oc: ""
INSTANCE_ID=USD(oc get nodes -l 'node-role.kubernetes.io/worker' -o jsonpath='{.items[0].spec.providerID}' | sed 's#[^ ]*/##g')
AWS_REGION=USD(oc get infrastructure/cluster -o jsonpath='{.status.platformStatus.aws.region}')
AWS_SG_IDS=(USD(aws ec2 describe-instances --instance-ids USD{INSTANCE_ID} --query 'Reservations[*].Instances[*].SecurityGroups[*].GroupId' --output text --region USDAWS_REGION))
for AWS_SG_ID in "USD{AWS_SG_IDS[@]}"; do aws ec2 authorize-security-group-ingress --group-id USDAWS_SG_ID --protocol tcp --port 15150 --source-group USDAWS_SG_ID --region USDAWS_REGION aws ec2 authorize-security-group-ingress --group-id USDAWS_SG_ID --protocol tcp --port 9000 --source-group USDAWS_SG_ID --region USDAWS_REGION done
apiVersion: v1 kind: Secret metadata: name: peer-pods-secret namespace: openshift-sandboxed-containers-operator type: Opaque stringData: AWS_ACCESS_KEY_ID: "<aws_access_key>" 1 AWS_SECRET_ACCESS_KEY: "<aws_secret_access_key>" 2
INSTANCE_ID=USD(oc get nodes -l 'node-role.kubernetes.io/worker' -o jsonpath='{.items[0].spec.providerID}' | sed 's#[^ ]*/##g')
AWS_REGION=USD(oc get infrastructure/cluster -o jsonpath='{.status.platformStatus.aws.region}') && echo "AWS_REGION: \"USDAWS_REGION\""
AWS_SUBNET_ID=USD(aws ec2 describe-instances --instance-ids USD{INSTANCE_ID} --query 'Reservations[*].Instances[*].SubnetId' --region USD{AWS_REGION} --output text) && echo "AWS_SUBNET_ID: \"USDAWS_SUBNET_ID\""
AWS_VPC_ID=USD(aws ec2 describe-instances --instance-ids USD{INSTANCE_ID} --query 'Reservations[*].Instances[*].VpcId' --region USD{AWS_REGION} --output text) && echo "AWS_VPC_ID: \"USDAWS_VPC_ID\""
AWS_SG_IDS=USD(aws ec2 describe-instances --instance-ids USD{INSTANCE_ID} --query 'Reservations[*].Instances[*].SecurityGroups[*].GroupId' --region USDAWS_REGION --output json | jq -r '.[][][]' | paste -sd ",") && echo "AWS_SG_IDS: \"USDAWS_SG_IDS\""
apiVersion: v1 kind: ConfigMap metadata: name: peer-pods-cm namespace: openshift-sandboxed-containers-operator data: CLOUD_PROVIDER: "aws" VXLAN_PORT: "9000" PODVM_INSTANCE_TYPE: "t3.medium" 1 PODVM_INSTANCE_TYPES: "t2.small,t2.medium,t3.large" 2 PROXY_TIMEOUT: "5m" PODVM_AMI_ID: "<podvm_ami_id>" 3 AWS_REGION: "<aws_region>" 4 AWS_SUBNET_ID: "<aws_subnet_id>" 5 AWS_VPC_ID: "<aws_vpc_id>" 6 AWS_SG_IDS: "<aws_sg_ids>" 7 DISABLECVM: "true"
oc get events -n openshift-sandboxed-containers-operator --field-selector involvedObject.name=osc-podvm-image-creation
oc logs -n openshift-sandboxed-containers-operator jobs/osc-podvm-image-creation
apiVersion: v1 kind: <object> spec: runtimeClassName: kata-remote
apiVersion: v1 kind: <object> metadata: annotations: io.katacontainers.config.hypervisor.machine_type: "t3.medium" 1
apiVersion: v1 kind: <Pod> metadata: annotations: io.katacontainers.config.hypervisor.default_vcpus: <vcpus> io.katacontainers.config.hypervisor.default_memory: <memory>
apiVersion: v1 kind: Namespace metadata: name: openshift-sandboxed-containers-operator
oc apply -f osc-namespace.yaml
apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: sandboxed-containers-operator-group namespace: openshift-sandboxed-containers-operator spec: targetNamespaces: - openshift-sandboxed-containers-operator
oc apply -f osc-operatorgroup.yaml
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: sandboxed-containers-operator namespace: openshift-sandboxed-containers-operator spec: channel: stable installPlanApproval: Automatic name: sandboxed-containers-operator source: redhat-operators sourceNamespace: openshift-marketplace startingCSV: sandboxed-containers-operator.v1.8.1
oc apply -f osc-subscription.yaml
oc get csv -n openshift-sandboxed-containers-operator
watch oc get csv -n openshift-sandboxed-containers-operator
NAME DISPLAY VERSION REPLACES PHASE openshift-sandboxed-containers openshift-sandboxed-containers-operator 1.8.1 1.7.0 Succeeded
oc get peerpodconfig peerpodconfig-openshift -n openshift-sandboxed-containers-operator -o jsonpath='{.spec.limit}{"\n"}'
oc patch peerpodconfig peerpodconfig-openshift -n openshift-sandboxed-containers-operator --type merge --patch '{"spec":{"limit":"<value>"}}' 1
INSTANCE_ID=USD(oc get nodes -l 'node-role.kubernetes.io/worker' -o jsonpath='{.items[0].spec.providerID}' | sed 's#[^ ]*/##g')
AWS_REGION=USD(oc get infrastructure/cluster -o jsonpath='{.status.platformStatus.aws.region}')
AWS_SG_IDS=(USD(aws ec2 describe-instances --instance-ids USD{INSTANCE_ID} --query 'Reservations[*].Instances[*].SecurityGroups[*].GroupId' --output text --region USDAWS_REGION))
for AWS_SG_ID in "USD{AWS_SG_IDS[@]}"; do aws ec2 authorize-security-group-ingress --group-id USDAWS_SG_ID --protocol tcp --port 15150 --source-group USDAWS_SG_ID --region USDAWS_REGION aws ec2 authorize-security-group-ingress --group-id USDAWS_SG_ID --protocol tcp --port 9000 --source-group USDAWS_SG_ID --region USDAWS_REGION done
apiVersion: v1 kind: Secret metadata: name: peer-pods-secret namespace: openshift-sandboxed-containers-operator type: Opaque stringData: AWS_ACCESS_KEY_ID: "<aws_access_key>" 1 AWS_SECRET_ACCESS_KEY: "<aws_secret_access_key>" 2
oc apply -f peer-pods-secret.yaml
INSTANCE_ID=USD(oc get nodes -l 'node-role.kubernetes.io/worker' -o jsonpath='{.items[0].spec.providerID}' | sed 's#[^ ]*/##g')
AWS_REGION=USD(oc get infrastructure/cluster -o jsonpath='{.status.platformStatus.aws.region}') && echo "AWS_REGION: \"USDAWS_REGION\""
AWS_SUBNET_ID=USD(aws ec2 describe-instances --instance-ids USD{INSTANCE_ID} --query 'Reservations[*].Instances[*].SubnetId' --region USD{AWS_REGION} --output text) && echo "AWS_SUBNET_ID: \"USDAWS_SUBNET_ID\""
AWS_VPC_ID=USD(aws ec2 describe-instances --instance-ids USD{INSTANCE_ID} --query 'Reservations[*].Instances[*].VpcId' --region USD{AWS_REGION} --output text) && echo "AWS_VPC_ID: \"USDAWS_VPC_ID\""
AWS_SG_IDS=USD(aws ec2 describe-instances --instance-ids USD{INSTANCE_ID} --query 'Reservations[*].Instances[*].SecurityGroups[*].GroupId' --region USDAWS_REGION --output json | jq -r '.[][][]' | paste -sd ",") && echo "AWS_SG_IDS: \"USDAWS_SG_IDS\""
apiVersion: v1 kind: ConfigMap metadata: name: peer-pods-cm namespace: openshift-sandboxed-containers-operator data: CLOUD_PROVIDER: "aws" VXLAN_PORT: "9000" PODVM_INSTANCE_TYPE: "t3.medium" 1 PODVM_INSTANCE_TYPES: "t2.small,t2.medium,t3.large" 2 PROXY_TIMEOUT: "5m" PODVM_AMI_ID: "<podvm_ami_id>" 3 AWS_REGION: "<aws_region>" 4 AWS_SUBNET_ID: "<aws_subnet_id>" 5 AWS_VPC_ID: "<aws_vpc_id>" 6 AWS_SG_IDS: "<aws_sg_ids>" 7 DISABLECVM: "true"
oc apply -f peer-pods-cm.yaml
apiVersion: kataconfiguration.openshift.io/v1 kind: KataConfig metadata: name: example-kataconfig spec: enablePeerPods: true logLevel: info kataConfigPoolSelector: matchLabels: <label_key>: '<label_value>' 1
oc apply -f example-kataconfig.yaml
watch "oc describe kataconfig | sed -n /^Status:/,/^Events/p"
oc get -n openshift-sandboxed-containers-operator ds/peerpodconfig-ctrl-caa-daemon
oc get runtimeclass
NAME HANDLER AGE kata kata 152m kata-remote kata-remote 152m
oc get configmap peer-pods-cm -n openshift-sandboxed-containers-operator -o yaml
oc get events -n openshift-sandboxed-containers-operator --field-selector involvedObject.name=osc-podvm-image-creation
oc logs -n openshift-sandboxed-containers-operator jobs/osc-podvm-image-creation
apiVersion: v1 kind: <object> spec: runtimeClassName: kata-remote
apiVersion: v1 kind: <object> metadata: annotations: io.katacontainers.config.hypervisor.machine_type: "t3.medium" 1
apiVersion: v1 kind: <Pod> metadata: annotations: io.katacontainers.config.hypervisor.default_vcpus: <vcpus> io.katacontainers.config.hypervisor.default_memory: <memory>
oc apply -f <object.yaml>
apiVersion: confidentialcontainers.org/v1alpha1 kind: PeerPodConfig metadata: name: peerpodconfig-openshift namespace: openshift-sandboxed-containers-operator spec: cloudSecretName: peer-pods-secret configMapName: peer-pods-cm limit: "10" 1 nodeSelector: node-role.kubernetes.io/kata-oc: ""
AZURE_SUBSCRIPTION_ID=USD(az account list --query "[?isDefault].id" -o tsv) && echo "AZURE_SUBSCRIPTION_ID: \"USDAZURE_SUBSCRIPTION_ID\""
az ad sp create-for-rbac --role Contributor --scopes /subscriptions/USDAZURE_SUBSCRIPTION_ID --query "{ client_id: appId, client_secret: password, tenant_id: tenant }"
{ "client_id": `AZURE_CLIENT_ID`, "client_secret": `AZURE_CLIENT_SECRET`, "tenant_id": `AZURE_TENANT_ID` }
apiVersion: v1 kind: Secret metadata: name: peer-pods-secret namespace: openshift-sandboxed-containers-operator type: Opaque stringData: AZURE_CLIENT_ID: "<azure_client_id>" 1 AZURE_CLIENT_SECRET: "<azure_client_secret>" 2 AZURE_TENANT_ID: "<azure_tenant_id>" 3 AZURE_SUBSCRIPTION_ID: "<azure_subscription_id>" 4
AZURE_RESOURCE_GROUP=USD(oc get infrastructure/cluster -o jsonpath='{.status.platformStatus.azure.resourceGroupName}') && echo "AZURE_RESOURCE_GROUP: \"USDAZURE_RESOURCE_GROUP\""
AZURE_VNET_NAME=USD(az network vnet list --resource-group USD{AZURE_RESOURCE_GROUP} --query "[].{Name:name}" --output tsv)
AZURE_SUBNET_ID=USD(az network vnet subnet list --resource-group USD{AZURE_RESOURCE_GROUP} --vnet-name USDAZURE_VNET_NAME --query "[].{Id:id} | [? contains(Id, 'worker')]" --output tsv) && echo "AZURE_SUBNET_ID: \"USDAZURE_SUBNET_ID\""
AZURE_NSG_ID=USD(az network nsg list --resource-group USD{AZURE_RESOURCE_GROUP} --query "[].{Id:id}" --output tsv) && echo "AZURE_NSG_ID: \"USDAZURE_NSG_ID\""
AZURE_REGION=USD(az group show --resource-group USD{AZURE_RESOURCE_GROUP} --query "{Location:location}" --output tsv) && echo "AZURE_REGION: \"USDAZURE_REGION\""
apiVersion: v1 kind: ConfigMap metadata: name: peer-pods-cm namespace: openshift-sandboxed-containers-operator data: CLOUD_PROVIDER: "azure" VXLAN_PORT: "9000" AZURE_INSTANCE_SIZE: "Standard_B2als_v2" 1 AZURE_INSTANCE_SIZES: "Standard_B2als_v2,Standard_D2as_v5,Standard_D4as_v5,Standard_D2ads_v5" 2 AZURE_SUBNET_ID: "<azure_subnet_id>" 3 AZURE_NSG_ID: "<azure_nsg_id>" 4 PROXY_TIMEOUT: "5m" AZURE_IMAGE_ID: "<azure_image_id>" 5 AZURE_REGION: "<azure_region>" 6 AZURE_RESOURCE_GROUP: "<azure_resource_group>" 7 DISABLECVM: "true"
ssh-keygen -f ./id_rsa -N ""
shred --remove id_rsa.pub id_rsa
oc get events -n openshift-sandboxed-containers-operator --field-selector involvedObject.name=osc-podvm-image-creation
oc logs -n openshift-sandboxed-containers-operator jobs/osc-podvm-image-creation
apiVersion: v1 kind: <object> spec: runtimeClassName: kata-remote
apiVersion: v1 kind: <object> metadata: annotations: io.katacontainers.config.hypervisor.machine_type: "Standard_B2als_v2" 1
apiVersion: v1 kind: <Pod> metadata: annotations: io.katacontainers.config.hypervisor.default_vcpus: <vcpus> io.katacontainers.config.hypervisor.default_memory: <memory>
apiVersion: v1 kind: Namespace metadata: name: openshift-sandboxed-containers-operator
oc apply -f osc-namespace.yaml
apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: sandboxed-containers-operator-group namespace: openshift-sandboxed-containers-operator spec: targetNamespaces: - openshift-sandboxed-containers-operator
oc apply -f osc-operatorgroup.yaml
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: sandboxed-containers-operator namespace: openshift-sandboxed-containers-operator spec: channel: stable installPlanApproval: Automatic name: sandboxed-containers-operator source: redhat-operators sourceNamespace: openshift-marketplace startingCSV: sandboxed-containers-operator.v1.8.1
oc apply -f osc-subscription.yaml
oc get csv -n openshift-sandboxed-containers-operator
watch oc get csv -n openshift-sandboxed-containers-operator
NAME DISPLAY VERSION REPLACES PHASE openshift-sandboxed-containers openshift-sandboxed-containers-operator 1.8.1 1.7.0 Succeeded
oc get peerpodconfig peerpodconfig-openshift -n openshift-sandboxed-containers-operator -o jsonpath='{.spec.limit}{"\n"}'
oc patch peerpodconfig peerpodconfig-openshift -n openshift-sandboxed-containers-operator --type merge --patch '{"spec":{"limit":"<value>"}}' 1
AZURE_SUBSCRIPTION_ID=USD(az account list --query "[?isDefault].id" -o tsv) && echo "AZURE_SUBSCRIPTION_ID: \"USDAZURE_SUBSCRIPTION_ID\""
az ad sp create-for-rbac --role Contributor --scopes /subscriptions/USDAZURE_SUBSCRIPTION_ID --query "{ client_id: appId, client_secret: password, tenant_id: tenant }"
{ "client_id": `AZURE_CLIENT_ID`, "client_secret": `AZURE_CLIENT_SECRET`, "tenant_id": `AZURE_TENANT_ID` }
apiVersion: v1 kind: Secret metadata: name: peer-pods-secret namespace: openshift-sandboxed-containers-operator type: Opaque stringData: AZURE_CLIENT_ID: "<azure_client_id>" 1 AZURE_CLIENT_SECRET: "<azure_client_secret>" 2 AZURE_TENANT_ID: "<azure_tenant_id>" 3 AZURE_SUBSCRIPTION_ID: "<azure_subscription_id>" 4
oc apply -f peer-pods-secret.yaml
AZURE_RESOURCE_GROUP=USD(oc get infrastructure/cluster -o jsonpath='{.status.platformStatus.azure.resourceGroupName}') && echo "AZURE_RESOURCE_GROUP: \"USDAZURE_RESOURCE_GROUP\""
AZURE_VNET_NAME=USD(az network vnet list --resource-group USD{AZURE_RESOURCE_GROUP} --query "[].{Name:name}" --output tsv)
AZURE_SUBNET_ID=USD(az network vnet subnet list --resource-group USD{AZURE_RESOURCE_GROUP} --vnet-name USDAZURE_VNET_NAME --query "[].{Id:id} | [? contains(Id, 'worker')]" --output tsv) && echo "AZURE_SUBNET_ID: \"USDAZURE_SUBNET_ID\""
AZURE_NSG_ID=USD(az network nsg list --resource-group USD{AZURE_RESOURCE_GROUP} --query "[].{Id:id}" --output tsv) && echo "AZURE_NSG_ID: \"USDAZURE_NSG_ID\""
AZURE_REGION=USD(az group show --resource-group USD{AZURE_RESOURCE_GROUP} --query "{Location:location}" --output tsv) && echo "AZURE_REGION: \"USDAZURE_REGION\""
apiVersion: v1 kind: ConfigMap metadata: name: peer-pods-cm namespace: openshift-sandboxed-containers-operator data: CLOUD_PROVIDER: "azure" VXLAN_PORT: "9000" AZURE_INSTANCE_SIZE: "Standard_B2als_v2" 1 AZURE_INSTANCE_SIZES: "Standard_B2als_v2,Standard_D2as_v5,Standard_D4as_v5,Standard_D2ads_v5" 2 AZURE_SUBNET_ID: "<azure_subnet_id>" 3 AZURE_NSG_ID: "<azure_nsg_id>" 4 PROXY_TIMEOUT: "5m" AZURE_IMAGE_ID: "<azure_image_id>" 5 AZURE_REGION: "<azure_region>" 6 AZURE_RESOURCE_GROUP: "<azure_resource_group>" 7 DISABLECVM: "true"
oc apply -f peer-pods-cm.yaml
ssh-keygen -f ./id_rsa -N ""
oc create secret generic ssh-key-secret -n openshift-sandboxed-containers-operator --from-file=id_rsa.pub=./id_rsa.pub --from-file=id_rsa=./id_rsa
shred --remove id_rsa.pub id_rsa
apiVersion: kataconfiguration.openshift.io/v1 kind: KataConfig metadata: name: example-kataconfig spec: enablePeerPods: true logLevel: info kataConfigPoolSelector: matchLabels: <label_key>: '<label_value>' 1
oc apply -f example-kataconfig.yaml
watch "oc describe kataconfig | sed -n /^Status:/,/^Events/p"
oc get -n openshift-sandboxed-containers-operator ds/peerpodconfig-ctrl-caa-daemon
oc get runtimeclass
NAME HANDLER AGE kata kata 152m kata-remote kata-remote 152m
oc get configmap peer-pods-cm -n openshift-sandboxed-containers-operator -o yaml
oc get events -n openshift-sandboxed-containers-operator --field-selector involvedObject.name=osc-podvm-image-creation
oc logs -n openshift-sandboxed-containers-operator jobs/osc-podvm-image-creation
apiVersion: v1 kind: <object> spec: runtimeClassName: kata-remote
apiVersion: v1 kind: <object> metadata: annotations: io.katacontainers.config.hypervisor.machine_type: "Standard_B2als_v2" 1
apiVersion: v1 kind: <Pod> metadata: annotations: io.katacontainers.config.hypervisor.default_vcpus: <vcpus> io.katacontainers.config.hypervisor.default_memory: <memory>
oc apply -f <object.yaml>
apiVersion: v1 kind: Namespace metadata: name: trustee-operator-system
oc apply -f trustee-namespace.yaml
apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: trustee-operator-group namespace: trustee-operator-system spec: targetNamespaces: - trustee-operator-system
oc apply -f trustee-operatorgroup.yaml
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: trustee-operator namespace: trustee-operator-system spec: channel: stable installPlanApproval: Automatic name: trustee-operator source: redhat-operators sourceNamespace: openshift-marketplace startingCSV: trustee-operator.v0.1.0
oc apply -f trustee-subscription.yaml
oc get csv -n trustee-operator-system
watch oc get csv -n trustee-operator-system
NAME DISPLAY PHASE trustee-operator.v0.1.0 Trustee Operator 0.1.0 Succeeded
apiVersion: v1 kind: ConfigMap metadata: name: osc-feature-gates namespace: openshift-sandboxed-containers-operator data: confidential: "true"
oc apply -f cc-feature-gate.yaml
oc create route edge --service=kbs-service --port kbs-port -n trustee-operator-system
TRUSTEE_HOST=USD(oc get route -n trustee-operator-system kbs-service -o jsonpath={.spec.host})
echo USDTRUSTEE_HOST
kbs-service-trustee-operator-system.apps.memvjias.eastus.aroapp.io
AZURE_RESOURCE_GROUP=USD(oc get infrastructure/cluster -o jsonpath='{.status.platformStatus.azure.resourceGroupName}') && echo "AZURE_RESOURCE_GROUP: \"USDAZURE_RESOURCE_GROUP\""
AZURE_VNET_NAME=USD(az network vnet list --resource-group USD{AZURE_RESOURCE_GROUP} --query "[].{Name:name}" --output tsv)
AZURE_SUBNET_ID=USD(az network vnet subnet list --resource-group USD{AZURE_RESOURCE_GROUP} --vnet-name USDAZURE_VNET_NAME --query "[].{Id:id} | [? contains(Id, 'worker')]" --output tsv) && echo "AZURE_SUBNET_ID: \"USDAZURE_SUBNET_ID\""
AZURE_NSG_ID=USD(az network nsg list --resource-group USD{AZURE_RESOURCE_GROUP} --query "[].{Id:id}" --output tsv) && echo "AZURE_NSG_ID: \"USDAZURE_NSG_ID\""
AZURE_REGION=USD(az group show --resource-group USD{AZURE_RESOURCE_GROUP} --query "{Location:location}" --output tsv) && echo "AZURE_REGION: \"USDAZURE_REGION\""
apiVersion: v1 kind: ConfigMap metadata: name: peer-pods-cm namespace: openshift-sandboxed-containers-operator data: CLOUD_PROVIDER: "azure" VXLAN_PORT: "9000" AZURE_INSTANCE_SIZE: "Standard_DC2as_v5" 1 AZURE_INSTANCE_SIZES: "Standard_DC2as_v5, Standard_DC4as_v5, Standard_DC8as_v5" 2 AZURE_SUBNET_ID: "<azure_subnet_id>" 3 AZURE_NSG_ID: "<azure_nsg_id>" 4 PROXY_TIMEOUT: "5m" AZURE_IMAGE_ID: "<azure_image_id>" 5 AZURE_REGION: "<azure_region>" 6 AZURE_RESOURCE_GROUP: "<azure_resource_group>" 7 DISABLECVM: "false" AA_KBC_PARAMS: "cc_kbc::https://USD{TRUSTEE_HOST}" 8 ENABLE_SECURE_BOOT: "true" 9
oc apply -f peer-pods-cm.yaml
oc set env ds/peerpodconfig-ctrl-caa-daemon -n openshift-sandboxed-containers-operator REBOOT="USD(date)"
oc delete kataconfig example-kataconfig
oc get kataconfig example-kataconfig
No example-kataconfig instances exist
apiVersion: kataconfiguration.openshift.io/v1 kind: KataConfig metadata: name: example-kataconfig spec: enablePeerPods: true logLevel: info kataConfigPoolSelector: matchLabels: <label_key>: '<label_value>' 1
oc apply -f example-kataconfig.yaml
watch "oc describe kataconfig | sed -n /^Status:/,/^Events/p"
oc get -n openshift-sandboxed-containers-operator ds/peerpodconfig-ctrl-caa-daemon
oc get runtimeclass
NAME HANDLER AGE kata kata 152m kata-remote kata-remote 152m
openssl genpkey -algorithm ed25519 > privateKey
openssl pkey -in privateKey -pubout -out publicKey
oc create secret generic kbs-auth-public-key --from-file=publicKey -n trustee-operator-system
oc get secret -n trustee-operator-system
apiVersion: v1 kind: ConfigMap metadata: name: kbs-config-cm namespace: trustee-operator-system data: kbs-config.json: | { "insecure_http" : true, "sockets": ["0.0.0.0:8080"], "auth_public_key": "/etc/auth-secret/publicKey", "attestation_token_config": { "attestation_token_type": "CoCo" }, "repository_config": { "type": "LocalFs", "dir_path": "/opt/confidential-containers/kbs/repository" }, "as_config": { "work_dir": "/opt/confidential-containers/attestation-service", "policy_engine": "opa", "attestation_token_broker": "Simple", "attestation_token_config": { "duration_min": 5 }, "rvps_config": { "store_type": "LocalJson", "store_config": { "file_path": "/opt/confidential-containers/rvps/reference-values/reference-values.json" } } }, "policy_engine_config": { "policy_path": "/opt/confidential-containers/opa/policy.rego" } }
oc apply -f kbs-config-cm.yaml
apiVersion: v1 kind: ConfigMap metadata: name: rvps-reference-values namespace: trustee-operator-system data: reference-values.json: | [ 1 ]
oc apply -f rvps-configmap.yaml
apiVersion: v1 kind: ConfigMap metadata: name: attestation-policy namespace: trustee-operator-system data: default.rego: | package policy 1 import future.keywords.every default allow = false allow { every k, v in input { judge_field(k, v) } } judge_field(input_key, input_value) { has_key(data.reference, input_key) reference_value := data.reference[input_key] match_value(reference_value, input_value) } judge_field(input_key, input_value) { not has_key(data.reference, input_key) } match_value(reference_value, input_value) { not is_array(reference_value) input_value == reference_value } match_value(reference_value, input_value) { is_array(reference_value) array_include(reference_value, input_value) } array_include(reference_value_array, input_value) { reference_value_array == [] } array_include(reference_value_array, input_value) { reference_value_array != [] some i reference_value_array[i] == input_value } has_key(m, k) { _ = m[k] }
oc apply -f attestation-policy.yaml
apiVersion: v1 kind: ConfigMap metadata: name: tdx-config namespace: trustee-operator-system data: sgx_default_qcnl.conf: | { "collateral_service": "https://api.trustedservices.intel.com/sgx/certification/v4/", "pccs_url": "<pccs_url>" 1 }
oc apply -f tdx-config.yaml
oc apply secret generic kbsres1 --from-literal key1=<custom_key1> \ 1 --from-literal key2=<custom_key2> -n trustee-operator-system
oc apply secret generic <type> \ 1 --from-file=<tag>=./<public_key_file> \ 2 -n trustee-operator-system
{ "default": [ { "type": "insecureAcceptAnything" }], "transports": {} }
{ "default": [ { "type": "insecureAcceptAnything" } ], "transports": { "<transport>": { 1 "<registry>/<image>": 2 [ { "type": "sigstoreSigned", "keyPath": "kbs:///default/<type>/<tag>" 3 } ] } } }
oc apply secret generic security-policy --from-file=osc=./<security-policy-config.json> -n trustee-operator-system
apiVersion: v1 kind: ConfigMap metadata: name: resource-policy namespace: trustee-operator-system data: policy.rego: | 1 package policy 2 default allow = false allow { input["tee"] != "sample" }
oc apply -f resourcepolicy-configmap.yaml
apiVersion: confidentialcontainers.org/v1alpha1 kind: KbsConfig metadata: labels: app.kubernetes.io/name: kbsconfig app.kubernetes.io/instance: kbsconfig app.kubernetes.io/part-of: trustee-operator app.kubernetes.io/managed-by: kustomize app.kubernetes.io/created-by: trustee-operator name: kbsconfig namespace: trustee-operator-system spec: kbsConfigMapName: kbs-config-cm kbsAuthSecretName: kbs-auth-public-key kbsDeploymentType: AllInOneDeployment kbsRvpsRefValuesConfigMapName: rvps-reference-values kbsSecretResources: ["kbsres1", "security-policy", "<type>"] 1 kbsResourcePolicyConfigMapName: resource-policy tdxConfigSpec: kbsTdxConfigMapName: tdx-config 2 kbsAttestationPolicyConfigMapName: attestation-policy 3 kbsServiceType: <service_type> 4
oc apply -f kbsconfig-cr.yaml
oc project trustee-operator-system
oc get pods -n trustee-operator-system
NAME READY STATUS RESTARTS AGE trustee-deployment-8585f98449-9bbgl 1/1 Running 0 22m trustee-operator-controller-manager-5fbd44cd97-55dlh 2/2 Running 0 59m
POD_NAME=USD(oc get pods -l app=kbs -o jsonpath='{.items[0].metadata.name}' -n trustee-operator-system)
oc logs -n trustee-operator-system USDPOD_NAME
[2024-05-30T13:44:24Z INFO kbs] Using config file /etc/kbs-config/kbs-config.json [2024-05-30T13:44:24Z WARN attestation_service::rvps] No RVPS address provided and will launch a built-in rvps [2024-05-30T13:44:24Z INFO attestation_service::token::simple] No Token Signer key in config file, create an ephemeral key and without CA pubkey cert [2024-05-30T13:44:24Z INFO api_server] Starting HTTPS server at [0.0.0.0:8080] [2024-05-30T13:44:24Z INFO actix_server::builder] starting 12 workers [2024-05-30T13:44:24Z INFO actix_server::server] Tokio runtime found; starting in existing Tokio runtime
apiVersion: v1 kind: Pod metadata: name: ocp-cc-pod labels: app: ocp-cc-pod annotations: io.katacontainers.config.agent.policy: cGFja2FnZSBhZ2VudF9wb2xpY3kKCmRlZmF1bHQgQWRkQVJQTmVpZ2hib3JzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgQWRkU3dhcFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENsb3NlU3RkaW5SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBDb3B5RmlsZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IENyZWF0ZVNhbmRib3hSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBEZXN0cm95U2FuZGJveFJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEV4ZWNQcm9jZXNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgR2V0TWV0cmljc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IEdldE9PTUV2ZW50UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgR3Vlc3REZXRhaWxzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgTGlzdEludGVyZmFjZXNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBMaXN0Um91dGVzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgTWVtSG90cGx1Z0J5UHJvYmVSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBPbmxpbmVDUFVNZW1SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBQYXVzZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFB1bGxJbWFnZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFJlYWRTdHJlYW1SZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZW1vdmVDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZW1vdmVTdGFsZVZpcnRpb2ZzU2hhcmVNb3VudHNSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZXNlZWRSYW5kb21EZXZSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBSZXN1bWVDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTZXRHdWVzdERhdGVUaW1lUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2V0UG9saWN5UmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU2lnbmFsUHJvY2Vzc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFN0YXJ0Q29udGFpbmVyUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhcnRUcmFjaW5nUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgU3RhdHNDb250YWluZXJSZXF1ZXN0IDo9IHRydWUKZGVmYXVsdCBTdG9wVHJhY2luZ1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFR0eVdpblJlc2l6ZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUNvbnRhaW5lclJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUVwaGVtZXJhbE1vdW50c1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZUludGVyZmFjZVJlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFVwZGF0ZVJvdXRlc1JlcXVlc3QgOj0gdHJ1ZQpkZWZhdWx0IFdhaXRQcm9jZXNzUmVxdWVzdCA6PSB0cnVlCmRlZmF1bHQgV3JpdGVTdHJlYW1SZXF1ZXN0IDo9IHRydWUK 1 spec: runtimeClassName: kata-remote containers: - name: skr-openshift image: registry.access.redhat.com/ubi9/ubi:9.3 command: - sleep - "36000" securityContext: privileged: false seccompProfile: type: RuntimeDefault
oc create -f verification-pod.yaml
oc exec -it ocp-cc-pod -- bash
curl http://127.0.0.1:8006/cdh/resource/default/kbsres1/key1
res1val1
apiVersion: confidentialcontainers.org/v1alpha1 kind: PeerPodConfig metadata: name: peerpodconfig-openshift namespace: openshift-sandboxed-containers-operator spec: cloudSecretName: peer-pods-secret configMapName: peer-pods-cm limit: "10" 1 nodeSelector: node-role.kubernetes.io/kata-oc: ""
apiVersion: v1 kind: Namespace metadata: name: openshift-sandboxed-containers-operator
oc apply -f osc-namespace.yaml
apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: sandboxed-containers-operator-group namespace: openshift-sandboxed-containers-operator spec: targetNamespaces: - openshift-sandboxed-containers-operator
oc apply -f osc-operatorgroup.yaml
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: sandboxed-containers-operator namespace: openshift-sandboxed-containers-operator spec: channel: stable installPlanApproval: Automatic name: sandboxed-containers-operator source: redhat-operators sourceNamespace: openshift-marketplace startingCSV: sandboxed-containers-operator.v1.8.1
oc apply -f osc-subscription.yaml
oc get csv -n openshift-sandboxed-containers-operator
watch oc get csv -n openshift-sandboxed-containers-operator
NAME DISPLAY VERSION REPLACES PHASE openshift-sandboxed-containers openshift-sandboxed-containers-operator 1.8.1 1.7.0 Succeeded
oc get peerpodconfig peerpodconfig-openshift -n openshift-sandboxed-containers-operator -o jsonpath='{.spec.limit}{"\n"}'
oc patch peerpodconfig peerpodconfig-openshift -n openshift-sandboxed-containers-operator --type merge --patch '{"spec":{"limit":"<value>"}}' 1
export LIBVIRT_POOL=<libvirt_pool>
export LIBVIRT_VOL_NAME=<libvirt_volume>
export LIBVIRT_POOL_DIRECTORY=<target_directory> 1
virsh pool-define-as USDLIBVIRT_POOL --type dir --target "USDLIBVIRT_POOL_DIRECTORY"
virsh pool-start USDLIBVIRT_POOL
virsh -c qemu:///system vol-create-as --pool USDLIBVIRT_POOL --name USDLIBVIRT_VOL_NAME --capacity 20G --allocation 2G --prealloc-metadata --format qcow2
FROM scratch ARG PODVM_IMAGE_SRC ENV PODVM_IMAGE_PATH="/image/podvm.qcow2" COPY USDPODVM_IMAGE_SRC USDPODVM_IMAGE_PATH
docker build -t podvm-libvirt --build-arg PODVM_IMAGE_SRC=<podvm_image_source> \ 1 --build-arg PODVM_IMAGE_PATH=<podvm_image_path> \ 2 -f Dockerfile.podvm-oci .
virtint=USD(bridge_line=USD(virsh net-info default | grep Bridge); echo "USD{bridge_line//Bridge:/}" | tr -d [:blank:]) LIBVIRT_URI=USD( ip -4 addr show USDvirtint | grep -oP '(?<=inet\s)\d+(\.\d+){3}') LIBVIRT_GATEWAY_URI="qemu+ssh://root@USD{LIBVIRT_URI}/system?no_verify=1"
apiVersion: v1 kind: Secret metadata: name: peer-pods-secret namespace: openshift-sandboxed-containers-operator type: Opaque stringData: CLOUD_PROVIDER: "libvirt" LIBVIRT_URI: "<libvirt_gateway_uri>" 1 LIBVIRT_POOL: "<libvirt_pool>" 2 LIBVIRT_VOL_NAME: "<libvirt_volume>" 3 REDHAT_OFFLINE_TOKEN: "<rh_offline_token>" 4
oc apply -f peer-pods-secret.yaml
apiVersion: v1 kind: ConfigMap metadata: name: peer-pods-cm namespace: openshift-sandboxed-containers-operator data: CLOUD_PROVIDER: "libvirt" DISABLECVM: "true"
oc apply -f peer-pods-cm.yaml
apiVersion: v1 kind: ConfigMap metadata: name: libvirt-podvm-image-cm namespace: openshift-sandboxed-containers-operator data: PODVM_DISTRO: "rhel" CAA_SRC: "https://github.com/confidential-containers/cloud-api-adaptor" CAA_REF: "<cloud_api_adaptor_version>" 1 DOWNLOAD_SOURCES: "no" CONFIDENTIAL_COMPUTE_ENABLED: "yes" UPDATE_PEERPODS_CM: "yes" ORG_ID: "<rhel_organization_id>" ACTIVATION_KEY: "<rhel_activation_key>" 2 IMAGE_NAME: "<podvm_libvirt_image>" PODVM_IMAGE_URI: "oci::<image_repo_url>:<image_tag>::<image_path>" 3 SE_BOOT: "true" 4 BASE_OS_VERSION: "<rhel_image_os_version>" 5
oc apply -f libvirt-podvm-image-cm.yaml
ssh-keygen -f ./id_rsa -N ""
ssh-copy-id -i ./id_rsa.pub <KVM_HOST_IP>
oc create secret generic ssh-key-secret -n openshift-sandboxed-containers-operator --from-file=id_rsa.pub=./id_rsa.pub --from-file=id_rsa=./id_rsa
shred --remove id_rsa.pub id_rsa
apiVersion: kataconfiguration.openshift.io/v1 kind: KataConfig metadata: name: example-kataconfig spec: enablePeerPods: true logLevel: info kataConfigPoolSelector: matchLabels: <label_key>: '<label_value>' 1
oc apply -f example-kataconfig.yaml
watch "oc describe kataconfig | sed -n /^Status:/,/^Events/p"
oc describe configmap peer-pods-cm -n openshift-sandboxed-containers-operator
Name: peer-pods-cm Namespace: openshift-sandboxed-containers-operator Labels: <none> Annotations: <none> Data ==== CLOUD_PROVIDER: libvirt BinaryData ==== Events: <none>
watch oc get mcp/kata-oc
oc get -n openshift-sandboxed-containers-operator ds/peerpodconfig-ctrl-caa-daemon
oc get runtimeclass
NAME HANDLER AGE kata kata 152m kata-remote kata-remote 152m
apiVersion: v1 kind: <object> spec: runtimeClassName: kata-remote
apiVersion: v1 kind: Namespace metadata: name: trustee-operator-system
oc apply -f trustee-namespace.yaml
apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: trustee-operator-group namespace: trustee-operator-system spec: targetNamespaces: - trustee-operator-system
oc apply -f trustee-operatorgroup.yaml
apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: trustee-operator namespace: trustee-operator-system spec: channel: stable installPlanApproval: Automatic name: trustee-operator source: redhat-operators sourceNamespace: openshift-marketplace startingCSV: trustee-operator.v0.1.0
oc apply -f trustee-subscription.yaml
oc get csv -n trustee-operator-system
watch oc get csv -n trustee-operator-system
NAME DISPLAY PHASE trustee-operator.v0.1.0 Trustee Operator 0.1.0 Succeeded
apiVersion: v1 kind: ConfigMap metadata: name: osc-feature-gates namespace: openshift-sandboxed-containers-operator data: confidential: "true"
oc apply -f cc-feature-gate.yaml
oc create route edge --service=kbs-service --port kbs-port -n trustee-operator-system
TRUSTEE_HOST=USD(oc get route -n trustee-operator-system kbs-service -o jsonpath={.spec.host})
echo USDTRUSTEE_HOST
kbs-service-trustee-operator-system.apps.memvjias.eastus.aroapp.io
apiVersion: v1 kind: ConfigMap metadata: name: peer-pods-cm namespace: openshift-sandboxed-containers-operator data: CLOUD_PROVIDER: "libvirt" DISABLECVM: "false" AA_KBC_PARAMS: "cc_kbc::https://USD{TRUSTEE_HOST}" 1
oc apply -f peer-pods-cm.yaml
oc set env ds/peerpodconfig-ctrl-caa-daemon -n openshift-sandboxed-containers-operator REBOOT="USD(date)"
oc delete kataconfig example-kataconfig
oc get kataconfig example-kataconfig
No example-kataconfig instances exist
apiVersion: v1 kind: Secret metadata: name: peer-pods-secret namespace: openshift-sandboxed-containers-operator type: Opaque stringData: REDHAT_OFFLINE_TOKEN: "<rh_offline_token>" 1 HKD_CRT: "<hkd_crt_value>" 2
oc apply -f peer-pods-secret.yaml
apiVersion: kataconfiguration.openshift.io/v1 kind: KataConfig metadata: name: example-kataconfig spec: enablePeerPods: true logLevel: info kataConfigPoolSelector: matchLabels: <label_key>: '<label_value>' 1
oc apply -f example-kataconfig.yaml
watch "oc describe kataconfig | sed -n /^Status:/,/^Events/p"
oc describe configmap peer-pods-cm -n openshift-sandboxed-containers-operator
Name: peer-pods-cm Namespace: openshift-sandboxed-containers-operator Labels: <none> Annotations: <none> Data ==== CLOUD_PROVIDER: libvirt DISABLECVM: false 1 LIBVIRT_IMAGE_ID: fa-pp-vol 2 BinaryData ==== Events: <none>
watch oc get mcp/kata-oc
oc get -n openshift-sandboxed-containers-operator ds/peerpodconfig-ctrl-caa-daemon
oc get runtimeclass
NAME HANDLER AGE kata kata 152m kata-remote kata-remote 152m
openssl genpkey -algorithm ed25519 > privateKey
openssl pkey -in privateKey -pubout -out publicKey
oc create secret generic kbs-auth-public-key --from-file=publicKey -n trustee-operator-system
oc get secret -n trustee-operator-system
apiVersion: v1 kind: ConfigMap metadata: name: kbs-config-cm namespace: trustee-operator-system data: kbs-config.json: | { "insecure_http" : true, "sockets": ["0.0.0.0:8080"], "auth_public_key": "/etc/auth-secret/publicKey", "attestation_token_config": { "attestation_token_type": "CoCo" }, "repository_config": { "type": "LocalFs", "dir_path": "/opt/confidential-containers/kbs/repository" }, "as_config": { "work_dir": "/opt/confidential-containers/attestation-service", "policy_engine": "opa", "attestation_token_broker": "Simple", "attestation_token_config": { "duration_min": 5 }, "rvps_config": { "store_type": "LocalJson", "store_config": { "file_path": "/opt/confidential-containers/rvps/reference-values/reference-values.json" } } }, "policy_engine_config": { "policy_path": "/opt/confidential-containers/opa/policy.rego" } }
oc apply -f kbs-config-cm.yaml
mkdir -p Rvps-Extraction/
wget https://github.com/openshift/sandboxed-containers-operator/raw/devel/scripts/rvps-extraction/GetRvps.sh -O USDPWD/GetRvps.sh
mkdir -p Rvps-Extraction/static-files
cd Rvps-Extraction/static-files
wget https://github.com/openshift/sandboxed-containers-operator/raw/devel/scripts/rvps-extraction/static-files/pvextract-hdr -O USDPWD/pvextract-hdr
chmod +x pvextract-hdr
wget https://github.com/openshift/sandboxed-containers-operator/raw/devel/scripts/rvps-extraction/static-files/se_parse_hdr.py -O USDPWD/se_parse_hdr.py
cp ~/path/to/<hkd_cert.crt> .
cd ..
chmod +x GetRvps.sh
./GetRvps.sh
***Installing necessary packages for RVPS values extraction *** Updating Subscription Management repositories. Last metadata expiration check: 0:37:12 ago on Mon Nov 18 09:20:29 2024. Package python3-3.9.19-8.el9_5.1.s390x is already installed. Package python3-cryptography-36.0.1-4.el9.s390x is already installed. Package kmod-28-10.el9.s390x is already installed. Dependencies resolved. Nothing to do. Complete! ***Installation Finished *** 1) Generate the RVPS From Local Image from User pc 2) Generate RVPS from Volume 3) Quit Please enter your choice:
Please enter your choice: 2
Enter the Libvirt Pool Name: fa-pp
Enter the Libvirt URI Name: <libvirt-uri> 1
Enter the Libvirt Volume Name: fa-pp-vol
Downloading from PODVM Volume mount: /mnt/myvm: special device /dev/nbd3p1 does not exist. Error: Failed to mount the image. Retrying Mounting on second attempt passed /dev/nbd3 disconnected SE header found at offset 0x014000 SE header written to '/root/Rvps-Extraction/output-files/hdr.bin' (640 bytes) se.tag: 42f3fe61e8a7e859cab3bb033fd11c61 se.image_phkh: 92d0aff6eb86719b6b1ea0cb98d2c99ff2ec693df3efff2158f54112f6961508 provenance = ewogICAgInNlLmF0dGVzdGF0aW9uX3Boa2giOiBbCiAgICAgICAgIjkyZDBhZmY2ZWI4NjcxOWI2YjFlYTBjYjk4ZDJjOTlmZjJlYzY5M2RmM2VmZmYyMTU4ZjU0MTEyZjY5NjE1MDgiCiAgICBdLAogICAgInNlLnRhZyI6IFsKICAgICAgICAiNDJmM2ZlNjFlOGE3ZTg1OWNhYjNiYjAzM2ZkMTFjNjEiCiAgICBdLAogICAgInNlLmltYWdlX3Boa2giOiBbCiAgICAgICAgIjkyZDBhZmY2ZWI4NjcxOWI2YjFlYTBjYjk4ZDJjOTlmZjJlYzY5M2RmM2VmZmYyMTU4ZjU0MTEyZjY5NjE1MDgiCiAgICBdLAogICAgInNlLnVzZXJfZGF0YSI6IFsKICAgICAgICAiMDAiCiAgICBdLAogICAgInNlLnZlcnNpb24iOiBbCiAgICAgICAgIjI1NiIKICAgIF0KfQo= -rw-r--r--. 1 root root 640 Dec 16 10:57 /root/Rvps-Extraction/output-files/hdr.bin -rw-r--r--. 1 root root 446 Dec 16 10:57 /root/Rvps-Extraction/output-files/ibmse-policy.rego -rw-r--r--. 1 root root 561 Dec 16 10:57 /root/Rvps-Extraction/output-files/se-message
mkdir /tmp/ibmse/certs
wget https://www.ibm.com/support/resourcelink/api/content/public/ibm-z-host-key-signing-gen2.crt -O /tmp/ibmse/certs/ibm-z-host-key-signing-gen2.crt
wget https://www.ibm.com/support/resourcelink/api/content/public/DigiCertCA.crt -O /tmp/ibmse/certs/DigiCertCA.crt
mkdir /tmp/ibmse/crls
wget https://www.ibm.com/support/resourcelink/api/content/public/ibm-z-host-key-gen2.crl -O /tmp/ibmse/crls/ibm-z-host-key-gen2.crl
wget http://crl3.digicert.com/DigiCertTrustedRootG4.crl -O /tmp/ibmse/crls/DigiCertTrustedRootG4.crl
wget http://crl3.digicert.com/DigiCertTrustedG4CodeSigningRSA4096SHA3842021CA1.crl -O /tmp/ibmse/crls/DigiCertTrustedG4CodeSigningRSA4096SHA3842021CA1.crl
mkdir -p /tmp/ibmse/hdr/
cp /root/Rvps-Extraction/output-files/hdr.bin /tmp/ibmse/hdr/
mkdir -p /tmp/ibmse/hkds
cp ~/path/to/<hkd_cert.crt> /tmp/ibmse/hkds/
openssl genrsa -aes256 -passout pass:<password> -out /tmp/encrypt_key-psw.pem 4096 1
mkdir -p /tmp/ibmse/rsa
openssl rsa -in /tmp/encrypt_key-psw.pem -passin pass:<password> -pubout -out /tmp/ibmse/rsa/encrypt_key.pub
openssl rsa -in /tmp/encrypt_key-psw.pem -passin pass:<password> -out /tmp/ibmse/rsa/encrypt_key.pem
tree /tmp/ibmse
/tmp/ibmse ├── certs │ ├── ibm-z-host-key-signing-gen2.crt | └── DigiCertCA.crt ├── crls │ └── ibm-z-host-key-gen2.crl │ └── DigiCertTrustedRootG4.crl │ └── DigiCertTrustedG4CodeSigningRSA4096SHA3842021CA1.crl ├── hdr │ └── hdr.bin ├── hkds │ └── <hkd_cert.crt> └── rsa ├── encrypt_key.pem └── encrypt_key.pub
tar -czf ibmse.tar.gz -C /tmp/ ibmse
scp /tmp/ibmse.tar.gz root@<ocp_bastion_ip>:/tmp 1
ssh root@<ocp_bastion_ip>
scp /tmp/ibmse.tar.gz core@<worker_node_ip>:/tmp 1
ssh core@<worker_node_ip> 'sudo mkdir -p /opt/confidential-containers/ && sudo tar -xzf /tmp/ibmse.tar.gz -C /opt/confidential-containers/'
ssh core@<worker_node_ip> 'sudo chmod -R 755 /opt/confidential-containers/ibmse/'
apiVersion: v1 kind: PersistentVolume metadata: name: ibmse-pv namespace: trustee-operator-system spec: capacity: storage: 100Mi accessModes: - ReadOnlyMany storageClassName: "" local: path: /opt/confidential-containers/ibmse nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: node-role.kubernetes.io/worker operator: Exists
oc apply -f persistent-volume.yaml
apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ibmse-pvc namespace: trustee-operator-system spec: accessModes: - ReadOnlyMany storageClassName: "" resources: requests: storage: 100Mi
oc apply -f persistent-volume-claim.yaml
apiVersion: v1 kind: ConfigMap metadata: name: rvps-reference-values namespace: trustee-operator-system data: reference-values.json: | [ 1 ]
oc apply -f rvps-configmap.yaml
apiVersion: v1 kind: ConfigMap metadata: name: attestation-policy namespace: trustee-operator-system data: default.rego: | 1 package policy import rego.v1 default allow = false converted_version := sprintf("%v", [input["se.version"]]) allow if { input["se.attestation_phkh"] == "<se.attestation_phkh>" 2 input["se.image_phkh"] == "<se.image_phkh>" input["se.tag"] == "<se.tag>" converted_version == "256" }
oc apply -f attestation-policy.yaml
oc apply secret generic kbsres1 --from-literal key1=<custom_key1> \ 1 --from-literal key2=<custom_key2> -n trustee-operator-system
oc apply secret generic <type> \ 1 --from-file=<tag>=./<public_key_file> \ 2 -n trustee-operator-system
{ "default": [ { "type": "insecureAcceptAnything" }], "transports": {} }
{ "default": [ { "type": "insecureAcceptAnything" } ], "transports": { "<transport>": { 1 "<registry>/<image>": 2 [ { "type": "sigstoreSigned", "keyPath": "kbs:///default/<type>/<tag>" 3 } ] } } }
oc apply secret generic security-policy --from-file=osc=./<security-policy-config.json> -n trustee-operator-system
apiVersion: v1 kind: ConfigMap metadata: name: resource-policy namespace: trustee-operator-system data: policy.rego: | 1 package policy 2 path := split(data["resource-path"], "/") default allow = false allow { count(path) == 3 input["tee"] == "se" }
oc apply -f resourcepolicy-configmap.yaml
apiVersion: confidentialcontainers.org/v1alpha1 kind: KbsConfig metadata: labels: app.kubernetes.io/name: kbsconfig app.kubernetes.io/instance: kbsconfig app.kubernetes.io/part-of: trustee-operator app.kubernetes.io/managed-by: kustomize app.kubernetes.io/created-by: trustee-operator name: kbsconfig namespace: trustee-operator-system spec: kbsConfigMapName: kbs-config-cm kbsAuthSecretName: kbs-auth-public-key kbsDeploymentType: AllInOneDeployment kbsRvpsRefValuesConfigMapName: rvps-reference-values kbsSecretResources: ["kbsres1", "security-policy", "<type>"] 1 kbsResourcePolicyConfigMapName: resource-policy kbsAttestationPolicyConfigMapName: attestation-policy kbsServiceType: NodePort ibmSEConfigSpec: certStorePvc: ibmse-pvc KbsEnvVars: SE_SKIP_CERTS_VERIFICATION: "true"
oc apply -f kbsconfig-cr.yaml
oc project trustee-operator-system
oc get pods -n trustee-operator-system
NAME READY STATUS RESTARTS AGE trustee-deployment-8585f98449-9bbgl 1/1 Running 0 22m trustee-operator-controller-manager-5fbd44cd97-55dlh 2/2 Running 0 59m
POD_NAME=USD(oc get pods -l app=kbs -o jsonpath='{.items[0].metadata.name}' -n trustee-operator-system)
oc logs -n trustee-operator-system USDPOD_NAME
[2024-05-30T13:44:24Z INFO kbs] Using config file /etc/kbs-config/kbs-config.json [2024-05-30T13:44:24Z WARN attestation_service::rvps] No RVPS address provided and will launch a built-in rvps [2024-05-30T13:44:24Z INFO attestation_service::token::simple] No Token Signer key in config file, create an ephemeral key and without CA pubkey cert [2024-05-30T13:44:24Z INFO api_server] Starting HTTPS server at [0.0.0.0:8080] [2024-05-30T13:44:24Z INFO actix_server::builder] starting 12 workers [2024-05-30T13:44:24Z INFO actix_server::server] Tokio runtime found; starting in existing Tokio runtime
oc get svc kbs-service -n trustee-operator-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE kbs-service NodePort 198.51.100.54 <none> 8080:31862/TCP 23h
oc get pods -n trustee-operator-system | grep -i trustee-deployment
NAME READY STATUS RESTARTS AGE trustee-deployment-d746679cd-plq82 1/1 Running 0 2m32s
oc get pod trustee-deployment-d746679cd-plq82 -o custom-columns="NODE-IP:.status.hostIP"
NODE-IP 192.168.122.36
oc get cm peer-pods-cm -n openshift-sandboxed-containers-operator -o yaml | grep AA_KBC_PARAMS
AA_KBC_PARAMS: cc_kbc::http://192.168.122.36:31862
apiVersion: v1 kind: Pod metadata: name: busybox namespace: default spec: runtimeClassName: kata-remote containers: - name: busybox image: quay.io/prometheus/busybox:latest imagePullPolicy: Always command: - "sleep" - "3600"
oc create -f busybox.yaml
oc exec -it busybox -n default -- /bin/sh
wget http://127.0.0.1:8006/cdh/resource/default/kbsres1/key1
Connecting to 127.0.0.1:8006 (127.0.0.1:8006) saving to 'key1' key1 100% |*******************************************| 8 0:00:00 ETA 'key1' saved
cat key1
res1val1/ #
oc get pods -A -o json | jq -r '.items[] | select(.spec.runtimeClassName == "<runtime>").metadata.name' 1
oc delete pod <pod>
oc delete kataconfig example-kataconfig
oc get kataconfig example-kataconfig
No example-kataconfig instances exist
oc delete subscription sandboxed-containers-operator -n openshift-sandboxed-containers-operator
oc delete namespace openshift-sandboxed-containers-operator
oc delete crd kataconfigs.kataconfiguration.openshift.io
oc get crd kataconfigs.kataconfiguration.openshift.io
Unknown CRD kataconfigs.kataconfiguration.openshift.io
oc delete kbsconfig kbsconfig
oc get kbsconfig kbsconfig
No kbsconfig instances exist
oc delete subscription trustee-operator -n trustee-operator-system
oc delete namespace trustee-operator-system
oc delete crd kbsconfigs.confidentialcontainers.org
oc get crd kbsconfigs.confidentialcontainers.org
Unknown CRD kbsconfigs.confidentialcontainers.org
oc delete kataconfig example-kataconfig
oc get kataconfig example-kataconfig
No example-kataconfig instances exist
oc get cm -n openshift-sandboxed-containers-operator peer-pods-cm -o jsonpath="{.data.AZURE_IMAGE_ID}"
oc patch configmap peer-pods-cm -n openshift-sandboxed-containers-operator -p '{"data":{"AZURE_IMAGE_ID":""}}'
apiVersion: kataconfiguration.openshift.io/v1 kind: KataConfig metadata: name: example-kataconfig spec: enablePeerPods: true logLevel: info kataConfigPoolSelector: matchLabels: <label_key>: '<label_value>' 1
oc apply -f example-kataconfig.yaml
watch "oc describe kataconfig | sed -n /^Status:/,/^Events/p"
oc get -n openshift-sandboxed-containers-operator ds/peerpodconfig-ctrl-caa-daemon
oc get runtimeclass
NAME HANDLER AGE kata kata 152m kata-remote kata-remote 152m
oc adm must-gather --image=registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:1.8.1
oc adm must-gather -- /usr/bin/gather_audit_logs
NAMESPACE NAME READY STATUS RESTARTS AGE openshift-must-gather-5drcj must-gather-bklx4 2/2 Running 0 72s openshift-must-gather-5drcj must-gather-s8sdh 2/2 Running 0 72s
oc adm must-gather --run-namespace <namespace> --image=registry.redhat.io/openshift-sandboxed-containers/osc-must-gather-rhel9:1.8.1
oc patch kataconfig <kataconfig> --type merge --patch '{"spec":{"logLevel":"debug"}}'
oc get mcp kata-oc
NAME CONFIG UPDATED UPDATING DEGRADED MACHINECOUNT READYMACHINECOUNT UPDATEDMACHINECOUNT DEGRADEDMACHINECOUNT AGE kata-oc rendered-kata-oc-169 False True False 3 1 1 0 9h
oc debug node/<node_name>
chroot /host
crio config | egrep 'log_level
log_level = "debug"
Mar 11 11:57:28 openshift-worker-0 kata[2241647]: time="2023-03-11T11:57:28.587116986Z" level=info msg="Start logging QEMU (qemuPid=2241693)" name=containerd-shim-v2 pid=2241647 sandbox=d1d4d68efc35e5ccb4331af73da459c13f46269b512774aa6bde7da34db48987 source=virtcontainers/hypervisor subsystem=qemu Mar 11 11:57:28 openshift-worker-0 kata[2241647]: time="2023-03-11T11:57:28.607339014Z" level=error msg="qemu-kvm: -machine q35,accel=kvm,kernel_irqchip=split,foo: Expected '=' after parameter 'foo'" name=containerd-shim-v2 pid=2241647 qemuPid=2241693 sandbox=d1d4d68efc35e5ccb4331af73da459c13f46269b512774aa6bde7da34db48987 source=virtcontainers/hypervisor subsystem=qemu Mar 11 11:57:28 openshift-worker-0 kata[2241647]: time="2023-03-11T11:57:28.60890737Z" level=info msg="Stop logging QEMU (qemuPid=2241693)" name=containerd-shim-v2 pid=2241647 sandbox=d1d4d68efc35e5ccb4331af73da459c13f46269b512774aa6bde7da34db48987 source=virtcontainers/hypervisor subsystem=qemu
oc debug node/<nodename> -- journalctl -D /host/var/log/journal -t kata -g "reading guest console"
oc debug node/<nodename> -- journalctl -D /host/var/log/journal -t kata
oc debug node/<nodename> -- journalctl -D /host/var/log/journal -t virtiofsd
oc debug node/<nodename> -- journalctl -D /host/var/log/journal -t kata -g "qemuPid=\d+"
conditions: message: Performing initial installation of kata-remote on cluster reason: Installing status: 'True' type: InProgress kataNodes: nodeCount: 0 readyNodeCount: 0
kataNodes: nodeCount: 2 readyNodeCount: 0 waitingToInstall: - worker-0 - worker-1
kataNodes: installing: - worker-1 nodeCount: 2 readyNodeCount: 0 waitingToInstall: - worker-0
kataNodes: installed: - worker-1 installing: - worker-0 nodeCount: 2 readyNodeCount: 1
conditions: message: "" reason: "" status: 'False' type: InProgress kataNodes: installed: - worker-0 - worker-1 nodeCount: 2 readyNodeCount: 2
conditions: message: Removing kata-remote from cluster reason: Uninstalling status: 'True' type: InProgress kataNodes: nodeCount: 0 readyNodeCount: 0 waitingToUninstall: - worker-0 - worker-1
kataNodes: nodeCount: 0 readyNodeCount: 0 uninstalling: - worker-1 waitingToUninstall: - worker-0
kataNodes: nodeCount: 0 readyNodeCount: 0 uninstalling: - worker-0