Tutorials Red Hat OpenShift Service on AWS 4 Red Hat OpenShift Service on AWS tutorials Red Hat OpenShift Documentation Team

Commands:
mkdir scratch cd scratch cat << 'EOF' > verify-permissions.sh #!/bin/bash while getopts 'p:' OPTION; do case "USDOPTION" in p) PREFIX="USDOPTARG" ;; ?) echo "script usage: USD(basename \USD0) [-p PREFIX]" >&2 exit 1 ;; esac done shift "USD((USDOPTIND -1))" rosa create account-roles --mode manual --prefix USDPREFIX INSTALLER_POLICY=USD(cat sts_installer_permission_policy.json | jq ) CONTROL_PLANE_POLICY=USD(cat sts_instance_controlplane_permission_policy.json | jq) WORKER_POLICY=USD(cat sts_instance_worker_permission_policy.json | jq) SUPPORT_POLICY=USD(cat sts_support_permission_policy.json | jq) simulatePolicy () { outputFile="USD{2}.results" echo USD2 aws iam simulate-custom-policy --policy-input-list "USD1" --action-names USD(jq '.Statement | map(select(.Effect == "Allow"))[].Action | if type == "string" then . else .[] end' "USD2" -r) --output text > USDoutputFile } simulatePolicy "USDINSTALLER_POLICY" "sts_installer_permission_policy.json" simulatePolicy "USDCONTROL_PLANE_POLICY" "sts_instance_controlplane_permission_policy.json" simulatePolicy "USDWORKER_POLICY" "sts_instance_worker_permission_policy.json" simulatePolicy "USDSUPPORT_POLICY" "sts_support_permission_policy.json" EOF chmod +x verify-permissions.sh ./verify-permissions.sh -p SimPolTest
for file in USD(ls *.results); do echo USDfile; cat USDfile; done
sts_installer_permission_policy.json.results EVALUATIONRESULTS autoscaling:DescribeAutoScalingGroups allowed * MATCHEDSTATEMENTS PolicyInputList.1 IAM Policy ENDPOSITION 6 195 STARTPOSITION 17 3 EVALUATIONRESULTS ec2:AllocateAddress allowed * MATCHEDSTATEMENTS PolicyInputList.1 IAM Policy ENDPOSITION 6 195 STARTPOSITION 17 3 EVALUATIONRESULTS ec2:AssociateAddress allowed * MATCHEDSTATEMENTS PolicyInputList.1 IAM Policy
export VPC_ID=<vpc_ID> 1 export REGION=<region> 2 export VPC_CIDR=<vpc_CIDR> 3
echo "VPC ID: USD{VPC_ID}, VPC CIDR Range: USD{VPC_CIDR}, Region: USD{REGION}"
SG_ID=USD(aws ec2 create-security-group --group-name rosa-inbound-resolver --description "Security group for ROSA inbound resolver" --vpc-id USD{VPC_ID} --region USD{REGION} --output text) aws ec2 authorize-security-group-ingress --group-id USD{SG_ID} --protocol tcp --port 53 --cidr USD{VPC_CIDR} --region USD{REGION} aws ec2 authorize-security-group-ingress --group-id USD{SG_ID} --protocol udp --port 53 --cidr USD{VPC_CIDR} --region USD{REGION}
RESOLVER_ID=USD(aws route53resolver create-resolver-endpoint --name rosa-inbound-resolver --creator-request-id rosa-USD(date '+%Y-%m-%d') --security-group-ids USD{SG_ID} --direction INBOUND --ip-addresses USD(aws ec2 describe-subnets --filter Name=vpc-id,Values=USD{VPC_ID} --region USD{REGION} | jq -jr '.Subnets | map("SubnetId=\(.SubnetId) ") | .[]') --region USD{REGION} --output text --query 'ResolverEndpoint.Id')
RESOLVER_ID=USD(aws route53resolver create-resolver-endpoint --name rosa-inbound-resolver --creator-request-id rosa-USD(date '+%Y-%m-%d') --security-group-ids USD{SG_ID} --direction INBOUND --ip-addresses SubnetId=<subnet_ID>,Ip=<endpoint_IP> SubnetId=<subnet_ID>,Ip=<endpoint_IP> \ 1 --region USD{REGION} --output text --query 'ResolverEndpoint.Id')
aws route53resolver list-resolver-endpoint-ip-addresses --resolver-endpoint-id USD{RESOLVER_ID} --region=USD{REGION} --query 'IpAddresses[*].Ip'
10.0.45.253, 10.0.23.131, 10.0.148.159
aws route53 list-hosted-zones-by-vpc --vpc-id USD{VPC_ID} --vpc-region USD{REGION} --query 'HostedZoneSummaries[*].Name' --output table
---------------------------------------------- | ListHostedZonesByVPC | +--------------------------------------------+ | domain-prefix.agls.p3.openshiftapps.com. | +--------------------------------------------+
zone "<domain-prefix>.<unique-ID>.p1.openshiftapps.com" { 1 type forward; forward only; forwarders { 2 10.0.45.253; 10.0.23.131; 10.0.148.159; }; };
export DOMAIN=apps.example.com 1 export AWS_PAGER="" export CLUSTER_NAME=USD(oc get infrastructure cluster -o=jsonpath="{.status.infrastructureName}" | sed 's/-[a-z0-9]\{5\}USD//') export REGION=USD(oc get infrastructure cluster -o=jsonpath="{.status.platformStatus.aws.region}") export AWS_ACCOUNT_ID=USD(aws sts get-caller-identity --query Account --output text) export SCRATCH="/tmp/USD{CLUSTER}/cloudfront-waf" mkdir -p USD{SCRATCH} echo "Cluster: USD{CLUSTER}, Region: USD{REGION}, AWS Account ID: USD{AWS_ACCOUNT_ID}"
export CLUSTER=my-custom-value
oc -n openshift-ingress create secret tls waf-tls --cert=fullchain.pem --key=privkey.pem
apiVersion: operator.openshift.io/v1 kind: IngressController metadata: name: cloudfront-waf namespace: openshift-ingress-operator spec: domain: apps.example.com 1 defaultCertificate: name: waf-tls endpointPublishingStrategy: loadBalancer: dnsManagementPolicy: Unmanaged providerParameters: aws: type: NLB type: AWS scope: External type: LoadBalancerService routeSelector: 2 matchLabels: route: waf
oc apply -f waf-ingress-controller.yaml
oc -n openshift-ingress get service/router-cloudfront-waf
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE router-cloudfront-waf LoadBalancer 172.30.16.141 a68a838a7f26440bf8647809b61c4bc8-4225395f488830bd.elb.us-east-1.amazonaws.com 80:30606/TCP,443:31065/TCP 2m19s
cat << EOF > USD{SCRATCH}/waf-rules.json [ { "Name": "AWS-AWSManagedRulesCommonRuleSet", "Priority": 0, "Statement": { "ManagedRuleGroupStatement": { "VendorName": "AWS", "Name": "AWSManagedRulesCommonRuleSet" } }, "OverrideAction": { "None": {} }, "VisibilityConfig": { "SampledRequestsEnabled": true, "CloudWatchMetricsEnabled": true, "MetricName": "AWS-AWSManagedRulesCommonRuleSet" } }, { "Name": "AWS-AWSManagedRulesSQLiRuleSet", "Priority": 1, "Statement": { "ManagedRuleGroupStatement": { "VendorName": "AWS", "Name": "AWSManagedRulesSQLiRuleSet" } }, "OverrideAction": { "None": {} }, "VisibilityConfig": { "SampledRequestsEnabled": true, "CloudWatchMetricsEnabled": true, "MetricName": "AWS-AWSManagedRulesSQLiRuleSet" } } ] EOF
WAF_WACL=USD(aws wafv2 create-web-acl --name cloudfront-waf --region USD{REGION} --default-action Allow={} --scope CLOUDFRONT --visibility-config SampledRequestsEnabled=true,CloudWatchMetricsEnabled=true,MetricName=USD{CLUSTER}-waf-metrics --rules file://USD{SCRATCH}/waf-rules.json --query 'Summary.Name' --output text)
NLB=USD(oc -n openshift-ingress get service router-cloudfront-waf -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')
aws acm import-certificate --certificate file://cert.pem --certificate-chain file://fullchain.pem --private-key file://privkey.pem --region us-east-1
aws cloudfront list-distributions --query "DistributionList.Items[?Origins.Items[?DomainName=='USD{NLB}']].DomainName" --output text
*.apps.example.com CNAME d1b2c3d4e5f6g7.cloudfront.net
oc new-project hello-world
oc -n hello-world new-app --image=docker.io/openshift/hello-openshift
oc -n hello-world create route edge --service=hello-openshift hello-openshift-tls --hostname hello-openshift.USD{DOMAIN}
oc -n hello-world label route.route.openshift.io/hello-openshift-tls route=waf
curl "https://hello-openshift.USD{DOMAIN}"
Hello OpenShift!
curl -X POST "https://hello-openshift.USD{DOMAIN}" -F "user='<script><alert>Hello></alert></script>'"
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> <HTML><HEAD><META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=iso-8859-1"> <TITLE>ERROR: The request could not be satisfied</TITLE> </HEAD><BODY> <H1>403 ERROR</H1> <H2>The request could not be satisfied.</H2> <HR noshade size="1px"> Request blocked. We can't connect to the server for this app or website at this time. There might be too much traffic or a configuration error. Try again later, or contact the app or website owner. <BR clear="all"> If you provide content to customers through CloudFront, you can find steps to troubleshoot and help prevent this error by reviewing the CloudFront documentation. <BR clear="all"> <HR noshade size="1px"> <PRE> Generated by cloudfront (CloudFront) Request ID: nFk9q2yB8jddI6FZOTjdliexzx-FwZtr8xUQUNT75HThPlrALDxbag== </PRE> <ADDRESS> </ADDRESS> </BODY></HTML>
export AWS_PAGER="" export CLUSTER=USD(oc get infrastructure cluster -o=jsonpath="{.status.infrastructureName}") export REGION=USD(oc get infrastructure cluster -o=jsonpath="{.status.platformStatus.aws.region}") export OIDC_ENDPOINT=USD(oc get authentication.config.openshift.io cluster -o jsonpath='{.spec.serviceAccountIssuer}' | sed 's|^https://||') export AWS_ACCOUNT_ID=USD(aws sts get-caller-identity --query Account --output text) export SCRATCH="/tmp/USD{CLUSTER}/alb-waf" mkdir -p USD{SCRATCH} echo "Cluster: USD(echo USD{CLUSTER} | sed 's/-[a-z0-9]\{5\}USD//'), Region: USD{REGION}, OIDC Endpoint: USD{OIDC_ENDPOINT}, AWS Account ID: USD{AWS_ACCOUNT_ID}"
export VPC_ID=<vpc-id> 1 export PUBLIC_SUBNET_IDS=(<space-separated-list-of-ids>) 2 export PRIVATE_SUBNET_IDS=(<space-separated-list-of-ids>) 3
aws ec2 create-tags --resources USD{VPC_ID} --tags Key=kubernetes.io/cluster/USD{CLUSTER},Value=shared --region USD{REGION}
aws ec2 create-tags --resources USD{PUBLIC_SUBNET_IDS} --tags Key=kubernetes.io/role/elb,Value='1' Key=kubernetes.io/cluster/USD{CLUSTER},Value=shared --region USD{REGION}
aws ec2 create-tags --resources USD{PRIVATE_SUBNET_IDS} --tags Key=kubernetes.io/role/internal-elb,Value='1' Key=kubernetes.io/cluster/USD{CLUSTER},Value=shared --region USD{REGION}
oc new-project aws-load-balancer-operator
POLICY_ARN=USD(aws iam list-policies --query "Policies[?PolicyName=='aws-load-balancer-operator-policy'].{ARN:Arn}" --output text)
if [[ -z "USD{POLICY_ARN}" ]]; then wget -O "USD{SCRATCH}/load-balancer-operator-policy.json" https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json POLICY_ARN=USD(aws --region "USDREGION" --query Policy.Arn --output text iam create-policy --policy-name aws-load-balancer-operator-policy --policy-document "file://USD{SCRATCH}/load-balancer-operator-policy.json") fi
cat <<EOF > "USD{SCRATCH}/trust-policy.json" { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Condition": { "StringEquals" : { "USD{OIDC_ENDPOINT}:sub": ["system:serviceaccount:aws-load-balancer-operator:aws-load-balancer-operator-controller-manager", "system:serviceaccount:aws-load-balancer-operator:aws-load-balancer-controller-cluster"] } }, "Principal": { "Federated": "arn:aws:iam::USDAWS_ACCOUNT_ID:oidc-provider/USD{OIDC_ENDPOINT}" }, "Action": "sts:AssumeRoleWithWebIdentity" } ] } EOF
ROLE_ARN=USD(aws iam create-role --role-name "USD{CLUSTER}-alb-operator" --assume-role-policy-document "file://USD{SCRATCH}/trust-policy.json" --query Role.Arn --output text)
aws iam attach-role-policy --role-name "USD{CLUSTER}-alb-operator" --policy-arn USD{POLICY_ARN}
cat << EOF | oc apply -f - apiVersion: v1 kind: Secret metadata: name: aws-load-balancer-operator namespace: aws-load-balancer-operator stringData: credentials: | [default] role_arn = USD{ROLE_ARN} web_identity_token_file = /var/run/secrets/openshift/serviceaccount/token EOF
cat << EOF | oc apply -f - apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: aws-load-balancer-operator namespace: aws-load-balancer-operator spec: upgradeStrategy: Default --- apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: aws-load-balancer-operator namespace: aws-load-balancer-operator spec: channel: stable-v1.0 installPlanApproval: Automatic name: aws-load-balancer-operator source: redhat-operators sourceNamespace: openshift-marketplace startingCSV: aws-load-balancer-operator.v1.0.0 EOF
cat << EOF | oc apply -f - apiVersion: networking.olm.openshift.io/v1 kind: AWSLoadBalancerController metadata: name: cluster spec: credentials: name: aws-load-balancer-operator enabledAddons: - AWSWAFv2 EOF
oc -n aws-load-balancer-operator get pods
NAME READY STATUS RESTARTS AGE aws-load-balancer-controller-cluster-6ddf658785-pdp5d 1/1 Running 0 99s aws-load-balancer-operator-controller-manager-577d9ffcb9-w6zqn 2/2 Running 0 2m4s
oc new-project hello-world
oc new-app -n hello-world --image=docker.io/openshift/hello-openshift
oc -n hello-world patch service hello-openshift -p '{"spec":{"type":"NodePort"}}'
cat << EOF | oc apply -f - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: hello-openshift-alb namespace: hello-world annotations: alb.ingress.kubernetes.io/scheme: internet-facing spec: ingressClassName: alb rules: - http: paths: - path: / pathType: Exact backend: service: name: hello-openshift port: number: 8080 EOF
INGRESS=USD(oc -n hello-world get ingress hello-openshift-alb -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') curl "http://USD{INGRESS}"
Hello OpenShift!
cat << EOF > USD{SCRATCH}/waf-rules.json [ { "Name": "AWS-AWSManagedRulesCommonRuleSet", "Priority": 0, "Statement": { "ManagedRuleGroupStatement": { "VendorName": "AWS", "Name": "AWSManagedRulesCommonRuleSet" } }, "OverrideAction": { "None": {} }, "VisibilityConfig": { "SampledRequestsEnabled": true, "CloudWatchMetricsEnabled": true, "MetricName": "AWS-AWSManagedRulesCommonRuleSet" } }, { "Name": "AWS-AWSManagedRulesSQLiRuleSet", "Priority": 1, "Statement": { "ManagedRuleGroupStatement": { "VendorName": "AWS", "Name": "AWSManagedRulesSQLiRuleSet" } }, "OverrideAction": { "None": {} }, "VisibilityConfig": { "SampledRequestsEnabled": true, "CloudWatchMetricsEnabled": true, "MetricName": "AWS-AWSManagedRulesSQLiRuleSet" } } ] EOF
WAF_ARN=USD(aws wafv2 create-web-acl --name USD{CLUSTER}-waf --region USD{REGION} --default-action Allow={} --scope REGIONAL --visibility-config SampledRequestsEnabled=true,CloudWatchMetricsEnabled=true,MetricName=USD{CLUSTER}-waf-metrics --rules file://USD{SCRATCH}/waf-rules.json --query 'Summary.ARN' --output text)
oc annotate -n hello-world ingress.networking.k8s.io/hello-openshift-alb alb.ingress.kubernetes.io/wafv2-acl-arn=USD{WAF_ARN}
curl "http://USD{INGRESS}"
Hello OpenShift!
curl -X POST "http://USD{INGRESS}" -F "user='<script><alert>Hello></alert></script>'"
<html> <head><title>403 Forbidden</title></head> <body> <center><h1>403 Forbidden</h1></center> </body> </html
export CLUSTER_NAME=USD(oc get infrastructure cluster -o=jsonpath="{.status.infrastructureName}" | sed 's/-[a-z0-9]\{5\}USD//') export ROSA_CLUSTER_ID=USD(rosa describe cluster -c USD{CLUSTER_NAME} --output json | jq -r .id) export REGION=USD(rosa describe cluster -c USD{CLUSTER_NAME} --output json | jq -r .region.id) export OIDC_ENDPOINT=USD(oc get authentication.config.openshift.io cluster -o jsonpath='{.spec.serviceAccountIssuer}' | sed 's|^https://||') export AWS_ACCOUNT_ID=`aws sts get-caller-identity --query Account --output text` export CLUSTER_VERSION=`rosa describe cluster -c USD{CLUSTER_NAME} -o json | jq -r .version.raw_id | cut -f -2 -d '.'` export ROLE_NAME="USD{CLUSTER_NAME}-openshift-oadp-aws-cloud-credentials" export AWS_PAGER="" export SCRATCH="/tmp/USD{CLUSTER_NAME}/oadp" mkdir -p USD{SCRATCH} echo "Cluster ID: USD{ROSA_CLUSTER_ID}, Region: USD{REGION}, OIDC Endpoint: USD{OIDC_ENDPOINT}, AWS Account ID: USD{AWS_ACCOUNT_ID}"
POLICY_ARN=USD(aws iam list-policies --query "Policies[?PolicyName=='RosaOadpVer1'].{ARN:Arn}" --output text) if [[ -z "USD{POLICY_ARN}" ]]; then cat << EOF > USD{SCRATCH}/policy.json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "s3:CreateBucket", "s3:DeleteBucket", "s3:PutBucketTagging", "s3:GetBucketTagging", "s3:PutEncryptionConfiguration", "s3:GetEncryptionConfiguration", "s3:PutLifecycleConfiguration", "s3:GetLifecycleConfiguration", "s3:GetBucketLocation", "s3:ListBucket", "s3:GetObject", "s3:PutObject", "s3:DeleteObject", "s3:ListBucketMultipartUploads", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts", "ec2:DescribeSnapshots", "ec2:DescribeVolumes", "ec2:DescribeVolumeAttribute", "ec2:DescribeVolumesModifications", "ec2:DescribeVolumeStatus", "ec2:CreateTags", "ec2:CreateVolume", "ec2:CreateSnapshot", "ec2:DeleteSnapshot" ], "Resource": "*" } ]} EOF POLICY_ARN=USD(aws iam create-policy --policy-name "RosaOadpVer1" --policy-document file:///USD{SCRATCH}/policy.json --query Policy.Arn --tags Key=rosa_openshift_version,Value=USD{CLUSTER_VERSION} Key=rosa_role_prefix,Value=ManagedOpenShift Key=operator_namespace,Value=openshift-oadp Key=operator_name,Value=openshift-oadp --output text) fi echo USD{POLICY_ARN}
cat <<EOF > USD{SCRATCH}/trust-policy.json { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "Federated": "arn:aws:iam::USD{AWS_ACCOUNT_ID}:oidc-provider/USD{OIDC_ENDPOINT}" }, "Action": "sts:AssumeRoleWithWebIdentity", "Condition": { "StringEquals": { "USD{OIDC_ENDPOINT}:sub": [ "system:serviceaccount:openshift-adp:openshift-adp-controller-manager", "system:serviceaccount:openshift-adp:velero"] } } }] } EOF ROLE_ARN=USD(aws iam create-role --role-name "USD{ROLE_NAME}" --assume-role-policy-document file://USD{SCRATCH}/trust-policy.json --tags Key=rosa_cluster_id,Value=USD{ROSA_CLUSTER_ID} Key=rosa_openshift_version,Value=USD{CLUSTER_VERSION} Key=rosa_role_prefix,Value=ManagedOpenShift Key=operator_namespace,Value=openshift-adp Key=operator_name,Value=openshift-oadp --query Role.Arn --output text) echo USD{ROLE_ARN}
aws iam attach-role-policy --role-name "USD{ROLE_NAME}" --policy-arn USD{POLICY_ARN}
oc create namespace openshift-adp
cat <<EOF > USD{SCRATCH}/credentials [default] role_arn = USD{ROLE_ARN} web_identity_token_file = /var/run/secrets/openshift/serviceaccount/token region=<aws_region> 1 EOF oc -n openshift-adp create secret generic cloud-credentials --from-file=USD{SCRATCH}/credentials
cat << EOF | oc create -f - apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: generateName: openshift-adp- namespace: openshift-adp name: oadp spec: targetNamespaces: - openshift-adp --- apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: redhat-oadp-operator namespace: openshift-adp spec: channel: stable-1.2 installPlanApproval: Automatic name: redhat-oadp-operator source: redhat-operators sourceNamespace: openshift-marketplace EOF
watch oc -n openshift-adp get pods
NAME READY STATUS RESTARTS AGE openshift-adp-controller-manager-546684844f-qqjhn 1/1 Running 0 22s
cat << EOF | oc create -f - apiVersion: oadp.openshift.io/v1alpha1 kind: CloudStorage metadata: name: USD{CLUSTER_NAME}-oadp namespace: openshift-adp spec: creationSecret: key: credentials name: cloud-credentials enableSharedConfig: true name: USD{CLUSTER_NAME}-oadp provider: aws region: USDREGION EOF
oc get pvc -n <namespace> 1
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE applog Bound pvc-351791ae-b6ab-4e8b-88a4-30f73caf5ef8 1Gi RWO gp3-csi 4d19h mysql Bound pvc-16b8e009-a20a-4379-accc-bc81fedd0621 1Gi RWO gp3-csi 4d19h
oc get storageclass
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE gp2 kubernetes.io/aws-ebs Delete WaitForFirstConsumer true 4d21h gp2-csi ebs.csi.aws.com Delete WaitForFirstConsumer true 4d21h gp3 ebs.csi.aws.com Delete WaitForFirstConsumer true 4d21h gp3-csi (default) ebs.csi.aws.com Delete WaitForFirstConsumer true 4d21h
cat << EOF | oc create -f - apiVersion: oadp.openshift.io/v1alpha1 kind: DataProtectionApplication metadata: name: USD{CLUSTER_NAME}-dpa namespace: openshift-adp spec: backupImages: true features: dataMover: enable: false backupLocations: - bucket: cloudStorageRef: name: USD{CLUSTER_NAME}-oadp credential: key: credentials name: cloud-credentials prefix: velero default: true config: region: USD{REGION} configuration: velero: defaultPlugins: - openshift - aws - csi restic: enable: false EOF
cat << EOF | oc create -f - apiVersion: oadp.openshift.io/v1alpha1 kind: DataProtectionApplication metadata: name: USD{CLUSTER_NAME}-dpa namespace: openshift-adp spec: backupImages: true features: dataMover: enable: false backupLocations: - bucket: cloudStorageRef: name: USD{CLUSTER_NAME}-oadp credential: key: credentials name: cloud-credentials prefix: velero default: true config: region: USD{REGION} configuration: velero: defaultPlugins: - openshift - aws restic: enable: false snapshotLocations: - velero: config: credentialsFile: /tmp/credentials/openshift-adp/cloud-credentials-credentials enableSharedConfig: 'true' profile: default region: USD{REGION} provider: aws EOF
oc create namespace hello-world oc new-app -n hello-world --image=docker.io/openshift/hello-openshift
oc expose service/hello-openshift -n hello-world
curl `oc get route/hello-openshift -n hello-world -o jsonpath='{.spec.host}'`
Hello OpenShift!
cat << EOF | oc create -f - apiVersion: velero.io/v1 kind: Backup metadata: name: hello-world namespace: openshift-adp spec: includedNamespaces: - hello-world storageLocation: USD{CLUSTER_NAME}-dpa-1 ttl: 720h0m0s EOF
watch "oc -n openshift-adp get backup hello-world -o json | jq .status"
{ "completionTimestamp": "2022-09-07T22:20:44Z", "expiration": "2022-10-07T22:20:22Z", "formatVersion": "1.1.0", "phase": "Completed", "progress": { "itemsBackedUp": 58, "totalItems": 58 }, "startTimestamp": "2022-09-07T22:20:22Z", "version": 1 }
oc delete ns hello-world
cat << EOF | oc create -f - apiVersion: velero.io/v1 kind: Restore metadata: name: hello-world namespace: openshift-adp spec: backupName: hello-world EOF
watch "oc -n openshift-adp get restore hello-world -o json | jq .status"
{ "completionTimestamp": "2022-09-07T22:25:47Z", "phase": "Completed", "progress": { "itemsRestored": 38, "totalItems": 38 }, "startTimestamp": "2022-09-07T22:25:28Z", "warnings": 9 }
oc -n hello-world get pods
NAME READY STATUS RESTARTS AGE hello-openshift-9f885f7c6-kdjpj 1/1 Running 0 90s
curl `oc get route/hello-openshift -n hello-world -o jsonpath='{.spec.host}'`
Hello OpenShift!
oc delete ns hello-world
oc delete backups.velero.io hello-world oc delete restores.velero.io hello-world
velero backup delete hello-world velero restore delete hello-world
oc -n openshift-adp delete dpa USD{CLUSTER_NAME}-dpa
oc -n openshift-adp delete cloudstorage USD{CLUSTER_NAME}-oadp
oc -n openshift-adp patch cloudstorage USD{CLUSTER_NAME}-oadp -p '{"metadata":{"finalizers":null}}' --type=merge
oc -n openshift-adp delete subscription oadp-operator
oc delete ns redhat-openshift-adp
for CRD in `oc get crds | grep velero | awk '{print USD1}'`; do oc delete crd USDCRD; done for CRD in `oc get crds | grep -i oadp | awk '{print USD1}'`; do oc delete crd USDCRD; done
aws s3 rm s3://USD{CLUSTER_NAME}-oadp --recursive aws s3api delete-bucket --bucket USD{CLUSTER_NAME}-oadp
aws iam detach-role-policy --role-name "USD{ROLE_NAME}" --policy-arn "USD{POLICY_ARN}"
aws iam delete-role --role-name "USD{ROLE_NAME}"
export AWS_PAGER="" export ROSA_CLUSTER_NAME=USD(oc get infrastructure cluster -o=jsonpath="{.status.infrastructureName}" | sed 's/-[a-z0-9]\{5\}USD//') export REGION=USD(oc get infrastructure cluster -o=jsonpath="{.status.platformStatus.aws.region}") export OIDC_ENDPOINT=USD(oc get authentication.config.openshift.io cluster -o jsonpath='{.spec.serviceAccountIssuer}' | sed 's|^https://||') export AWS_ACCOUNT_ID=USD(aws sts get-caller-identity --query Account --output text) export SCRATCH="/tmp/USD{ROSA_CLUSTER_NAME}/alb-operator" mkdir -p USD{SCRATCH} echo "Cluster: USD{ROSA_CLUSTER_NAME}, Region: USD{REGION}, OIDC Endpoint: USD{OIDC_ENDPOINT}, AWS Account ID: USD{AWS_ACCOUNT_ID}"
export VPC_ID=<vpc-id> export PUBLIC_SUBNET_IDS=<public-subnets> export PRIVATE_SUBNET_IDS=<private-subnets> export CLUSTER_NAME=USD(oc get infrastructure cluster -o=jsonpath="{.status.infrastructureName}")
aws ec2 create-tags --resources USD{VPC_ID} --tags Key=kubernetes.io/cluster/USD{CLUSTER_NAME},Value=owned --region USD{REGION}
aws ec2 create-tags --resources USD{PUBLIC_SUBNET_IDS} --tags Key=kubernetes.io/role/elb,Value='' --region USD{REGION}
aws ec2 create-tags --resources "USD{PRIVATE_SUBNET_IDS}" --tags Key=kubernetes.io/role/internal-elb,Value='' --region USD{REGION}
oc new-project aws-load-balancer-operator POLICY_ARN=USD(aws iam list-policies --query "Policies[?PolicyName=='aws-load-balancer-operator-policy'].{ARN:Arn}" --output text) if [[ -z "USD{POLICY_ARN}" ]]; then wget -O "USD{SCRATCH}/load-balancer-operator-policy.json" https://raw.githubusercontent.com/rh-mobb/documentation/main/content/rosa/aws-load-balancer-operator/load-balancer-operator-policy.json POLICY_ARN=USD(aws --region "USDREGION" --query Policy.Arn --output text iam create-policy --policy-name aws-load-balancer-operator-policy --policy-document "file://USD{SCRATCH}/load-balancer-operator-policy.json") fi echo USDPOLICY_ARN
cat <<EOF > "USD{SCRATCH}/trust-policy.json" { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Condition": { "StringEquals" : { "USD{OIDC_ENDPOINT}:sub": ["system:serviceaccount:aws-load-balancer-operator:aws-load-balancer-operator-controller-manager", "system:serviceaccount:aws-load-balancer-operator:aws-load-balancer-controller-cluster"] } }, "Principal": { "Federated": "arn:aws:iam::USDAWS_ACCOUNT_ID:oidc-provider/USD{OIDC_ENDPOINT}" }, "Action": "sts:AssumeRoleWithWebIdentity" } ] } EOF
ROLE_ARN=USD(aws iam create-role --role-name "USD{ROSA_CLUSTER_NAME}-alb-operator" --assume-role-policy-document "file://USD{SCRATCH}/trust-policy.json" --query Role.Arn --output text) echo USDROLE_ARN aws iam attach-role-policy --role-name "USD{ROSA_CLUSTER_NAME}-alb-operator" --policy-arn USDPOLICY_ARN
cat << EOF | oc apply -f - apiVersion: v1 kind: Secret metadata: name: aws-load-balancer-operator namespace: aws-load-balancer-operator stringData: credentials: | [default] role_arn = USDROLE_ARN web_identity_token_file = /var/run/secrets/openshift/serviceaccount/token EOF
cat << EOF | oc apply -f - apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: aws-load-balancer-operator namespace: aws-load-balancer-operator spec: upgradeStrategy: Default --- apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: aws-load-balancer-operator namespace: aws-load-balancer-operator spec: channel: stable-v1.0 installPlanApproval: Automatic name: aws-load-balancer-operator source: redhat-operators sourceNamespace: openshift-marketplace startingCSV: aws-load-balancer-operator.v1.0.0 EOF
cat << EOF | oc apply -f - apiVersion: networking.olm.openshift.io/v1 kind: AWSLoadBalancerController metadata: name: cluster spec: credentials: name: aws-load-balancer-operator EOF
oc -n aws-load-balancer-operator get pods
NAME READY STATUS RESTARTS AGE aws-load-balancer-controller-cluster-6ddf658785-pdp5d 1/1 Running 0 99s aws-load-balancer-operator-controller-manager-577d9ffcb9-w6zqn 2/2 Running 0 2m4s
oc new-project hello-world
oc new-app -n hello-world --image=docker.io/openshift/hello-openshift
cat << EOF | oc apply -f - apiVersion: v1 kind: Service metadata: name: hello-openshift-nodeport namespace: hello-world spec: ports: - port: 80 targetPort: 8080 protocol: TCP type: NodePort selector: deployment: hello-openshift EOF
cat << EOF | oc apply -f - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: hello-openshift-alb namespace: hello-world annotations: alb.ingress.kubernetes.io/scheme: internet-facing spec: ingressClassName: alb rules: - http: paths: - path: / pathType: Exact backend: service: name: hello-openshift-nodeport port: number: 80 EOF
INGRESS=USD(oc -n hello-world get ingress hello-openshift-alb -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') curl "http://USD{INGRESS}"
Hello OpenShift!
cat << EOF | oc apply -f - apiVersion: v1 kind: Service metadata: name: hello-openshift-nlb namespace: hello-world annotations: service.beta.kubernetes.io/aws-load-balancer-type: external service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: instance service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing spec: ports: - port: 80 targetPort: 8080 protocol: TCP type: LoadBalancer selector: deployment: hello-openshift EOF
NLB=USD(oc -n hello-world get service hello-openshift-nlb -o jsonpath='{.status.loadBalancer.ingress[0].hostname}') curl "http://USD{NLB}"
Hello OpenShift!
oc delete project hello-world
oc delete subscription aws-load-balancer-operator -n aws-load-balancer-operator aws iam detach-role-policy --role-name "USD{ROSA_CLUSTER_NAME}-alb-operator" --policy-arn USDPOLICY_ARN aws iam delete-role --role-name "USD{ROSA_CLUSTER_NAME}-alb-operator"
aws iam delete-policy --policy-arn USDPOLICY_ARN
domain=USD(rosa describe cluster -c <cluster_name> | grep "DNS" | grep -oE '\S+.openshiftapps.com') echo "OAuth callback URL: https://oauth.USD{domain}/oauth2callback/AAD"
CLUSTER_NAME=example-cluster 1 IDP_NAME=AAD 2 APP_ID=yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy 3 CLIENT_SECRET=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 4 TENANT_ID=zzzzzzzz-zzzz-zzzz-zzzz-zzzzzzzzzzzz 5
rosa create idp --cluster USD{CLUSTER_NAME} --type openid --name USD{IDP_NAME} --client-id USD{APP_ID} --client-secret USD{CLIENT_SECRET} --issuer-url https://login.microsoftonline.com/USD{TENANT_ID}/v2.0 --email-claims email --name-claims name --username-claims preferred_username --extra-scopes email,profile --groups-claims groups
rosa create idp --cluster USD{CLUSTER_NAME} --type openid --name USD{IDP_NAME} --client-id USD{APP_ID} --client-secret USD{CLIENT_SECRET} --issuer-url https://login.microsoftonline.com/USD{TENANT_ID}/v2.0 --email-claims email --name-claims name --username-claims preferred_username --extra-scopes email,profile
rosa grant user cluster-admin --user=<USERNAME> 1 --cluster=USD{CLUSTER_NAME}
oc create clusterrolebinding cluster-admin-group --clusterrole=cluster-admin --group=<GROUP_ID> 1
oc login --token=<your-token> --server=<your-server-url>
oc get authentication.config.openshift.io cluster -o json | jq .spec.serviceAccountIssuer
"https://xxxxx.cloudfront.net/xxxxx"
oc new-project csi-secrets-store oc adm policy add-scc-to-user privileged system:serviceaccount:csi-secrets-store:secrets-store-csi-driver oc adm policy add-scc-to-user privileged system:serviceaccount:csi-secrets-store:csi-secrets-store-provider-aws
export REGION=USD(oc get infrastructure cluster -o=jsonpath="{.status.platformStatus.aws.region}") export OIDC_ENDPOINT=USD(oc get authentication.config.openshift.io cluster -o jsonpath='{.spec.serviceAccountIssuer}' | sed 's|^https://||') export AWS_ACCOUNT_ID=`aws sts get-caller-identity --query Account --output text` export AWS_PAGER=""
helm repo add secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts
helm repo update
helm upgrade --install -n csi-secrets-store csi-secrets-store-driver secrets-store-csi-driver/secrets-store-csi-driver
oc -n csi-secrets-store apply -f https://raw.githubusercontent.com/rh-mobb/documentation/main/content/misc/secrets-store-csi/aws-provider-installer.yaml
oc -n csi-secrets-store get ds csi-secrets-store-provider-aws csi-secrets-store-driver-secrets-store-csi-driver
oc label csidriver.storage.k8s.io/secrets-store.csi.k8s.io security.openshift.io/csi-ephemeral-volume-profile=restricted
SECRET_ARN=USD(aws --region "USDREGION" secretsmanager create-secret --name MySecret --secret-string '{"username":"shadowman", "password":"hunter2"}' --query ARN --output text); echo USDSECRET_ARN
cat << EOF > policy.json { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Action": [ "secretsmanager:GetSecretValue", "secretsmanager:DescribeSecret" ], "Resource": ["USDSECRET_ARN"] }] } EOF
POLICY_ARN=USD(aws --region "USDREGION" --query Policy.Arn --output text iam create-policy --policy-name openshift-access-to-mysecret-policy --policy-document file://policy.json); echo USDPOLICY_ARN
cat <<EOF > trust-policy.json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Condition": { "StringEquals" : { "USD{OIDC_ENDPOINT}:sub": ["system:serviceaccount:my-application:default"] } }, "Principal": { "Federated": "arn:aws:iam::USDAWS_ACCOUNT_ID:oidc-provider/USD{OIDC_ENDPOINT}" }, "Action": "sts:AssumeRoleWithWebIdentity" } ] } EOF
ROLE_ARN=USD(aws iam create-role --role-name openshift-access-to-mysecret --assume-role-policy-document file://trust-policy.json --query Role.Arn --output text); echo USDROLE_ARN
aws iam attach-role-policy --role-name openshift-access-to-mysecret --policy-arn USDPOLICY_ARN
oc new-project my-application
oc annotate -n my-application serviceaccount default eks.amazonaws.com/role-arn=USDROLE_ARN
cat << EOF | oc apply -f - apiVersion: secrets-store.csi.x-k8s.io/v1 kind: SecretProviderClass metadata: name: my-application-aws-secrets spec: provider: aws parameters: objects: | - objectName: "MySecret" objectType: "secretsmanager" EOF
cat << EOF | oc apply -f - apiVersion: v1 kind: Pod metadata: name: my-application labels: app: my-application spec: volumes: - name: secrets-store-inline csi: driver: secrets-store.csi.k8s.io readOnly: true volumeAttributes: secretProviderClass: "my-application-aws-secrets" containers: - name: my-application-deployment image: k8s.gcr.io/e2e-test-images/busybox:1.29 command: - "/bin/sleep" - "10000" volumeMounts: - name: secrets-store-inline mountPath: "/mnt/secrets-store" readOnly: true EOF
oc exec -it my-application -- cat /mnt/secrets-store/MySecret
oc delete project my-application
helm delete -n csi-secrets-store csi-secrets-store-driver
oc adm policy remove-scc-from-user privileged system:serviceaccount:csi-secrets-store:secrets-store-csi-driver; oc adm policy remove-scc-from-user privileged system:serviceaccount:csi-secrets-store:csi-secrets-store-provider-aws
oc -n csi-secrets-store delete -f https://raw.githubusercontent.com/rh-mobb/documentation/main/content/misc/secrets-store-csi/aws-provider-installer.yaml
aws iam detach-role-policy --role-name openshift-access-to-mysecret --policy-arn USDPOLICY_ARN; aws iam delete-role --role-name openshift-access-to-mysecret; aws iam delete-policy --policy-arn USDPOLICY_ARN
aws secretsmanager --region USDREGION delete-secret --secret-id USDSECRET_ARN
export CLUSTER_NAME=USD(oc get infrastructure cluster -o=jsonpath="{.status.infrastructureName}" | sed 's/-[a-z0-9]\{5\}USD//') export REGION=USD(rosa describe cluster -c USD{ROSA_CLUSTER_NAME} --output json | jq -r .region.id) export OIDC_ENDPOINT=USD(oc get authentication.config.openshift.io cluster -o json | jq -r .spec.serviceAccountIssuer | sed 's|^https://||') export AWS_ACCOUNT_ID=`aws sts get-caller-identity --query Account --output text` export ACK_SERVICE=s3 export ACK_SERVICE_ACCOUNT=ack-USD{ACK_SERVICE}-controller export POLICY_ARN=arn:aws:iam::aws:policy/AmazonS3FullAccess export AWS_PAGER="" export SCRATCH="/tmp/USD{ROSA_CLUSTER_NAME}/ack" mkdir -p USD{SCRATCH}
echo "Cluster: USD{ROSA_CLUSTER_NAME}, Region: USD{REGION}, OIDC Endpoint: USD{OIDC_ENDPOINT}, AWS Account ID: USD{AWS_ACCOUNT_ID}"
cat <<EOF > "USD{SCRATCH}/trust-policy.json" { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Condition": { "StringEquals" : { "USD{OIDC_ENDPOINT}:sub": "system:serviceaccount:ack-system:USD{ACK_SERVICE_ACCOUNT}" } }, "Principal": { "Federated": "arn:aws:iam::USDAWS_ACCOUNT_ID:oidc-provider/USD{OIDC_ENDPOINT}" }, "Action": "sts:AssumeRoleWithWebIdentity" } ] } EOF
ROLE_ARN=USD(aws iam create-role --role-name "ack-USD{ACK_SERVICE}-controller" --assume-role-policy-document "file://USD{SCRATCH}/trust-policy.json" --query Role.Arn --output text) echo USDROLE_ARN aws iam attach-role-policy --role-name "ack-USD{ACK_SERVICE}-controller" --policy-arn USD{POLICY_ARN}
oc new-project ack-system
cat <<EOF > "USD{SCRATCH}/config.txt" ACK_ENABLE_DEVELOPMENT_LOGGING=true ACK_LOG_LEVEL=debug ACK_WATCH_NAMESPACE= AWS_REGION=USD{REGION} AWS_ENDPOINT_URL= ACK_RESOURCE_TAGS=USD{CLUSTER_NAME} ENABLE_LEADER_ELECTION=true LEADER_ELECTION_NAMESPACE= EOF
oc -n ack-system create configmap --from-env-file=USD{SCRATCH}/config.txt ack-USD{ACK_SERVICE}-user-config
cat << EOF | oc apply -f - apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: ack-USD{ACK_SERVICE}-controller namespace: ack-system spec: upgradeStrategy: Default --- apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: ack-USD{ACK_SERVICE}-controller namespace: ack-system spec: channel: alpha installPlanApproval: Automatic name: ack-USD{ACK_SERVICE}-controller source: community-operators sourceNamespace: openshift-marketplace EOF
oc -n ack-system annotate serviceaccount USD{ACK_SERVICE_ACCOUNT} eks.amazonaws.com/role-arn=USD{ROLE_ARN} && oc -n ack-system rollout restart deployment ack-USD{ACK_SERVICE}-controller
oc -n ack-system get pods
NAME READY STATUS RESTARTS AGE ack-s3-controller-585f6775db-s4lfz 1/1 Running 0 51s
cat << EOF | oc apply -f - apiVersion: s3.services.k8s.aws/v1alpha1 kind: Bucket metadata: name: USD{CLUSTER-NAME}-bucket namespace: ack-system spec: name: USD{CLUSTER-NAME}-bucket EOF
aws s3 ls | grep USD{CLUSTER_NAME}-bucket
2023-10-04 14:51:45 mrmc-test-maz-bucket
oc -n ack-system delete bucket.s3.services.k8s.aws/USD{CLUSTER-NAME}-bucket
oc -n ack-system delete subscription ack-USD{ACK_SERVICE}-controller aws iam detach-role-policy --role-name "ack-USD{ACK_SERVICE}-controller" --policy-arn USD{POLICY_ARN} aws iam delete-role --role-name "ack-USD{ACK_SERVICE}-controller"
oc delete project ack-system
export DOMAIN=<apps.example.com> 1 export AWS_PAGER="" export CLUSTER=USD(oc get infrastructure cluster -o=jsonpath="{.status.infrastructureName}" | sed 's/-[a-z0-9]\{5\}USD//') export REGION=USD(oc get infrastructure cluster -o=jsonpath="{.status.platformStatus.aws.region}") export AWS_ACCOUNT_ID=USD(aws sts get-caller-identity --query Account --output text) export SCRATCH="/tmp/USD{CLUSTER}/external-dns" mkdir -p USD{SCRATCH}
echo "Cluster: USD{CLUSTER}, Region: USD{REGION}, AWS Account ID: USD{AWS_ACCOUNT_ID}"
export CLUSTER=my-custom-value
oc -n openshift-ingress create secret tls external-dns-tls --cert=fullchain.pem --key=privkey.pem
cat << EOF | oc apply -f - apiVersion: operator.openshift.io/v1 kind: IngressController metadata: name: external-dns-ingress namespace: openshift-ingress-operator spec: domain: USD{DOMAIN} defaultCertificate: name: external-dns-tls endpointPublishingStrategy: loadBalancer: dnsManagementPolicy: Unmanaged providerParameters: aws: type: NLB type: AWS scope: External type: LoadBalancerService EOF
oc -n openshift-ingress get service/router-external-dns-ingress
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE router-external-dns-ingress LoadBalancer 172.30.71.250 a4838bb991c6748439134ab89f132a43-aeae124077b50c01.elb.us-east-1.amazonaws.com 80:32227/TCP,443:30310/TCP 43s
export ZONE_ID=USD(aws route53 list-hosted-zones-by-name --output json --dns-name "USD{DOMAIN}." --query 'HostedZones[0]'.Id --out text | sed 's/\/hostedzone\///')
NLB_HOST=USD(oc -n openshift-ingress get service/router-external-dns-ingress -ojsonpath="{.status.loadBalancer.ingress[0].hostname}") cat << EOF > "USD{SCRATCH}/create-cname.json" { "Comment":"Add CNAME to ingress controller canonical domain", "Changes":[{ "Action":"CREATE", "ResourceRecordSet":{ "Name": "router-external-dns-ingress.USD{DOMAIN}", "Type":"CNAME", "TTL":30, "ResourceRecords":[{ "Value": "USD{NLB_HOST}" }] } }] } EOF
aws route53 change-resource-record-sets --hosted-zone-id USD{ZONE_ID} --change-batch file://USD{SCRATCH}/create-cname.json
cat << EOF > "USD{SCRATCH}/external-dns-policy.json" { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "route53:ChangeResourceRecordSets" ], "Resource": [ "arn:aws:route53:::hostedzone/USD{ZONE_ID}" ] }, { "Effect": "Allow", "Action": [ "route53:ListHostedZones", "route53:ListResourceRecordSets" ], "Resource": [ "*" ] } ] } EOF
aws iam create-user --user-name "USD{CLUSTER}-external-dns-operator"
aws iam attach-user-policy --user-name "USD{CLUSTER}-external-dns-operator" --policy-arn USDPOLICY_ARN
SECRET_ACCESS_KEY=USD(aws iam create-access-key --user-name "USD{CLUSTER}-external-dns-operator")
cat << EOF > "USD{SCRATCH}/credentials" [default] aws_access_key_id = USD(echo USDSECRET_ACCESS_KEY | jq -r '.AccessKey.AccessKeyId') aws_secret_access_key = USD(echo USDSECRET_ACCESS_KEY | jq -r '.AccessKey.SecretAccessKey') EOF
oc new-project external-dns-operator
cat << EOF | oc apply -f - apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: external-dns-group namespace: external-dns-operator spec: targetNamespaces: - external-dns-operator --- apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: external-dns-operator namespace: external-dns-operator spec: channel: stable-v1.1 installPlanApproval: Automatic name: external-dns-operator source: redhat-operators sourceNamespace: openshift-marketplace EOF
oc rollout status deploy external-dns-operator --timeout=300s
oc -n external-dns-operator create secret generic external-dns --from-file "USD{SCRATCH}/credentials"
cat << EOF | oc apply -f - apiVersion: externaldns.olm.openshift.io/v1beta1 kind: ExternalDNS metadata: name: USD{DOMAIN} spec: domains: - filterType: Include matchType: Exact name: USD{DOMAIN} provider: aws: credentials: name: external-dns type: AWS source: openshiftRouteOptions: routerName: external-dns-ingress type: OpenShiftRoute zones: - USD{ZONE_ID} EOF
oc rollout status deploy external-dns-USD{DOMAIN} --timeout=300s
oc new-project hello-world
oc new-app -n hello-world --image=docker.io/openshift/hello-openshift
oc -n hello-world create route edge --service=hello-openshift hello-openshift-tls --hostname hello-openshift.USD{DOMAIN}
aws route53 list-resource-record-sets --hosted-zone-id USD{ZONE_ID} --query "ResourceRecordSets[?Type == 'CNAME']" | grep hello-openshift
aws route53 list-resource-record-sets --hosted-zone-id USD{ZONE_ID} --query "ResourceRecordSets[?Type == 'TXT']" | grep USD{DOMAIN}
curl https://hello-openshift.USD{DOMAIN}
Hello OpenShift!
export DOMAIN=apps.example.com 1 export EMAIL=email@example.com 2 export AWS_PAGER="" export CLUSTER=USD(oc get infrastructure cluster -o=jsonpath="{.status.infrastructureName}" | sed 's/-[a-z0-9]\{5\}USD//') export OIDC_ENDPOINT=USD(oc get authentication.config.openshift.io cluster -o json | jq -r .spec.serviceAccountIssuer | sed 's|^https://||') export REGION=USD(oc get infrastructure cluster -o=jsonpath="{.status.platformStatus.aws.region}") export AWS_ACCOUNT_ID=USD(aws sts get-caller-identity --query Account --output text) export SCRATCH="/tmp/USD{CLUSTER}/dynamic-certs" mkdir -p USD{SCRATCH}
echo "Cluster: USD{CLUSTER}, Region: USD{REGION}, OIDC Endpoint: USD{OIDC_ENDPOINT}, AWS Account ID: USD{AWS_ACCOUNT_ID}"
export CLUSTER=my-custom-value
export ZONE_ID=USD(aws route53 list-hosted-zones-by-name --output json --dns-name "USD{DOMAIN}." --query 'HostedZones[0]'.Id --out text | sed 's/\/hostedzone\///')
cat <<EOF > "USD{SCRATCH}/cert-manager-policy.json" { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": "route53:GetChange", "Resource": "arn:aws:route53:::change/*" }, { "Effect": "Allow", "Action": [ "route53:ChangeResourceRecordSets", "route53:ListResourceRecordSets" ], "Resource": "arn:aws:route53:::hostedzone/USD{ZONE_ID}" }, { "Effect": "Allow", "Action": "route53:ListHostedZonesByName", "Resource": "*" } ] } EOF
POLICY_ARN=USD(aws iam create-policy --policy-name "USD{CLUSTER}-cert-manager-policy" --policy-document file://USD{SCRATCH}/cert-manager-policy.json --query 'Policy.Arn' --output text)
cat <<EOF > "USD{SCRATCH}/trust-policy.json" { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Condition": { "StringEquals" : { "USD{OIDC_ENDPOINT}:sub": "system:serviceaccount:cert-manager:cert-manager" } }, "Principal": { "Federated": "arn:aws:iam::USDAWS_ACCOUNT_ID:oidc-provider/USD{OIDC_ENDPOINT}" }, "Action": "sts:AssumeRoleWithWebIdentity" } ] } EOF
ROLE_ARN=USD(aws iam create-role --role-name "USD{CLUSTER}-cert-manager-operator" --assume-role-policy-document "file://USD{SCRATCH}/trust-policy.json" --query Role.Arn --output text)
aws iam attach-role-policy --role-name "USD{CLUSTER}-cert-manager-operator" --policy-arn USD{POLICY_ARN}
oc new-project cert-manager-operator
cat << EOF | oc apply -f - apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: name: openshift-cert-manager-operator-group namespace: cert-manager-operator spec: targetNamespaces: - cert-manager-operator --- apiVersion: operators.coreos.com/v1alpha1 kind: Subscription metadata: name: openshift-cert-manager-operator namespace: cert-manager-operator spec: channel: stable-v1 installPlanApproval: Automatic name: openshift-cert-manager-operator source: redhat-operators sourceNamespace: openshift-marketplace EOF
oc -n cert-manager-operator get pods
NAME READY STATUS RESTARTS AGE cert-manager-operator-controller-manager-84b8799db5-gv8mx 2/2 Running 0 12s
oc -n cert-manager annotate serviceaccount cert-manager eks.amazonaws.com/role-arn=USD{ROLE_ARN}
oc -n cert-manager delete pods -l app.kubernetes.io/name=cert-manager
oc patch certmanager.operator.openshift.io/cluster --type merge -p '{"spec":{"controllerConfig":{"overrideArgs":["--dns01-recursive-nameservers-only","--dns01-recursive-nameservers=1.1.1.1:53"]}}}'
cat << EOF | oc apply -f - apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: name: letsencrypt-production spec: acme: server: https://acme-v02.api.letsencrypt.org/directory email: USD{EMAIL} # This key doesn't exist, cert-manager creates it privateKeySecretRef: name: prod-letsencrypt-issuer-account-key solvers: - dns01: route53: hostedZoneID: USD{ZONE_ID} region: USD{REGION} secretAccessKeySecretRef: name: '' EOF
oc get clusterissuer.cert-manager.io/letsencrypt-production
NAME READY AGE letsencrypt-production True 47s
cat << EOF | oc apply -f - apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: custom-domain-ingress-cert namespace: openshift-ingress spec: secretName: custom-domain-ingress-cert-tls issuerRef: name: letsencrypt-production kind: ClusterIssuer commonName: "USD{DOMAIN}" dnsNames: - "USD{DOMAIN}" EOF
oc -n openshift-ingress get certificate.cert-manager.io/custom-domain-ingress-cert
NAME READY SECRET AGE custom-domain-ingress-cert True custom-domain-ingress-cert-tls 9m53s
cat << EOF | oc apply -f - apiVersion: operator.openshift.io/v1 kind: IngressController metadata: name: custom-domain-ingress namespace: openshift-ingress-operator spec: domain: USD{DOMAIN} defaultCertificate: name: custom-domain-ingress-cert-tls endpointPublishingStrategy: loadBalancer: dnsManagementPolicy: Unmanaged providerParameters: aws: type: NLB type: AWS scope: External type: LoadBalancerService EOF
oc -n openshift-ingress get service/router-custom-domain-ingress
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE router-custom-domain-ingress LoadBalancer 172.30.174.34 a309962c3bd6e42c08cadb9202eca683-1f5bbb64a1f1ec65.elb.us-east-1.amazonaws.com 80:31342/TCP,443:31821/TCP 7m28s
INGRESS=USD(oc -n openshift-ingress get service/router-custom-domain-ingress -ojsonpath="{.status.loadBalancer.ingress[0].hostname}") cat << EOF > "USD{SCRATCH}/create-cname.json" { "Comment":"Add CNAME to custom domain endpoint", "Changes":[{ "Action":"CREATE", "ResourceRecordSet":{ "Name": "*.USD{DOMAIN}", "Type":"CNAME", "TTL":30, "ResourceRecords":[{ "Value": "USD{INGRESS}" }] } }] } EOF
aws route53 change-resource-record-sets --hosted-zone-id USD{ZONE_ID} --change-batch file://USD{SCRATCH}/create-cname.json
oc -n cert-manager apply -f https://github.com/cert-manager/openshift-routes/releases/latest/download/cert-manager-openshift-routes.yaml
oc -n cert-manager get pods
NAME READY STATUS RESTARTS AGE cert-manager-866d8f788c-9kspc 1/1 Running 0 4h21m cert-manager-cainjector-6885c585bd-znws8 1/1 Running 0 4h41m cert-manager-openshift-routes-75b6bb44cd-f8kd5 1/1 Running 0 6s cert-manager-webhook-8498785dd9-bvfdf 1/1 Running 0 4h41m
oc new-project hello-world
oc -n hello-world new-app --image=docker.io/openshift/hello-openshift
oc -n hello-world create route edge --service=hello-openshift hello-openshift-tls --hostname hello.USD{DOMAIN}
curl -I https://hello.USD{DOMAIN}
curl: (60) SSL: no alternative certificate subject name matches target host name 'hello.example.com' More details here: https://curl.se/docs/sslcerts.html curl failed to verify the legitimacy of the server and therefore could not establish a secure connection to it. To learn more about this situation and how to fix it, please visit the web page mentioned above.
oc -n hello-world annotate route hello-openshift-tls cert-manager.io/issuer-kind=ClusterIssuer cert-manager.io/issuer-name=letsencrypt-production
curl -I https://hello.USD{DOMAIN}
HTTP/2 200 date: Thu, 05 Oct 2023 23:45:33 GMT content-length: 17 content-type: text/plain; charset=utf-8 set-cookie: 52e4465485b6fb4f8a1b1bed128d0f3b=68676068bb32d24f0f558f094ed8e4d7; path=/; HttpOnly; Secure; SameSite=None cache-control: private
oc get certificate,certificaterequest,order,challenge
export ROSA_CLUSTER_NAME=USD(oc get infrastructure cluster -o=jsonpath="{.status.infrastructureName}" | sed 's/-[a-z0-9]\{5\}USD//') export ROSA_MACHINE_POOL_NAME=worker
oc get node -o json | jq '.items[] | { "name": .metadata.name, "ips": (.status.addresses | map(select(.type == "InternalIP") | .address)), "capacity": (.metadata.annotations."cloud.network.openshift.io/egress-ipconfig" | fromjson[] | .capacity.ipv4) }'
--- { "name": "ip-10-10-145-88.ec2.internal", "ips": [ "10.10.145.88" ], "capacity": 14 } { "name": "ip-10-10-154-175.ec2.internal", "ips": [ "10.10.154.175" ], "capacity": 14 } ---
oc new-project demo-egress-ns
cat <<EOF | oc apply -f - apiVersion: k8s.ovn.org/v1 kind: EgressIP metadata: name: demo-egress-ns spec: # NOTE: these egress IPs are within the subnet range(s) in which my worker nodes # are deployed. egressIPs: - 10.10.100.253 - 10.10.150.253 - 10.10.200.253 namespaceSelector: matchLabels: kubernetes.io/metadata.name: demo-egress-ns EOF
oc new-project demo-egress-pod
cat <<EOF | oc apply -f - apiVersion: k8s.ovn.org/v1 kind: EgressIP metadata: name: demo-egress-pod spec: # NOTE: these egress IPs are within the subnet range(s) in which my worker nodes # are deployed. egressIPs: - 10.10.100.254 - 10.10.150.254 - 10.10.200.254 namespaceSelector: matchLabels: kubernetes.io/metadata.name: demo-egress-pod podSelector: matchLabels: run: demo-egress-pod EOF
oc get egressips
NAME EGRESSIPS ASSIGNED NODE ASSIGNED EGRESSIPS demo-egress-ns 10.10.100.253 demo-egress-pod 10.10.100.254
rosa update machinepool USD{ROSA_MACHINE_POOL_NAME} --cluster="USD{ROSA_CLUSTER_NAME}" --labels "k8s.ovn.org/egress-assignable="
oc get egressips
NAME EGRESSIPS ASSIGNED NODE ASSIGNED EGRESSIPS demo-egress-ns 10.10.100.253 ip-10-10-156-122.ec2.internal 10.10.150.253 demo-egress-pod 10.10.100.254 ip-10-10-156-122.ec2.internal 10.10.150.254
oc -n default run demo-service --image=gcr.io/google_containers/echoserver:1.4
cat <<EOF | oc apply -f - apiVersion: v1 kind: Service metadata: name: demo-service namespace: default annotations: service.beta.kubernetes.io/aws-load-balancer-scheme: "internal" service.beta.kubernetes.io/aws-load-balancer-internal: "true" spec: selector: run: demo-service ports: - port: 80 targetPort: 8080 type: LoadBalancer externalTrafficPolicy: Local # NOTE: this limits the source IPs that are allowed to connect to our service. It # is being used as part of this demo, restricting connectivity to our egress # IP addresses only. # NOTE: these egress IPs are within the subnet range(s) in which my worker nodes # are deployed. loadBalancerSourceRanges: - 10.10.100.254/32 - 10.10.150.254/32 - 10.10.200.254/32 - 10.10.100.253/32 - 10.10.150.253/32 - 10.10.200.253/32 EOF
export LOAD_BALANCER_HOSTNAME=USD(oc get svc -n default demo-service -o json | jq -r '.status.loadBalancer.ingress[].hostname')
oc run demo-egress-ns -it --namespace=demo-egress-ns --env=LOAD_BALANCER_HOSTNAME=USDLOAD_BALANCER_HOSTNAME --image=registry.access.redhat.com/ubi9/ubi -- bash
curl -s http://USDLOAD_BALANCER_HOSTNAME
CLIENT VALUES: client_address=10.10.207.247 command=GET real path=/ query=nil request_version=1.1 request_uri=http://internal-a3e61de18bfca4a53a94a208752b7263-148284314.us-east-1.elb.amazonaws.com:8080/ SERVER VALUES: server_version=nginx: 1.10.0 - lua: 10001 HEADERS RECEIVED: accept=*/* host=internal-a3e61de18bfca4a53a94a208752b7263-148284314.us-east-1.elb.amazonaws.com user-agent=curl/7.76.1 BODY: -no body in request-
exit
oc run demo-egress-pod -it --namespace=demo-egress-pod --env=LOAD_BALANCER_HOSTNAME=USDLOAD_BALANCER_HOSTNAME --image=registry.access.redhat.com/ubi9/ubi -- bash
curl -s http://USDLOAD_BALANCER_HOSTNAME
CLIENT VALUES: client_address=10.10.207.247 command=GET real path=/ query=nil request_version=1.1 request_uri=http://internal-a3e61de18bfca4a53a94a208752b7263-148284314.us-east-1.elb.amazonaws.com:8080/ SERVER VALUES: server_version=nginx: 1.10.0 - lua: 10001 HEADERS RECEIVED: accept=*/* host=internal-a3e61de18bfca4a53a94a208752b7263-148284314.us-east-1.elb.amazonaws.com user-agent=curl/7.76.1 BODY: -no body in request-
exit
oc run demo-egress-pod-fail -it --namespace=demo-egress-pod --env=LOAD_BALANCER_HOSTNAME=USDLOAD_BALANCER_HOSTNAME --image=registry.access.redhat.com/ubi9/ubi -- bash
curl -s http://USDLOAD_BALANCER_HOSTNAME
exit
oc delete svc demo-service -n default; oc delete pod demo-service -n default; oc delete project demo-egress-ns; oc delete project demo-egress-pod; oc delete egressip demo-egress-ns; oc delete egressip demo-egress-pod
rosa update machinepool USD{ROSA_MACHINE_POOL_NAME} --cluster="USD{ROSA_CLUSTER_NAME}" --labels ""
export CLUSTER_NAME=USD(oc get infrastructure cluster -o=jsonpath="{.status.infrastructureName}" | sed 's/-[a-z0-9]\{5\}USD//')
echo "Cluster: USD{CLUSTER_NAME}"
Cluster: my-rosa-cluster
oc get routes -n openshift-console oc get routes -n openshift-authentication
NAME HOST/PORT PATH SERVICES PORT TERMINATION WILDCARD console console-openshift-console.apps.my-example-cluster-aws.z9a9.p1.openshiftapps.com ... 1 more console https reencrypt/Redirect None downloads downloads-openshift-console.apps.my-example-cluster-aws.z9a9.p1.openshiftapps.com ... 1 more downloads http edge/Redirect None NAME HOST/PORT PATH SERVICES PORT TERMINATION WILDCARD oauth-openshift oauth-openshift.apps.my-example-cluster-aws.z9a9.p1.openshiftapps.com ... 1 more oauth-openshift 6443 passthrough/Redirect None
export INGRESS_ID=USD(rosa list ingress -c USD{CLUSTER_NAME} -o json | jq -r '.[] | select(.default == true) | .id')
echo "Ingress ID: USD{INGRESS_ID}"
Ingress ID: r3l6
rosa edit ingress -h Edit a cluster ingress for a cluster. Usage: rosa edit ingress ID [flags] [...] --component-routes string Component routes settings. Available keys [oauth, console, downloads]. For each key a pair of hostname and tlsSecretRef is expected to be supplied. Format should be a comma separate list 'oauth: hostname=example-hostname;tlsSecretRef=example-secret-ref,downloads:...'
openssl req -newkey rsa:2048 -new -nodes -x509 -days 365 -keyout key-console.pem -out cert-console.pem -subj "/CN=console.my-new-domain.dev" openssl req -newkey rsa:2048 -new -nodes -x509 -days 365 -keyout key-downloads.pem -out cert-downloads.pem -subj "/CN=downloads.my-new-domain.dev" openssl req -newkey rsa:2048 -new -nodes -x509 -days 365 -keyout key-oauth.pem -out cert-oauth.pem -subj "/CN=oauth.my-new-domain.dev"
oc create secret tls console-tls --cert=cert-console.pem --key=key-console.pem -n openshift-config oc create secret tls downloads-tls --cert=cert-downloads.pem --key=key-downloads.pem -n openshift-config oc create secret tls oauth-tls --cert=cert-oauth.pem --key=key-oauth.pem -n openshift-config
oc get svc -n openshift-ingress NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE router-default LoadBalancer 172.30.237.88 a234gsr3242rsfsfs-1342r624.us-east-1.elb.amazonaws.com 80:31175/TCP,443:31554/TCP 76d
rosa edit ingress -c USD{CLUSTER_NAME} USD{INGRESS_ID} --component-routes 'console: hostname=console.my-new-domain.dev;tlsSecretRef=console-tls,downloads: hostname=downloads.my-new-domain.dev;tlsSecretRef=downloads-tls,oauth: hostname=oauth.my-new-domain.dev;tlsSecretRef=oauth-tls'
rosa edit ingress -c USD{CLUSTER_NAME} USD{INGRESS_ID} --component-routes 'console: hostname=console.my-new-domain.dev;tlsSecretRef=console-tls,downloads: hostname="";tlsSecretRef="", oauth: hostname=oauth.my-new-domain.dev;tlsSecretRef=oauth-tls'
rosa list ingress -c USD{CLUSTER_NAME} -ojson | jq ".[] | select(.id == \"USD{INGRESS_ID}\") | .component_routes"
{ "console": { "kind": "ComponentRoute", "hostname": "console.my-new-domain.dev", "tls_secret_ref": "console-tls" }, "downloads": { "kind": "ComponentRoute", "hostname": "downloads.my-new-domain.dev", "tls_secret_ref": "downloads-tls" }, "oauth": { "kind": "ComponentRoute", "hostname": "oauth.my-new-domain.dev", "tls_secret_ref": "oauth-tls" } }
rosa edit ingress -c USD{CLUSTER_NAME} USD{INGRESS_ID} --component-routes 'console: hostname="";tlsSecretRef="",downloads: hostname="";tlsSecretRef="", oauth: hostname="";tlsSecretRef=""'
rosa create account-roles --mode auto --yes
rosa create cluster --cluster-name <cluster-name> --sts --mode auto --yes
rosa list clusters
rosa create account-roles --mode auto --yes
I: Creating roles using 'arn:aws:iam::000000000000:user/rosa-user' I: Created role 'ManagedOpenShift-ControlPlane-Role' with ARN 'arn:aws:iam::000000000000:role/ManagedOpenShift-ControlPlane-Role' I: Created role 'ManagedOpenShift-Worker-Role' with ARN 'arn:aws:iam::000000000000:role/ManagedOpenShift-Worker-Role' I: Created role 'ManagedOpenShift-Support-Role' with ARN 'arn:aws:iam::000000000000:role/ManagedOpenShift-Support-Role' I: Created role 'ManagedOpenShift-Installer-Role' with ARN 'arn:aws:iam::000000000000:role/ManagedOpenShift-Installer-Role' I: Created policy with ARN 'arn:aws:iam::000000000000:policy/ManagedOpenShift-openshift-machine-api-aws-cloud-credentials' I: Created policy with ARN 'arn:aws:iam::000000000000:policy/ManagedOpenShift-openshift-cloud-credential-operator-cloud-crede' I: Created policy with ARN 'arn:aws:iam::000000000000:policy/ManagedOpenShift-openshift-image-registry-installer-cloud-creden' I: Created policy with ARN 'arn:aws:iam::000000000000:policy/ManagedOpenShift-openshift-ingress-operator-cloud-credentials' I: Created policy with ARN 'arn:aws:iam::000000000000:policy/ManagedOpenShift-openshift-cluster-csi-drivers-ebs-cloud-credent' I: To create a cluster with these roles, run the following command: rosa create cluster --sts
rosa create cluster --cluster-name <cluster-name> --sts --mode auto --yes
rosa create cluster --cluster-name my-rosa-cluster --sts --mode auto --yes
I: Creating cluster 'my-rosa-cluster' I: To view a list of clusters and their status, run 'rosa list clusters' I: Cluster 'my-rosa-cluster' has been created. I: Once the cluster is installed you will need to add an Identity Provider before you can login into the cluster. See 'rosa create idp --help' for more information. I: To determine when your cluster is Ready, run 'rosa describe cluster -c my-rosa-cluster'. I: To watch your cluster installation logs, run 'rosa logs install -c my-rosa-cluster --watch'. Name: my-rosa-cluster ID: 1mlhulb3bo0l54ojd0ji000000000000 External ID: OpenShift Version: Channel Group: stable DNS: my-rosa-cluster.ibhp.p1.openshiftapps.com AWS Account: 000000000000 API URL: Console URL: Region: us-west-2 Multi-AZ: false Nodes: - Master: 3 - Infra: 2 - Compute: 2 Network: - Service CIDR: 172.30.0.0/16 - Machine CIDR: 10.0.0.0/16 - Pod CIDR: 10.128.0.0/14 - Host Prefix: /23 STS Role ARN: arn:aws:iam::000000000000:role/ManagedOpenShift-Installer-Role Support Role ARN: arn:aws:iam::000000000000:role/ManagedOpenShift-Support-Role Instance IAM Roles: - Master: arn:aws:iam::000000000000:role/ManagedOpenShift-ControlPlane-Role - Worker: arn:aws:iam::000000000000:role/ManagedOpenShift-Worker-Role Operator IAM Roles: - arn:aws:iam::000000000000:role/my-rosa-cluster-openshift-image-registry-installer-cloud-credentials - arn:aws:iam::000000000000:role/my-rosa-cluster-openshift-ingress-operator-cloud-credentials - arn:aws:iam::000000000000:role/my-rosa-cluster-openshift-cluster-csi-drivers-ebs-cloud-credentials - arn:aws:iam::000000000000:role/my-rosa-cluster-openshift-machine-api-aws-cloud-credentials - arn:aws:iam::000000000000:role/my-rosa-cluster-openshift-cloud-credential-operator-cloud-credential-oper State: waiting (Waiting for OIDC configuration) Private: No Created: Oct 28 2021 20:28:09 UTC Details Page: https://console.redhat.com/openshift/details/s/1wupmiQy45xr1nN000000000000 OIDC Endpoint URL: https://rh-oidc.s3.us-east-1.amazonaws.com/1mlhulb3bo0l54ojd0ji000000000000
rosa describe cluster --cluster <cluster-name>
rosa list clusters
rosa create account-roles --mode manual
I: All policy files saved to the current directory I: Run the following commands to create the account roles and policies: aws iam create-role --role-name ManagedOpenShift-Worker-Role --assume-role-policy-document file://sts_instance_worker_trust_policy.json --tags Key=rosa_openshift_version,Value=4.8 Key=rosa_role_prefix,Value=ManagedOpenShift Key=rosa_role_type,Value=instance_worker aws iam put-role-policy --role-name ManagedOpenShift-Worker-Role --policy-name ManagedOpenShift-Worker-Role-Policy --policy-document file://sts_instance_worker_permission_policy.json
ls openshift_cloud_credential_operator_cloud_credential_operator_iam_ro_creds_policy.json sts_instance_controlplane_permission_policy.json openshift_cluster_csi_drivers_ebs_cloud_credentials_policy.json sts_instance_controlplane_trust_policy.json openshift_image_registry_installer_cloud_credentials_policy.json sts_instance_worker_permission_policy.json openshift_ingress_operator_cloud_credentials_policy.json sts_instance_worker_trust_policy.json openshift_machine_api_aws_cloud_credentials_policy.json sts_support_permission_policy.json sts_installer_permission_policy.json sts_support_trust_policy.json sts_installer_trust_policy.json
cat sts_installer_permission_policy.json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "autoscaling:DescribeAutoScalingGroups", "ec2:AllocateAddress", "ec2:AssociateAddress", "ec2:AssociateDhcpOptions", "ec2:AssociateRouteTable", "ec2:AttachInternetGateway", "ec2:AttachNetworkInterface", "ec2:AuthorizeSecurityGroupEgress", "ec2:AuthorizeSecurityGroupIngress", [...]
rosa create cluster --interactive --sts
Cluster name: my-rosa-cluster OpenShift version: <choose version> External ID (optional): <leave blank> Operator roles prefix: <accept default> Multiple availability zones: No AWS region: <choose region> PrivateLink cluster: No Install into an existing VPC: No Enable Customer Managed key: No Compute nodes instance type: m5.xlarge Enable autoscaling: No Compute nodes: 2 Machine CIDR: <accept default> Service CIDR: <accept default> Pod CIDR: <accept default> Host prefix: <accept default> Encrypt etcd data (optional): No Disable Workload monitoring: No
I: Creating cluster 'my-rosa-cluster' I: To create this cluster again in the future, you can run: rosa create cluster --cluster-name my-rosa-cluster --role-arn arn:aws:iam::000000000000:role/ManagedOpenShift-Installer-Role --support-role-arn arn:aws:iam::000000000000:role/ManagedOpenShift-Support-Role --master-iam-role arn:aws:iam::000000000000:role/ManagedOpenShift-ControlPlane-Role --worker-iam-role arn:aws:iam::000000000000:role/ManagedOpenShift-Worker-Role --operator-roles-prefix my-rosa-cluster --region us-west-2 --version 4.8.13 --compute-nodes 2 --machine-cidr 10.0.0.0/16 --service-cidr 172.30.0.0/16 --pod-cidr 10.128.0.0/14 --host-prefix 23 I: To view a list of clusters and their status, run 'rosa list clusters' I: Cluster 'my-rosa-cluster' has been created. I: Once the cluster is installed you will need to add an Identity Provider before you can login into the cluster. See 'rosa create idp --help' for more information. Name: my-rosa-cluster ID: 1t6i760dbum4mqltqh6o000000000000 External ID: OpenShift Version: Channel Group: stable DNS: my-rosa-cluster.abcd.p1.openshiftapps.com AWS Account: 000000000000 API URL: Console URL: Region: us-west-2 Multi-AZ: false Nodes: - Control plane: 3 - Infra: 2 - Compute: 2 Network: - Service CIDR: 172.30.0.0/16 - Machine CIDR: 10.0.0.0/16 - Pod CIDR: 10.128.0.0/14 - Host Prefix: /23 STS Role ARN: arn:aws:iam::000000000000:role/ManagedOpenShift-Installer-Role Support Role ARN: arn:aws:iam::000000000000:role/ManagedOpenShift-Support-Role Instance IAM Roles: - Control plane: arn:aws:iam::000000000000:role/ManagedOpenShift-ControlPlane-Role - Worker: arn:aws:iam::000000000000:role/ManagedOpenShift-Worker-Role Operator IAM Roles: - arn:aws:iam::000000000000:role/my-rosa-cluster-w7i6-openshift-ingress-operator-cloud-credentials - arn:aws:iam::000000000000:role/my-rosa-cluster-w7i6-openshift-cluster-csi-drivers-ebs-cloud-credentials - arn:aws:iam::000000000000:role/my-rosa-cluster-w7i6-openshift-cloud-network-config-controller-cloud-cre - arn:aws:iam::000000000000:role/my-rosa-cluster-openshift-machine-api-aws-cloud-credentials - arn:aws:iam::000000000000:role/my-rosa-cluster-openshift-cloud-credential-operator-cloud-credentia - arn:aws:iam::000000000000:role/my-rosa-cluster-openshift-image-registry-installer-cloud-credential State: waiting (Waiting for OIDC configuration) Private: No Created: Jul 1 2022 22:13:50 UTC Details Page: https://console.redhat.com/openshift/details/s/2BMQm8xz8Hq5yEN000000000000 OIDC Endpoint URL: https://rh-oidc.s3.us-east-1.amazonaws.com/1t6i760dbum4mqltqh6o000000000000 I: Run the following commands to continue the cluster creation: rosa create operator-roles --cluster my-rosa-cluster rosa create oidc-provider --cluster my-rosa-cluster I: To determine when your cluster is Ready, run 'rosa describe cluster -c my-rosa-cluster'. I: To watch your cluster installation logs, run 'rosa logs install -c my-rosa-cluster --watch'.
rosa create operator-roles --mode manual --cluster <cluster-name>
I: Run the following commands to create the operator roles: aws iam create-role --role-name my-rosa-cluster-openshift-image-registry-installer-cloud-credentials --assume-role-policy-document file://operator_image_registry_installer_cloud_credentials_policy.json --tags Key=rosa_cluster_id,Value=1mkesci269png3tck000000000000000 Key=rosa_openshift_version,Value=4.8 Key=rosa_role_prefix,Value= Key=operator_namespace,Value=openshift-image-registry Key=operator_name,Value=installer-cloud-credentials aws iam attach-role-policy --role-name my-rosa-cluster-openshift-image-registry-installer-cloud-credentials --policy-arn arn:aws:iam::000000000000:policy/ManagedOpenShift-openshift-image-registry-installer-cloud-creden [...]
rosa create oidc-provider --mode manual --cluster <cluster-name>
I: Run the following commands to create the OIDC provider: aws iam create-open-id-connect-provider --url https://rh-oidc.s3.us-east-1.amazonaws.com/1mkesci269png3tckknhh0rfs2da5fj9 --client-id-list openshift sts.amazonaws.com --thumbprint-list a9d53002e97e00e043244f3d170d000000000000 aws iam create-open-id-connect-provider --url https://rh-oidc.s3.us-east-1.amazonaws.com/1mkesci269png3tckknhh0rfs2da5fj9 --client-id-list openshift sts.amazonaws.com --thumbprint-list a9d53002e97e00e043244f3d170d000000000000
rosa describe cluster --cluster <cluster-name>
rosa list clusters
rosa describe cluster -c <cluster-name> | grep Console
rosa create account-roles --mode auto --yes
rosa create ocm-role --mode auto --admin --yes
rosa create user-role --mode auto --yes
rosa create account-roles --mode auto --yes
I: Creating roles using 'arn:aws:iam::000000000000:user/rosa-user' I: Created role 'ManagedOpenShift-ControlPlane-Role' with ARN 'arn:aws:iam::000000000000:role/ManagedOpenShift-ControlPlane-Role' I: Created role 'ManagedOpenShift-Worker-Role' with ARN 'arn:aws:iam::000000000000:role/ManagedOpenShift-Worker-Role' I: Created role 'ManagedOpenShift-Support-Role' with ARN 'arn:aws:iam::000000000000:role/ManagedOpenShift-Support-Role' I: Created role 'ManagedOpenShift-Installer-Role' with ARN 'arn:aws:iam::000000000000:role/ManagedOpenShift-Installer-Role' I: Created policy with ARN 'arn:aws:iam::000000000000:policy/ManagedOpenShift-openshift-machine-api-aws-cloud-credentials' I: Created policy with ARN 'arn:aws:iam::000000000000:policy/ManagedOpenShift-openshift-cloud-credential-operator-cloud-crede' I: Created policy with ARN 'arn:aws:iam::000000000000:policy/ManagedOpenShift-openshift-image-registry-installer-cloud-creden' I: Created policy with ARN 'arn:aws:iam::000000000000:policy/ManagedOpenShift-openshift-ingress-operator-cloud-credentials' I: Created policy with ARN 'arn:aws:iam::000000000000:policy/ManagedOpenShift-openshift-cluster-csi-drivers-ebs-cloud-credent' I: To create a cluster with these roles, run the following command: rosa create cluster --sts
rosa list ocm-role
rosa create ocm-role --mode auto --admin --yes
I: Creating ocm role I: Creating role using 'arn:aws:iam::000000000000:user/rosa-user' I: Created role 'ManagedOpenShift-OCM-Role-12561000' with ARN 'arn:aws:iam::000000000000:role/ManagedOpenShift-OCM-Role-12561000' I: Linking OCM role I: Successfully linked role-arn 'arn:aws:iam::000000000000:role/ManagedOpenShift-OCM-Role-12561000' with organization account '1MpZfntsZeUdjWHg7XRgP000000'
rosa create ocm-role --mode manual --admin --yes
rosa create ocm-role --mode auto --yes
rosa list user-role
rosa create user-role --mode auto --yes
I: Creating User role I: Creating ocm user role using 'arn:aws:iam::000000000000:user/rosa-user' I: Created role 'ManagedOpenShift-User-rosa-user-Role' with ARN 'arn:aws:iam::000000000000:role/ManagedOpenShift-User-rosa-user-Role' I: Linking User role I: Successfully linked role ARN 'arn:aws:iam::000000000000:role/ManagedOpenShift-User-rosa-user-Role' with account '1rbOQez0z5j1YolInhcXY000000'
rosa create account-roles --mode auto
rosa create operator-roles --mode auto --cluster <cluster-name> --yes
I: Creating roles using 'arn:aws:iam::000000000000:user/rosauser' I: Created role 'rosacluster-b736-openshift-ingress-operator-cloud-credentials' with ARN 'arn:aws:iam::000000000000:role/rosacluster-b736-openshift-ingress-operator-cloud-credentials' I: Created role 'rosacluster-b736-openshift-cluster-csi-drivers-ebs-cloud-credent' with ARN 'arn:aws:iam::000000000000:role/rosacluster-b736-openshift-cluster-csi-drivers-ebs-cloud-credent' I: Created role 'rosacluster-b736-openshift-cloud-network-config-controller-cloud' with ARN 'arn:aws:iam::000000000000:role/rosacluster-b736-openshift-cloud-network-config-controller-cloud' I: Created role 'rosacluster-b736-openshift-machine-api-aws-cloud-credentials' with ARN 'arn:aws:iam::000000000000:role/rosacluster-b736-openshift-machine-api-aws-cloud-credentials' I: Created role 'rosacluster-b736-openshift-cloud-credential-operator-cloud-crede' with ARN 'arn:aws:iam::000000000000:role/rosacluster-b736-openshift-cloud-credential-operator-cloud-crede' I: Created role 'rosacluster-b736-openshift-image-registry-installer-cloud-creden' with ARN 'arn:aws:iam::000000000000:role/rosacluster-b736-openshift-image-registry-installer-cloud-creden'
rosa create oidc-provider --mode auto --cluster <cluster-name> --yes
I: Creating OIDC provider using 'arn:aws:iam::000000000000:user/rosauser' I: Created OIDC provider with ARN 'arn:aws:iam::000000000000:oidc-provider/rh-oidc.s3.us-east-1.amazonaws.com/1tt4kvrr2kha2rgs8gjfvf0000000000'
rosa list regions --hosted-cp
#!/bin/bash set -e ########## This script will create the network requirements for a ROSA cluster. This will be a public cluster. This creates: - VPC - Public and private subnets - Internet Gateway - Relevant route tables - NAT Gateway # This will automatically use the region configured for the aws cli # ########## VPC_CIDR=10.0.0.0/16 PUBLIC_CIDR_SUBNET=10.0.1.0/24 PRIVATE_CIDR_SUBNET=10.0.0.0/24 Create VPC echo -n "Creating VPC..." VPC_ID=USD(aws ec2 create-vpc --cidr-block USDVPC_CIDR --query Vpc.VpcId --output text) Create tag name aws ec2 create-tags --resources USDVPC_ID --tags Key=Name,Value=USDCLUSTER_NAME Enable dns hostname aws ec2 modify-vpc-attribute --vpc-id USDVPC_ID --enable-dns-hostnames echo "done." Create Public Subnet echo -n "Creating public subnet..." PUBLIC_SUBNET_ID=USD(aws ec2 create-subnet --vpc-id USDVPC_ID --cidr-block USDPUBLIC_CIDR_SUBNET --query Subnet.SubnetId --output text) aws ec2 create-tags --resources USDPUBLIC_SUBNET_ID --tags Key=Name,Value=USDCLUSTER_NAME-public echo "done." Create private subnet echo -n "Creating private subnet..." PRIVATE_SUBNET_ID=USD(aws ec2 create-subnet --vpc-id USDVPC_ID --cidr-block USDPRIVATE_CIDR_SUBNET --query Subnet.SubnetId --output text) aws ec2 create-tags --resources USDPRIVATE_SUBNET_ID --tags Key=Name,Value=USDCLUSTER_NAME-private echo "done." Create an internet gateway for outbound traffic and attach it to the VPC. echo -n "Creating internet gateway..." IGW_ID=USD(aws ec2 create-internet-gateway --query InternetGateway.InternetGatewayId --output text) echo "done." aws ec2 create-tags --resources USDIGW_ID --tags Key=Name,Value=USDCLUSTER_NAME aws ec2 attach-internet-gateway --vpc-id USDVPC_ID --internet-gateway-id USDIGW_ID > /dev/null 2>&1 echo "Attached IGW to VPC." Create a route table for outbound traffic and associate it to the public subnet. echo -n "Creating route table for public subnet..." PUBLIC_ROUTE_TABLE_ID=USD(aws ec2 create-route-table --vpc-id USDVPC_ID --query RouteTable.RouteTableId --output text) aws ec2 create-tags --resources USDPUBLIC_ROUTE_TABLE_ID --tags Key=Name,Value=USDCLUSTER_NAME echo "done." aws ec2 create-route --route-table-id USDPUBLIC_ROUTE_TABLE_ID --destination-cidr-block 0.0.0.0/0 --gateway-id USDIGW_ID > /dev/null 2>&1 echo "Created default public route." aws ec2 associate-route-table --subnet-id USDPUBLIC_SUBNET_ID --route-table-id USDPUBLIC_ROUTE_TABLE_ID > /dev/null 2>&1 echo "Public route table associated" Create a NAT gateway in the public subnet for outgoing traffic from the private network. echo -n "Creating NAT Gateway..." NAT_IP_ADDRESS=USD(aws ec2 allocate-address --domain vpc --query AllocationId --output text) NAT_GATEWAY_ID=USD(aws ec2 create-nat-gateway --subnet-id USDPUBLIC_SUBNET_ID --allocation-id USDNAT_IP_ADDRESS --query NatGateway.NatGatewayId --output text) aws ec2 create-tags --resources USDNAT_IP_ADDRESS --resources USDNAT_GATEWAY_ID --tags Key=Name,Value=USDCLUSTER_NAME sleep 10 echo "done." Create a route table for the private subnet to the NAT gateway. echo -n "Creating a route table for the private subnet to the NAT gateway..." PRIVATE_ROUTE_TABLE_ID=USD(aws ec2 create-route-table --vpc-id USDVPC_ID --query RouteTable.RouteTableId --output text) aws ec2 create-tags --resources USDPRIVATE_ROUTE_TABLE_ID USDNAT_IP_ADDRESS --tags Key=Name,Value=USDCLUSTER_NAME-private aws ec2 create-route --route-table-id USDPRIVATE_ROUTE_TABLE_ID --destination-cidr-block 0.0.0.0/0 --gateway-id USDNAT_GATEWAY_ID > /dev/null 2>&1 aws ec2 associate-route-table --subnet-id USDPRIVATE_SUBNET_ID --route-table-id USDPRIVATE_ROUTE_TABLE_ID > /dev/null 2>&1 echo "done." echo "***********VARIABLE VALUES*********" echo "VPC_ID="USDVPC_ID echo "PUBLIC_SUBNET_ID="USDPUBLIC_SUBNET_ID echo "PRIVATE_SUBNET_ID="USDPRIVATE_SUBNET_ID echo "PUBLIC_ROUTE_TABLE_ID="USDPUBLIC_ROUTE_TABLE_ID echo "PRIVATE_ROUTE_TABLE_ID="USDPRIVATE_ROUTE_TABLE_ID echo "NAT_GATEWAY_ID="USDNAT_GATEWAY_ID echo "IGW_ID="USDIGW_ID echo "NAT_IP_ADDRESS="USDNAT_IP_ADDRESS echo "Setup complete." echo "" echo "To make the cluster create commands easier, please run the following commands to set the environment variables:" echo "export PUBLIC_SUBNET_ID=USDPUBLIC_SUBNET_ID" echo "export PRIVATE_SUBNET_ID=USDPRIVATE_SUBNET_ID"
export PUBLIC_SUBNET_ID=USDPUBLIC_SUBNET_ID export PRIVATE_SUBNET_ID=USDPRIVATE_SUBNET_ID
echo "Public Subnet: USDPUBLIC_SUBNET_ID"; echo "Private Subnet: USDPRIVATE_SUBNET_ID"
Public Subnet: subnet-0faeeeb0000000000 Private Subnet: subnet-011fe340000000000
export OIDC_ID=USD(rosa create oidc-config --mode auto --managed --yes -o json | jq -r '.id')
export CLUSTER_NAME=<cluster_name> export REGION=<VPC_region>
rosa create account-roles --mode auto --yes
rosa create cluster --cluster-name USDCLUSTER_NAME --subnet-ids USD{PUBLIC_SUBNET_ID},USD{PRIVATE_SUBNET_ID} --hosted-cp --region USDREGION --oidc-config-id USDOIDC_ID --sts --mode auto --yes
rosa describe cluster --cluster USDCLUSTER_NAME
rosa list clusters
rosa logs install --cluster USDCLUSTER_NAME --watch
rosa create admin --cluster=<cluster-name>
W: It is recommended to add an identity provider to login to this cluster. See 'rosa create idp --help' for more information. I: Admin account has been added to cluster 'my-rosa-cluster'. It may take up to a minute for the account to become active. I: To login, run the following command: login https://api.my-rosa-cluster.abcd.p1.openshiftapps.com:6443 --username cluster-admin --password FWGYL-2mkJI-00000-00000
oc login https://api.my-rosa-cluster.abcd.p1.openshiftapps.com:6443 > --username cluster-admin > --password FWGYL-2mkJI-00000-00000
Login successful. You have access to 79 projects, the list has been suppressed. You can list all projects with ' projects' Using project "default".
oc whoami
cluster-admin
get all -n openshift-apiserver
rosa create idp --help
rosa create idp --cluster=<cluster name> --interactive
Type of identity provider: github Identity Provider Name: <IDP-name> Restrict to members of: organizations GitHub organizations: <organization-account-name>
rosa grant user cluster-admin --user <idp_user_name> --cluster=<cluster-name>
rosa grant user dedicated-admin --user <idp_user_name> --cluster=<cluster-name>
rosa list users --cluster=<cluster-name>
rosa list users --cluster=my-rosa-cluster ID GROUPS <idp_user_name> cluster-admins
oc get all -n openshift-apiserver
oc login --token=sha256~GBAfS4JQ0t1UTKYHbWAK6OUWGUkdMGz000000000000 --server=https://api.my-rosa-cluster.abcd.p1.openshiftapps.com:6443
Logged into "https://api.my-rosa-cluster.abcd.p1.openshiftapps.com:6443" as "rosa-user" using the token provided. You have access to 79 projects, the list has been suppressed. You can list all projects with ' projects' Using project "default".
oc whoami
rosa-user
rosa describe cluster -c <cluster-name> | grep Console
rosa create machinepool --cluster=<cluster-name> --name=<machinepool-name> --replicas=<number-nodes>
rosa create machinepool --cluster=my-rosa-cluster --name=new-mp --replicas=2
I: Machine pool 'new-mp' created successfully on cluster 'my-rosa-cluster' I: To view all machine pools, run 'rosa list machinepools -c my-rosa-cluster'
rosa create machinepool --cluster=<cluster-name> --name=<machinepool-name> --replicas=<number-nodes> --labels=`<key=pair>`
rosa create machinepool --cluster=my-rosa-cluster --name=db-nodes-mp --replicas=2 --labels='app=db','tier=backend'
I: Machine pool 'db-nodes-mp' created successfully on cluster 'my-rosa-cluster'
rosa list machinepools --cluster=<cluster-name>
ID AUTOSCALING REPLICAS INSTANCE TYPE LABELS TAINTS AVAILABILITY ZONES Default No 2 m5.xlarge us-east-1a
rosa list machinepools --cluster=<cluster-name>
ID AUTOSCALING REPLICAS INSTANCE TYPE LABELS TAINTS AVAILABILITY ZONES Default No 2 m5.xlarge us-east-1a
rosa edit machinepool --cluster=<cluster-name> --replicas=<number-nodes> <machinepool-name>
rosa edit machinepool --cluster=my-rosa-cluster --replicas 3 Default
rosa describe cluster --cluster=<cluster-name> | grep Compute
rosa describe cluster --cluster=my-rosa-cluster | grep Compute
- Compute: 3 (m5.xlarge)
rosa edit machinepool --cluster=<cluster-name> --replicas=<number-nodes> --labels='key=value' <machinepool-name>
rosa edit machinepool --cluster=my-rosa-cluster --replicas=2 --labels 'foo=bar','baz=one' new-mp
rosa create machinepool --cluster=<cluster-name> --name=<mp-name> --replicas=<number-nodes> --labels='<key=pair>' --instance-type=<type>
rosa create machinepool --cluster=my-rosa-cluster --name=db-nodes-large-mp --replicas=2 --labels='app=db','tier=backend' --instance-type=m5.2xlarge
rosa list instance-types
rosa create machinepool -c <cluster-name> --interactive
rosa list machinepools -c <cluster-name>
rosa list machinepools -c <cluster-name>
ID AUTOSCALING REPLICAS INSTANCE TYPE LABELS TAINTS AVAILABILITY ZONES Default No 2 m5.xlarge us-east-1a
rosa edit machinepool -c <cluster-name> --enable-autoscaling <machinepool-name> --min-replicas=<num> --max-replicas=<num>
rosa edit machinepool -c my-rosa-cluster --enable-autoscaling Default --min-replicas=2 --max-replicas=4
rosa list machinepools -c <cluster-name>
ID AUTOSCALING REPLICAS INSTANCE TYPE LABELS TAINTS AVAILABILITY ZONES Default Yes 2-4 m5.xlarge us-east-1a
rosa upgrade cluster --help
rosa list upgrade -c <cluster-name>
rosa list upgrade -c <cluster-name> VERSION NOTES 4.14.7 recommended 4.14.6
rosa upgrade cluster -c <cluster-name> --version <desired-version>
rosa upgrade cluster -c <cluster-name> --version <desired-version> --schedule-date <future-date-for-update> --schedule-time <future-time-for-update>
rosa list clusters
rosa delete cluster --cluster <cluster-name>
rosa list clusters
rosa delete oidc-provider -c <clusterID> --mode auto --yes
rosa delete operator-roles -c <clusterID> --mode auto --yes
rosa delete account-roles --prefix <prefix> --mode auto --yes
rosa create ocm-role
rosa create user-role
apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ostoy-pvc spec: accessModes: - ReadWriteOnce resources: requests: storage: 1Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: ostoy-frontend labels: app: ostoy spec: selector: matchLabels: app: ostoy-frontend strategy: type: Recreate replicas: 1 template: metadata: labels: app: ostoy-frontend spec: # Uncomment to use with ACK portion of the workshop # If you chose a different service account name please replace it. # serviceAccount: ostoy-sa containers: - name: ostoy-frontend securityContext: allowPrivilegeEscalation: false runAsNonRoot: true seccompProfile: type: RuntimeDefault capabilities: drop: - ALL image: quay.io/ostoylab/ostoy-frontend:1.6.0 imagePullPolicy: IfNotPresent ports: - name: ostoy-port containerPort: 8080 resources: requests: memory: "256Mi" cpu: "100m" limits: memory: "512Mi" cpu: "200m" volumeMounts: - name: configvol mountPath: /var/config - name: secretvol mountPath: /var/secret - name: datavol mountPath: /var/demo_files livenessProbe: httpGet: path: /health port: 8080 initialDelaySeconds: 10 periodSeconds: 5 env: - name: ENV_TOY_SECRET valueFrom: secretKeyRef: name: ostoy-secret-env key: ENV_TOY_SECRET - name: MICROSERVICE_NAME value: OSTOY_MICROSERVICE_SVC - name: NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace volumes: - name: configvol configMap: name: ostoy-configmap-files - name: secretvol secret: defaultMode: 420 secretName: ostoy-secret - name: datavol persistentVolumeClaim: claimName: ostoy-pvc --- apiVersion: v1 kind: Service metadata: name: ostoy-frontend-svc labels: app: ostoy-frontend spec: type: ClusterIP ports: - port: 8080 targetPort: ostoy-port protocol: TCP name: ostoy selector: app: ostoy-frontend --- apiVersion: route.openshift.io/v1 kind: Route metadata: name: ostoy-route spec: to: kind: Service name: ostoy-frontend-svc --- apiVersion: v1 kind: Secret metadata: name: ostoy-secret-env type: Opaque data: ENV_TOY_SECRET: VGhpcyBpcyBhIHRlc3Q= --- kind: ConfigMap apiVersion: v1 metadata: name: ostoy-configmap-files data: config.json: '{ "default": "123" }' --- apiVersion: v1 kind: Secret metadata: name: ostoy-secret data: secret.txt: VVNFUk5BTUU9bXlfdXNlcgpQQVNTV09SRD1AT3RCbCVYQXAhIzYzMlk1RndDQE1UUWsKU01UUD1sb2NhbGhvc3QKU01UUF9QT1JUPTI1 type: Opaque
apiVersion: apps/v1 kind: Deployment metadata: name: ostoy-microservice labels: app: ostoy spec: selector: matchLabels: app: ostoy-microservice replicas: 1 template: metadata: labels: app: ostoy-microservice spec: containers: - name: ostoy-microservice securityContext: allowPrivilegeEscalation: false runAsNonRoot: true seccompProfile: type: RuntimeDefault capabilities: drop: - ALL image: quay.io/ostoylab/ostoy-microservice:1.5.0 imagePullPolicy: IfNotPresent ports: - containerPort: 8080 protocol: TCP resources: requests: memory: "128Mi" cpu: "50m" limits: memory: "256Mi" cpu: "100m" --- apiVersion: v1 kind: Service metadata: name: ostoy-microservice-svc labels: app: ostoy-microservice spec: type: ClusterIP ports: - port: 8080 targetPort: 8080 protocol: TCP selector: app: ostoy-microservice
apiVersion: s3.services.k8s.aws/v1alpha1 kind: Bucket metadata: name: ostoy-bucket namespace: ostoy spec: name: ostoy-bucket
oc login --token=<your_token> --server=https://api.osd4-demo.abc1.p1.openshiftapps.com:6443 Logged into "https://api.myrosacluster.abcd.p1.openshiftapps.com:6443" as "rosa-user" using the token provided. You don't have any projects. You can try to create a new project, by running new-project <project name>
oc new-project ostoy
Now using project "ostoy" on server "https://api.myrosacluster.abcd.p1.openshiftapps.com:6443".
oc new-project ostoy-USD(uuidgen | cut -d - -f 2 | tr '[:upper:]' '[:lower:]')
oc apply -f https://raw.githubusercontent.com/openshift-cs/rosaworkshop/master/rosa-workshop/ostoy/yaml/ostoy-microservice-deployment.yaml
oc apply -f https://raw.githubusercontent.com/openshift-cs/rosaworkshop/master/rosa-workshop/ostoy/yaml/ostoy-microservice-deployment.yaml deployment.apps/ostoy-microservice created service/ostoy-microservice-svc created
oc apply -f https://raw.githubusercontent.com/openshift-cs/rosaworkshop/master/rosa-workshop/ostoy/yaml/ostoy-frontend-deployment.yaml
persistentvolumeclaim/ostoy-pvc created deployment.apps/ostoy-frontend created service/ostoy-frontend-svc created route.route.openshift.io/ostoy-route created configmap/ostoy-configmap-env created secret/ostoy-secret-env created configmap/ostoy-configmap-files created secret/ostoy-secret created
oc get route
NAME HOST/PORT PATH SERVICES PORT TERMINATION WILDCARD ostoy-route ostoy-route-ostoy.apps.<your-rosa-cluster>.abcd.p1.openshiftapps.com ostoy-frontend-svc <all> None
oc get pods
oc rsh <pod_name>
cd /var/demo_files
ls
cat test-pv.txt
oc get pods NAME READY STATUS RESTARTS AGE ostoy-frontend-5fc8d486dc-wsw24 1/1 Running 0 18m ostoy-microservice-6cf764974f-hx4qm 1/1 Running 0 18m oc rsh ostoy-frontend-5fc8d486dc-wsw24 cd /var/demo_files/ ls lost+found test-pv.txt cat test-pv.txt OpenShift is the greatest thing since sliced bread!
kind: ConfigMap apiVersion: v1 metadata: name: ostoy-configmap-files data: config.json: '{ "default": "123" }'
USERNAME=my_user PASSWORD=VVNFUk5BTUU9bXlfdXNlcgpQQVNTV09SRD1AT3RCbCVYQXAhIzYzMlk1RndDQE1UUWsKU01UUD1sb2NhbGhvc3QKU01UUF9QT1JUPTI1 SMTP=localhost SMTP_PORT=25
{ "npm_config_local_prefix": "/opt/app-root/src", "STI_SCRIPTS_PATH": "/usr/libexec/s2i", "npm_package_version": "1.7.0", "APP_ROOT": "/opt/app-root", "NPM_CONFIG_PREFIX": "/opt/app-root/src/.npm-global", "OSTOY_MICROSERVICE_PORT_8080_TCP_PORT": "8080", "NODE": "/usr/bin/node", "LD_PRELOAD": "libnss_wrapper.so", "KUBERNETES_SERVICE_HOST": "172.30.0.1", "OSTOY_MICROSERVICE_PORT": "tcp://172.30.60.255:8080", "OSTOY_PORT": "tcp://172.30.152.25:8080", "npm_package_name": "ostoy", "OSTOY_SERVICE_PORT_8080_TCP": "8080", "_": "/usr/bin/node" "ENV_TOY_CONFIGMAP": "ostoy-configmap -env" }
oc get service <name_of_service> -o yaml
apiVersion: v1 kind: Service metadata: name: ostoy-microservice-svc labels: app: ostoy-microservice spec: type: ClusterIP ports: - port: 8080 targetPort: 8080 protocol: TCP selector: app: ostoy-microservice
oc get pods
NAME READY STATUS RESTARTS AGE ostoy-frontend-679cb85695-5cn7x 1/1 Running 0 1h ostoy-microservice-86b4c6f559-p594d 1/1 Running 0 1h
spec: selector: matchLabels: app: ostoy-microservice replicas: 3
oc apply -f ostoy-microservice-deployment.yaml
oc get pods
NAME READY STATUS RESTARTS AGE ostoy-frontend-5fbcc7d9-rzlgz 1/1 Running 0 26m ostoy-microservice-6666dcf455-2lcv4 1/1 Running 0 81s ostoy-microservice-6666dcf455-5z56w 1/1 Running 0 81s ostoy-microservice-6666dcf455-tqzmn 1/1 Running 0 26m
oc scale deployment ostoy-microservice --replicas=2
oc get pods
NAME READY STATUS RESTARTS AGE ostoy-frontend-5fbcc7d9-rzlgz 1/1 Running 0 75m ostoy-microservice-6666dcf455-2lcv4 1/1 Running 0 50m ostoy-microservice-6666dcf455-tqzmn 1/1 Running 0 75m
oc autoscale deployment/ostoy-microservice --cpu-percent=80 --min=1 --max=10
get pods --field-selector=status.phase=Running | grep microservice
ostoy-microservice-79894f6945-cdmbd 1/1 Running 0 3m14s ostoy-microservice-79894f6945-mgwk7 1/1 Running 0 4h24m ostoy-microservice-79894f6945-q925d 1/1 Running 0 3m14s
oc new-project autoscale-ex
oc create -f https://raw.githubusercontent.com/openshift-cs/rosaworkshop/master/rosa-workshop/ostoy/yaml/job-work-queue.yaml
oc get pods
NAME READY STATUS RESTARTS AGE work-queue-5x2nq-24xxn 0/1 Pending 0 10s work-queue-5x2nq-57zpt 0/1 Pending 0 10s work-queue-5x2nq-58bvs 0/1 Pending 0 10s work-queue-5x2nq-6c5tl 1/1 Running 0 10s work-queue-5x2nq-7b84p 0/1 Pending 0 10s work-queue-5x2nq-7hktm 0/1 Pending 0 10s work-queue-5x2nq-7md52 0/1 Pending 0 10s work-queue-5x2nq-7qgmp 0/1 Pending 0 10s work-queue-5x2nq-8279r 0/1 Pending 0 10s work-queue-5x2nq-8rkj2 0/1 Pending 0 10s work-queue-5x2nq-96cdl 0/1 Pending 0 10s work-queue-5x2nq-96tfr 0/1 Pending 0 10s
oc get nodes
NAME STATUS ROLES AGE VERSION ip-10-0-138-106.us-west-2.compute.internal Ready infra,worker 22h v1.23.5+3afdacb ip-10-0-153-68.us-west-2.compute.internal Ready worker 2m12s v1.23.5+3afdacb ip-10-0-165-183.us-west-2.compute.internal Ready worker 2m8s v1.23.5+3afdacb ip-10-0-176-123.us-west-2.compute.internal Ready infra,worker 22h v1.23.5+3afdacb ip-10-0-195-210.us-west-2.compute.internal Ready master 23h v1.23.5+3afdacb ip-10-0-196-84.us-west-2.compute.internal Ready master 23h v1.23.5+3afdacb ip-10-0-203-104.us-west-2.compute.internal Ready worker 2m6s v1.23.5+3afdacb ip-10-0-217-202.us-west-2.compute.internal Ready master 23h v1.23.5+3afdacb ip-10-0-225-141.us-west-2.compute.internal Ready worker 23h v1.23.5+3afdacb ip-10-0-231-245.us-west-2.compute.internal Ready worker 2m11s v1.23.5+3afdacb ip-10-0-245-27.us-west-2.compute.internal Ready worker 2m8s v1.23.5+3afdacb ip-10-0-245-7.us-west-2.compute.internal Ready worker 23h v1.23.5+3afdacb
oc project ostoy
curl https://raw.githubusercontent.com/openshift-cs/rosaworkshop/master/rosa-workshop/ostoy/resources/configure-cloudwatch.sh | bash
Varaibles are set...ok. Policy already exists...ok. Created RosaCloudWatch-mycluster role. Attached role policy. Deploying the Red Hat OpenShift Logging Operator namespace/openshift-logging configured operatorgroup.operators.coreos.com/cluster-logging created subscription.operators.coreos.com/cluster-logging created Waiting for Red Hat OpenShift Logging Operator deployment to complete Red Hat OpenShift Logging Operator deployed. secret/cloudwatch-credentials created clusterlogforwarder.logging.openshift.io/instance created clusterlogging.logging.openshift.io/instance created Complete.
aws logs describe-log-groups --log-group-name-prefix rosa-mycluster
{ "logGroups": [ { "logGroupName": "rosa-mycluster.application", "creationTime": 1724104537717, "metricFilterCount": 0, "arn": "arn:aws:logs:us-west-2:000000000000:log-group:rosa-mycluster.application:*", "storedBytes": 0, "logGroupClass": "STANDARD", "logGroupArn": "arn:aws:logs:us-west-2:000000000000:log-group:rosa-mycluster.application" }, { "logGroupName": "rosa-mycluster.audit", "creationTime": 1724104152968, "metricFilterCount": 0, "arn": "arn:aws:logs:us-west-2:000000000000:log-group:rosa-mycluster.audit:*", "storedBytes": 0, "logGroupClass": "STANDARD", "logGroupArn": "arn:aws:logs:us-west-2:000000000000:log-group:rosa-mycluster.audit" },
oc get pods -o name
pod/ostoy-frontend-679cb85695-5cn7x 1 pod/ostoy-microservice-86b4c6f559-p594d
oc logs <pod-name>
oc logs ostoy-frontend-679cb85695-5cn7x [...] ostoy-frontend-679cb85695-5cn7x: server starting on port 8080 Redirecting to /home stdout: All is well! stderr: Oh no! Error!
oc login --token=RYhFlXXXXXXXXXXXX --server=https://api.osd4-demo.abc1.p1.openshiftapps.com:6443
Logged into "https://api.myrosacluster.abcd.p1.openshiftapps.com:6443" as "rosa-user" using the token provided. You don't have any projects. You can try to create a new project, by running new-project <project name>
oc new-project ostoy-s2i
oc create -f https://raw.githubusercontent.com/<UserName>/ostoy/master/deployment/yaml/secret.yaml
oc create -f https://raw.githubusercontent.com/<UserName>/ostoy/master/deployment/yaml/configmap.yaml
oc new-app https://github.com/<UserName>/ostoy --context-dir=microservice --name=ostoy-microservice --labels=app=ostoy
--> Creating resources with label app=ostoy imagestream.image.openshift.io "ostoy-microservice" created buildconfig.build.openshift.io "ostoy-microservice" created deployment.apps "ostoy-microservice" created service "ostoy-microservice" created --> Success Build scheduled, use 'oc logs -f buildconfig/ostoy-microservice' to track its progress. Application is not exposed. You can expose services to the outside world by executing one or more of the commands below: 'oc expose service/ostoy-microservice' Run 'oc status' to view your app.
oc status
In project ostoy-s2i on server https://api.myrosacluster.g14t.p1.openshiftapps.com:6443 svc/ostoy-microservice - 172.30.47.74:8080 dc/ostoy-microservice deploys istag/ostoy-microservice:latest <- bc/ostoy-microservice source builds https://github.com/UserName/ostoy on openshift/nodejs:14-ubi8 deployment #1 deployed 34 seconds ago - 1 pod
oc new-app https://github.com/<UserName>/ostoy --env=MICROSERVICE_NAME=OSTOY_MICROSERVICE
--> Creating resources imagestream.image.openshift.io "ostoy" created buildconfig.build.openshift.io "ostoy" created deployment.apps "ostoy" created service "ostoy" created --> Success Build scheduled, use 'oc logs -f buildconfig/ostoy' to track its progress. Application is not exposed. You can expose services to the outside world by executing one or more of the commands below: 'oc expose service/ostoy' Run 'oc status' to view your app.
oc patch deployment ostoy --type=json -p '[{"op": "replace", "path": "/spec/strategy/type", "value": "Recreate"}, {"op": "remove", "path": "/spec/strategy/rollingUpdate"}]'
oc set probe deployment ostoy --liveness --get-url=http://:8080/health
oc set volume deployment ostoy --add --secret-name=ostoy-secret --mount-path=/var/secret
oc set volume deployment ostoy --add --configmap-name=ostoy-config -m /var/config
oc set volume deployment ostoy --add --type=pvc --claim-size=1G -m /var/demo_files
oc create route edge --service=ostoy --insecure-policy=Redirect
python -m webbrowser "USD(oc get route ostoy -o template --template='https://{{.spec.host}}')"
oc get route
oc get bc/ostoy-microservice -o=jsonpath='{.spec.triggers..github.secret}'
`o_3x9M1qoI2Wj_cz1WiK`
oc describe bc/ostoy-microservice
[...] Webhook GitHub: URL: https://api.demo1234.openshift.com:443/apis/build.openshift.io/v1/namespaces/ostoy-s2i/buildconfigs/ostoy/webhooks/<secret>/github [...]
https://api.demo1234.openshift.com:443/apis/build.openshift.io/v1/namespaces/ostoy-s2i/buildconfigs/ostoy-microservice/webhooks/o_3x9M1qoI2Wj_czR1WiK/github
7 app.get('/', function(request, response) { 8 //let randomColor = getRandomColor(); // <-- comment this 9 let randomColor = getRandomGrayScaleColor(); // <-- uncomment this 10 11 response.writeHead(200, {'Content-Type': 'application/json'});
curl https://raw.githubusercontent.com/openshift-cs/rosaworkshop/master/rosa-workshop/ostoy/resources/setup-s3-ack-controller.sh | bash
oc describe pod ack-s3-controller -n ack-system | grep "^\s*AWS_"
AWS_ROLE_ARN: arn:aws:iam::000000000000:role/ack-s3-controller AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token
oc rollout restart deployment ack-s3-controller -n ack-system
oc new-project ostoy-USD(uuidgen | cut -d - -f 2 | tr '[:upper:]' '[:lower:]')
export OSTOY_NAMESPACE=USD(oc config view --minify -o 'jsonpath={..namespace}')
export AWS_ACCOUNT_ID=USD(aws sts get-caller-identity --query Account --output text)
export OIDC_PROVIDER=USD(rosa describe cluster -c <cluster-name> -o yaml | awk '/oidc_endpoint_url/ {print USD2}' | cut -d '/' -f 3,4)
cat <<EOF > ./ostoy-sa-trust.json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Federated": "arn:aws:iam::USD{AWS_ACCOUNT_ID}:oidc-provider/USD{OIDC_PROVIDER}" }, "Action": "sts:AssumeRoleWithWebIdentity", "Condition": { "StringEquals": { "USD{OIDC_PROVIDER}:sub": "system:serviceaccount:USD{OSTOY_NAMESPACE}:ostoy-sa" } } } ] } EOF
aws iam create-role --role-name "ostoy-sa-role" --assume-role-policy-document file://ostoy-sa-trust.json
export POLICY_ARN=USD(aws iam list-policies --query 'Policies[?PolicyName==`AmazonS3FullAccess`].Arn' --output text)
aws iam attach-role-policy --role-name "ostoy-sa-role" --policy-arn "USD{POLICY_ARN}"
export APP_IAM_ROLE_ARN=USD(aws iam get-role --role-name=ostoy-sa-role --query Role.Arn --output text)
cat <<EOF | oc apply -f - apiVersion: v1 kind: ServiceAccount metadata: name: ostoy-sa namespace: USD{OSTOY_NAMESPACE} annotations: eks.amazonaws.com/role-arn: "USDAPP_IAM_ROLE_ARN" EOF
oc adm policy add-scc-to-user restricted system:serviceaccount:USD{OSTOY_NAMESPACE}:ostoy-sa
oc describe serviceaccount ostoy-sa -n USD{OSTOY_NAMESPACE}
Name: ostoy-sa Namespace: ostoy Labels: <none> Annotations: eks.amazonaws.com/role-arn: arn:aws:iam::000000000000:role/ostoy-sa-role Image pull secrets: ostoy-sa-dockercfg-b2l94 Mountable secrets: ostoy-sa-dockercfg-b2l94 Tokens: ostoy-sa-token-jlc6d Events: <none>
cat <<EOF | oc apply -f - apiVersion: s3.services.k8s.aws/v1alpha1 kind: Bucket metadata: name: USD{OSTOY_NAMESPACE}-bucket namespace: USD{OSTOY_NAMESPACE} spec: name: USD{OSTOY_NAMESPACE}-bucket EOF
aws s3 ls | grep USD{OSTOY_NAMESPACE}-bucket
- oc apply -f https://raw.githubusercontent.com/openshift-cs/rosaworkshop/master/rosa-workshop/ostoy/yaml/ostoy-microservice-deployment.yaml
- oc apply -f https://raw.githubusercontent.com/openshift-cs/rosaworkshop/master/rosa-workshop/ostoy/yaml/ostoy-frontend-deployment.yaml
oc patch deploy ostoy-frontend -n USD{OSTOY_NAMESPACE} --type=merge --patch '{"spec": {"template": {"spec":{"serviceAccount":"ostoy-sa"}}}}'
spec: # Uncomment to use with ACK portion of the workshop # If you chose a different service account name please replace it. serviceAccount: ostoy-sa containers: - name: ostoy-frontend image: quay.io/ostoylab/ostoy-frontend:1.6.0 imagePullPolicy: IfNotPresent [...]
oc describe pod ostoy-frontend -n USD{OSTOY_NAMESPACE} | grep "^\s*AWS_"
AWS_ROLE_ARN: arn:aws:iam::000000000000:role/ostoy-sa AWS_WEB_IDENTITY_TOKEN_FILE: /var/run/secrets/eks.amazonaws.com/serviceaccount/token
oc get route ostoy-route -n USD{OSTOY_NAMESPACE} -o jsonpath='{.spec.host}{"\n"}'
aws s3 ls s3://USD{OSTOY_NAMESPACE}-bucket
aws s3 ls s3://ostoy-bucket 2023-05-04 22:20:51 51 OSToy.txt