High Availability Guide Red Hat build of Keycloak 26.0 Red Hat Customer Content Services

Commands:
aws ec2 create-vpc --cidr-block 192.168.0.0/16 --tag-specifications "ResourceType=vpc, Tags=[{Key=AuroraCluster,Value=keycloak-aurora}]" \ 1 --region eu-west-1
{ "Vpc": { "CidrBlock": "192.168.0.0/16", "DhcpOptionsId": "dopt-0bae7798158bc344f", "State": "pending", "VpcId": "vpc-0b40bd7c59dbe4277", "OwnerId": "606671647913", "InstanceTenancy": "default", "Ipv6CidrBlockAssociationSet": [], "CidrBlockAssociationSet": [ { "AssociationId": "vpc-cidr-assoc-09a02a83059ba5ab6", "CidrBlock": "192.168.0.0/16", "CidrBlockState": { "State": "associated" } } ], "IsDefault": false } }
aws ec2 create-subnet --availability-zone "eu-west-1a" --vpc-id vpc-0b40bd7c59dbe4277 --cidr-block 192.168.0.0/19 --region eu-west-1
{ "Subnet": { "AvailabilityZone": "eu-west-1a", "AvailabilityZoneId": "euw1-az3", "AvailableIpAddressCount": 8187, "CidrBlock": "192.168.0.0/19", "DefaultForAz": false, "MapPublicIpOnLaunch": false, "State": "available", "SubnetId": "subnet-0d491a1a798aa878d", "VpcId": "vpc-0b40bd7c59dbe4277", "OwnerId": "606671647913", "AssignIpv6AddressOnCreation": false, "Ipv6CidrBlockAssociationSet": [], "SubnetArn": "arn:aws:ec2:eu-west-1:606671647913:subnet/subnet-0d491a1a798aa878d", "EnableDns64": false, "Ipv6Native": false, "PrivateDnsNameOptionsOnLaunch": { "HostnameType": "ip-name", "EnableResourceNameDnsARecord": false, "EnableResourceNameDnsAAAARecord": false } } }
aws ec2 create-subnet --availability-zone "eu-west-1b" --vpc-id vpc-0b40bd7c59dbe4277 --cidr-block 192.168.32.0/19 --region eu-west-1
{ "Subnet": { "AvailabilityZone": "eu-west-1b", "AvailabilityZoneId": "euw1-az1", "AvailableIpAddressCount": 8187, "CidrBlock": "192.168.32.0/19", "DefaultForAz": false, "MapPublicIpOnLaunch": false, "State": "available", "SubnetId": "subnet-057181b1e3728530e", "VpcId": "vpc-0b40bd7c59dbe4277", "OwnerId": "606671647913", "AssignIpv6AddressOnCreation": false, "Ipv6CidrBlockAssociationSet": [], "SubnetArn": "arn:aws:ec2:eu-west-1:606671647913:subnet/subnet-057181b1e3728530e", "EnableDns64": false, "Ipv6Native": false, "PrivateDnsNameOptionsOnLaunch": { "HostnameType": "ip-name", "EnableResourceNameDnsARecord": false, "EnableResourceNameDnsAAAARecord": false } } }
aws ec2 describe-route-tables --filters Name=vpc-id,Values=vpc-0b40bd7c59dbe4277 --region eu-west-1
{ "RouteTables": [ { "Associations": [ { "Main": true, "RouteTableAssociationId": "rtbassoc-02dfa06f4c7b4f99a", "RouteTableId": "rtb-04a644ad3cd7de351", "AssociationState": { "State": "associated" } } ], "PropagatingVgws": [], "RouteTableId": "rtb-04a644ad3cd7de351", "Routes": [ { "DestinationCidrBlock": "192.168.0.0/16", "GatewayId": "local", "Origin": "CreateRouteTable", "State": "active" } ], "Tags": [], "VpcId": "vpc-0b40bd7c59dbe4277", "OwnerId": "606671647913" } ] }
aws ec2 associate-route-table --route-table-id rtb-04a644ad3cd7de351 --subnet-id subnet-0d491a1a798aa878d --region eu-west-1
aws ec2 associate-route-table --route-table-id rtb-04a644ad3cd7de351 --subnet-id subnet-057181b1e3728530e --region eu-west-1
aws rds create-db-subnet-group --db-subnet-group-name keycloak-aurora-subnet-group --db-subnet-group-description "Aurora DB Subnet Group" --subnet-ids subnet-0d491a1a798aa878d subnet-057181b1e3728530e --region eu-west-1
aws ec2 create-security-group --group-name keycloak-aurora-security-group --description "Aurora DB Security Group" --vpc-id vpc-0b40bd7c59dbe4277 --region eu-west-1
GroupId: sg-0d746cc8ad8d2e63b
aws rds create-db-cluster --db-cluster-identifier keycloak-aurora --database-name keycloak --engine aurora-postgresql --engine-version USD{properties["aurora-postgresql.version"]} --master-username keycloak --master-user-password secret99 --vpc-security-group-ids sg-0d746cc8ad8d2e63b --db-subnet-group-name keycloak-aurora-subnet-group --region eu-west-1
{ "DBCluster": { "AllocatedStorage": 1, "AvailabilityZones": [ "eu-west-1b", "eu-west-1c", "eu-west-1a" ], "BackupRetentionPeriod": 1, "DatabaseName": "keycloak", "DBClusterIdentifier": "keycloak-aurora", "DBClusterParameterGroup": "default.aurora-postgresql15", "DBSubnetGroup": "keycloak-aurora-subnet-group", "Status": "creating", "Endpoint": "keycloak-aurora.cluster-clhthfqe0h8p.eu-west-1.rds.amazonaws.com", "ReaderEndpoint": "keycloak-aurora.cluster-ro-clhthfqe0h8p.eu-west-1.rds.amazonaws.com", "MultiAZ": false, "Engine": "aurora-postgresql", "EngineVersion": "15.5", "Port": 5432, "MasterUsername": "keycloak", "PreferredBackupWindow": "02:21-02:51", "PreferredMaintenanceWindow": "fri:03:34-fri:04:04", "ReadReplicaIdentifiers": [], "DBClusterMembers": [], "VpcSecurityGroups": [ { "VpcSecurityGroupId": "sg-0d746cc8ad8d2e63b", "Status": "active" } ], "HostedZoneId": "Z29XKXDKYMONMX", "StorageEncrypted": false, "DbClusterResourceId": "cluster-IBWXUWQYM3MS5BH557ZJ6ZQU4I", "DBClusterArn": "arn:aws:rds:eu-west-1:606671647913:cluster:keycloak-aurora", "AssociatedRoles": [], "IAMDatabaseAuthenticationEnabled": false, "ClusterCreateTime": "2023-11-01T10:40:45.964000+00:00", "EngineMode": "provisioned", "DeletionProtection": false, "HttpEndpointEnabled": false, "CopyTagsToSnapshot": false, "CrossAccountClone": false, "DomainMemberships": [], "TagList": [], "AutoMinorVersionUpgrade": true, "NetworkType": "IPV4" } }
aws rds create-db-instance --db-cluster-identifier keycloak-aurora --db-instance-identifier "keycloak-aurora-instance-1" --db-instance-class db.t4g.large --engine aurora-postgresql --region eu-west-1
aws rds create-db-instance --db-cluster-identifier keycloak-aurora --db-instance-identifier "keycloak-aurora-instance-2" --db-instance-class db.t4g.large --engine aurora-postgresql --region eu-west-1
aws rds wait db-instance-available --db-instance-identifier keycloak-aurora-instance-1 --region eu-west-1 aws rds wait db-instance-available --db-instance-identifier keycloak-aurora-instance-2 --region eu-west-1
aws rds describe-db-clusters --db-cluster-identifier keycloak-aurora --query 'DBClusters[*].Endpoint' --region eu-west-1 --output text
keycloak-aurora.cluster-clhthfqe0h8p.eu-west-1.rds.amazonaws.com
aws ec2 describe-vpcs --filters "Name=tag:AuroraCluster,Values=keycloak-aurora" --query 'Vpcs[*].VpcId' --region eu-west-1 --output text
vpc-0b40bd7c59dbe4277
NODE=USD(oc get nodes --selector=node-role.kubernetes.io/worker -o jsonpath='{.items[0].metadata.name}') aws ec2 describe-instances --filters "Name=private-dns-name,Values=USD{NODE}" --query 'Reservations[0].Instances[0].VpcId' --region eu-west-1 --output text
vpc-0b721449398429559
aws ec2 create-vpc-peering-connection --vpc-id vpc-0b721449398429559 \ 1 --peer-vpc-id vpc-0b40bd7c59dbe4277 \ 2 --peer-region eu-west-1 --region eu-west-1
{ "VpcPeeringConnection": { "AccepterVpcInfo": { "OwnerId": "606671647913", "VpcId": "vpc-0b40bd7c59dbe4277", "Region": "eu-west-1" }, "ExpirationTime": "2023-11-08T13:26:30+00:00", "RequesterVpcInfo": { "CidrBlock": "10.0.17.0/24", "CidrBlockSet": [ { "CidrBlock": "10.0.17.0/24" } ], "OwnerId": "606671647913", "PeeringOptions": { "AllowDnsResolutionFromRemoteVpc": false, "AllowEgressFromLocalClassicLinkToRemoteVpc": false, "AllowEgressFromLocalVpcToRemoteClassicLink": false }, "VpcId": "vpc-0b721449398429559", "Region": "eu-west-1" }, "Status": { "Code": "initiating-request", "Message": "Initiating Request to 606671647913" }, "Tags": [], "VpcPeeringConnectionId": "pcx-0cb23d66dea3dca9f" } }
aws ec2 wait vpc-peering-connection-exists --vpc-peering-connection-ids pcx-0cb23d66dea3dca9f
aws ec2 accept-vpc-peering-connection --vpc-peering-connection-id pcx-0cb23d66dea3dca9f --region eu-west-1
{ "VpcPeeringConnection": { "AccepterVpcInfo": { "CidrBlock": "192.168.0.0/16", "CidrBlockSet": [ { "CidrBlock": "192.168.0.0/16" } ], "OwnerId": "606671647913", "PeeringOptions": { "AllowDnsResolutionFromRemoteVpc": false, "AllowEgressFromLocalClassicLinkToRemoteVpc": false, "AllowEgressFromLocalVpcToRemoteClassicLink": false }, "VpcId": "vpc-0b40bd7c59dbe4277", "Region": "eu-west-1" }, "RequesterVpcInfo": { "CidrBlock": "10.0.17.0/24", "CidrBlockSet": [ { "CidrBlock": "10.0.17.0/24" } ], "OwnerId": "606671647913", "PeeringOptions": { "AllowDnsResolutionFromRemoteVpc": false, "AllowEgressFromLocalClassicLinkToRemoteVpc": false, "AllowEgressFromLocalVpcToRemoteClassicLink": false }, "VpcId": "vpc-0b721449398429559", "Region": "eu-west-1" }, "Status": { "Code": "provisioning", "Message": "Provisioning" }, "Tags": [], "VpcPeeringConnectionId": "pcx-0cb23d66dea3dca9f" } }
ROSA_PUBLIC_ROUTE_TABLE_ID=USD(aws ec2 describe-route-tables --filters "Name=vpc-id,Values=vpc-0b721449398429559" "Name=association.main,Values=true" \ 1 --query "RouteTables[*].RouteTableId" --output text --region eu-west-1 ) aws ec2 create-route --route-table-id USD{ROSA_PUBLIC_ROUTE_TABLE_ID} --destination-cidr-block 192.168.0.0/16 \ 2 --vpc-peering-connection-id pcx-0cb23d66dea3dca9f --region eu-west-1
AURORA_SECURITY_GROUP_ID=USD(aws ec2 describe-security-groups --filters "Name=group-name,Values=keycloak-aurora-security-group" --query "SecurityGroups[*].GroupId" --region eu-west-1 --output text ) aws ec2 authorize-security-group-ingress --group-id USD{AURORA_SECURITY_GROUP_ID} --protocol tcp --port 5432 --cidr 10.0.17.0/24 \ 1 --region eu-west-1
{ "Return": true, "SecurityGroupRules": [ { "SecurityGroupRuleId": "sgr-0785d2f04b9cec3f5", "GroupId": "sg-0d746cc8ad8d2e63b", "GroupOwnerId": "606671647913", "IsEgress": false, "IpProtocol": "tcp", "FromPort": 5432, "ToPort": 5432, "CidrIpv4": "10.0.17.0/24" } ] }
USER=keycloak 1 PASSWORD=secret99 2 DATABASE=keycloak 3 HOST=USD(aws rds describe-db-clusters --db-cluster-identifier keycloak-aurora \ 4 --query 'DBClusters[*].Endpoint' --region eu-west-1 --output text ) run -i --tty --rm debug --image=postgres:15 --restart=Never -- psql postgresql://USD{USER}:USD{PASSWORD}@USD{HOST}/USD{DATABASE}
credentials: - username: developer password: strong-password roles: - admin
apiVersion: v1 kind: Secret type: Opaque metadata: name: connect-secret namespace: keycloak data: identities.yaml: Y3JlZGVudGlhbHM6CiAgLSB1c2VybmFtZTogZGV2ZWxvcGVyCiAgICBwYXNzd29yZDogc3Ryb25nLXBhc3N3b3JkCiAgICByb2xlczoKICAgICAgLSBhZG1pbgo= 1
create secret generic connect-secret --from-file=identities.yaml
apiVersion: v1 kind: Secret metadata: name: ispn-xsite-sa-token 1 annotations: kubernetes.io/service-account.name: "xsite-sa" 2 type: kubernetes.io/service-account-token
create sa -n keycloak xsite-sa policy add-role-to-user view -n keycloak -z xsite-sa create -f xsite-sa-secret-token.yaml get secrets ispn-xsite-sa-token -o jsonpath="{.data.token}" | base64 -d > Site-A-token.txt
create sa -n keycloak xsite-sa policy add-role-to-user view -n keycloak -z xsite-sa create -f xsite-sa-secret-token.yaml get secrets ispn-xsite-sa-token -o jsonpath="{.data.token}" | base64 -d > Site-B-token.txt
create secret generic -n keycloak xsite-token-secret --from-literal=token="USD(cat Site-B-token.txt)"
create secret generic -n keycloak xsite-token-secret --from-literal=token="USD(cat Site-A-token.txt)"
-n keycloak create secret generic xsite-keystore-secret --from-file=keystore.p12="./certs/keystore.p12" \ 1 --from-literal=password=secret \ 2 --from-literal=type=pkcs12 3
-n keycloak create secret generic xsite-truststore-secret --from-file=truststore.p12="./certs/truststore.p12" \ 1 --from-literal=password=caSecret \ 2 --from-literal=type=pkcs12 3
apiVersion: infinispan.org/v1 kind: Infinispan metadata: name: infinispan 1 namespace: keycloak annotations: infinispan.org/monitoring: 'true' 2 spec: replicas: 3 jmx: enabled: true security: endpointSecretName: connect-secret 3 service: type: DataGrid sites: local: name: site-a 4 expose: type: Route 5 maxRelayNodes: 128 encryption: transportKeyStore: secretName: xsite-keystore-secret 6 alias: xsite 7 filename: keystore.p12 8 routerKeyStore: secretName: xsite-keystore-secret 9 alias: xsite 10 filename: keystore.p12 11 trustStore: secretName: xsite-truststore-secret 12 filename: truststore.p12 13 locations: - name: site-b 14 clusterName: infinispan namespace: keycloak 15 url: openshift://api.site-b 16 secretName: xsite-token-secret 17
apiVersion: infinispan.org/v1 kind: Infinispan metadata: name: infinispan 1 namespace: keycloak annotations: infinispan.org/monitoring: 'true' 2 spec: replicas: 3 jmx: enabled: true security: endpointSecretName: connect-secret 3 service: type: DataGrid sites: local: name: site-b 4 expose: type: Route 5 maxRelayNodes: 128 encryption: transportKeyStore: secretName: xsite-keystore-secret 6 alias: xsite 7 filename: keystore.p12 8 routerKeyStore: secretName: xsite-keystore-secret 9 alias: xsite 10 filename: keystore.p12 11 trustStore: secretName: xsite-truststore-secret 12 filename: truststore.p12 13 locations: - name: site-a 14 clusterName: infinispan namespace: keycloak 15 url: openshift://api.site-a 16 secretName: xsite-token-secret 17
apiVersion: infinispan.org/v2alpha1 kind: Cache metadata: name: authenticationsessions namespace: keycloak spec: clusterName: infinispan name: authenticationSessions template: |- distributedCache: mode: "SYNC" owners: "2" statistics: "true" remoteTimeout: "5000" encoding: media-type: "application/x-protostream" locking: acquireTimeout: "4000" transaction: mode: "NON_XA" 1 locking: "PESSIMISTIC" 2 stateTransfer: chunkSize: "16" backups: site-b: 3 backup: strategy: "SYNC" 4 timeout: "4500" 5 failurePolicy: "FAIL" 6 stateTransfer: chunkSize: "16"
apiVersion: infinispan.org/v2alpha1 kind: Cache metadata: name: authenticationsessions namespace: keycloak spec: clusterName: infinispan name: authenticationSessions template: |- distributedCache: mode: "SYNC" owners: "2" statistics: "true" remoteTimeout: "5000" encoding: media-type: "application/x-protostream" locking: acquireTimeout: "4000" transaction: mode: "NON_XA" 1 locking: "PESSIMISTIC" 2 stateTransfer: chunkSize: "16" backups: site-a: 3 backup: strategy: "SYNC" 4 timeout: "4500" 5 failurePolicy: "FAIL" 6 stateTransfer: chunkSize: "16"
wait --for condition=WellFormed --timeout=300s infinispans.infinispan.org -n keycloak infinispan
wait --for condition=CrossSiteViewFormed --timeout=300s infinispans.infinispan.org -n keycloak infinispan
apiVersion: v1 kind: Secret metadata: name: remote-store-secret namespace: keycloak type: Opaque data: username: ZGV2ZWxvcGVy # base64 encoding for 'developer' password: c2VjdXJlX3Bhc3N3b3Jk # base64 encoding for 'secure_password'
apiVersion: k8s.keycloak.org/v2alpha1 kind: Keycloak metadata: labels: app: keycloak name: keycloak namespace: keycloak spec: additionalOptions: - name: cache-remote-host 1 value: "infinispan.keycloak.svc" - name: cache-remote-port 2 value: "11222" - name: cache-remote-username 3 secret: name: remote-store-secret key: username - name: cache-remote-password 4 secret: name: remote-store-secret key: password - name: spi-connections-infinispan-quarkus-site-name 5 value: keycloak
apiVersion: k8s.keycloak.org/v2alpha1 kind: Keycloak metadata: labels: app: keycloak name: keycloak namespace: keycloak spec: hostname: hostname: <KEYCLOAK_URL_HERE> resources: requests: cpu: "2" memory: "1250M" limits: cpu: "6" memory: "2250M" db: vendor: postgres url: jdbc:aws-wrapper:postgresql://<AWS_AURORA_URL_HERE>:5432/keycloak poolMinSize: 30 1 poolInitialSize: 30 poolMaxSize: 30 usernameSecret: name: keycloak-db-secret key: username passwordSecret: name: keycloak-db-secret key: password image: <KEYCLOAK_IMAGE_HERE> 2 startOptimized: false 3 features: enabled: - multi-site 4 transaction: xaEnabled: false 5 additionalOptions: - name: http-max-queued-requests value: "1000" - name: log-console-output value: json - name: metrics-enabled 6 value: 'true' - name: http-pool-max-threads 7 value: "66" - name: cache-remote-host value: "infinispan.keycloak.svc" - name: cache-remote-port value: "11222" - name: cache-remote-username secret: name: remote-store-secret key: username - name: cache-remote-password secret: name: remote-store-secret key: password - name: spi-connections-infinispan-quarkus-site-name value: keycloak - name: db-driver value: software.amazon.jdbc.Driver http: tlsSecret: keycloak-tls-secret instances: 3
wait --for=condition=Ready keycloaks.k8s.keycloak.org/keycloak wait --for=condition=RollingUpdate=False keycloaks.k8s.keycloak.org/keycloak
spec: additionalOptions: - name: http-max-queued-requests value: "1000"
spec: ingress: enabled: true annotations: # When running load tests, disable sticky sessions on the OpenShift HAProxy router # to avoid receiving all requests on a single Red Hat build of Keycloak Pod. haproxy.router.openshift.io/balance: roundrobin haproxy.router.openshift.io/disable_cookies: 'true'
curl -s https://keycloak:managementport/health
curl -s https://keycloak-load-balancer-url/lb-check
curl -s https://keycloak_site_a_url/lb-check curl -s https://keycloak_site_b_url/lb-check
curl -s https://infinispan_rest_url/rest/v2/cache-managers/default/health/status
curl -u <infinispan_user>:<infinispan_pwd> -s https://infinispan_rest_url/rest/v2/cache-managers/default/health | jq 'if .cluster_health.health_status == "HEALTHY" and (all(.cache_health[].status; . == "HEALTHY")) then "HEALTHY" else "UNHEALTHY" end'
curl <infinispan_user>:<infinispan_pwd> -s https://infinispan_rest_url/rest/v2/cluster\?action\=distribution | jq --argjson expectedCount 3 'if map(select(.node_addresses | length > 0)) | length == USDexpectedCount then "HEALTHY" else "UNHEALTHY" end'
get infinispan -n <NAMESPACE> -o json | jq '.items[].status.conditions' | jq 'map({(.type): .status})' | jq 'reduce .[] as USDitem ([]; . + [keys[] | select(USDitem[.] != "True")]) | if length == 0 then "HEALTHY" else "UNHEALTHY: " + (join(", ")) end'
wait --for=condition=Ready --timeout=10s keycloaks.k8s.keycloak.org/keycloak -n <NAMESPACE> wait --for=condition=RollingUpdate=False --timeout=10s keycloaks.k8s.keycloak.org/keycloak -n <NAMESPACE>
NAMESPACE= 1 REGION= 2 HOSTNAME=USD(oc -n USDNAMESPACE get svc accelerator-loadbalancer --template="{{range .status.loadBalancer.ingress}}{{.hostname}}{{end}}") aws elbv2 describe-load-balancers --query "LoadBalancers[?DNSName=='USD{HOSTNAME}'].LoadBalancerArn" --region USD{REGION} --output text
arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/a49e56e51e16843b9a3bc686327c907b/9b786f80ed4eba3d
ACCELERATOR_NAME= 1 ACCELERATOR_ARN=USD(aws globalaccelerator list-accelerators --query "Accelerators[?Name=='USD{ACCELERATOR_NAME}'].AcceleratorArn" --region us-west-2 \ 2 --output text ) LISTENER_ARN=USD(aws globalaccelerator list-listeners --accelerator-arn USD{ACCELERATOR_ARN} --query "Listeners[*].ListenerArn" --region us-west-2 --output text ) aws globalaccelerator list-endpoint-groups --listener-arn USD{LISTENER_ARN} --region us-west-2
{ "EndpointGroups": [ { "EndpointGroupArn": "arn:aws:globalaccelerator::606671647913:accelerator/d280fc09-3057-4ab6-9330-6cbf1f450748/listener/8769072f/endpoint-group/a30b64ec1700", "EndpointGroupRegion": "eu-west-1", "EndpointDescriptions": [ { "EndpointId": "arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/a3c75f239541c4a6e9c48cf8d48d602f/5ba333e87019ccf0", "Weight": 128, "HealthState": "HEALTHY", "ClientIPPreservationEnabled": false } ], "TrafficDialPercentage": 100.0, "HealthCheckPort": 443, "HealthCheckProtocol": "TCP", "HealthCheckIntervalSeconds": 30, "ThresholdCount": 3 } ] }
aws globalaccelerator update-endpoint-group --endpoint-group-arn arn:aws:globalaccelerator::606671647913:accelerator/d280fc09-3057-4ab6-9330-6cbf1f450748/listener/8769072f/endpoint-group/a30b64ec1700 --region us-west-2 --endpoint-configurations ' [ { "EndpointId": "arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/a3c75f239541c4a6e9c48cf8d48d602f/5ba333e87019ccf0", "Weight": 128, "ClientIPPreservationEnabled": false }, { "EndpointId": "arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/a49e56e51e16843b9a3bc686327c907b/9b786f80ed4eba3d", "Weight": 128, "ClientIPPreservationEnabled": false } ] '
apiVersion: infinispan.org/v2alpha1 kind: Batch metadata: name: take-offline namespace: keycloak 1 spec: cluster: infinispan 2 config: | 3 site take-offline --all-caches --site=site-a site status --all-caches --site=site-a
-n keycloak wait --for=jsonpath='{.status.phase}'=Succeeded Batch/take-offline
cat <<EOF | oc apply -n USDNAMESPACE -f - 1 apiVersion: v1 kind: Service metadata: name: accelerator-loadbalancer annotations: service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: accelerator=USD{ACCELERATOR_NAME},site=USD{CLUSTER_NAME},namespace=USD{NAMESPACE} 2 service.beta.kubernetes.io/aws-load-balancer-type: "nlb" service.beta.kubernetes.io/aws-load-balancer-healthcheck-path: "/lb-check" service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol: "https" service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval: "10" 3 service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold: "3" 4 service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold: "3" 5 spec: ports: - name: https port: 443 protocol: TCP targetPort: 8443 selector: app: keycloak app.kubernetes.io/instance: keycloak app.kubernetes.io/managed-by: keycloak-operator sessionAffinity: None type: LoadBalancer EOF
-n USDNAMESPACE get svc accelerator-loadbalancer --template="{{range .status.loadBalancer.ingress}}{{.hostname}}{{end}}"
abab80a363ce8479ea9c4349d116bce2-6b65e8b4272fa4b5.elb.eu-west-1.amazonaws.com
aws globalaccelerator create-accelerator --name example-accelerator \ 1 --ip-address-type DUAL_STACK \ 2 --region us-west-2 3
{ "Accelerator": { "AcceleratorArn": "arn:aws:globalaccelerator::606671647913:accelerator/e35a94dd-391f-4e3e-9a3d-d5ad22a78c71", 1 "Name": "example-accelerator", "IpAddressType": "DUAL_STACK", "Enabled": true, "IpSets": [ { "IpFamily": "IPv4", "IpAddresses": [ "75.2.42.125", "99.83.132.135" ], "IpAddressFamily": "IPv4" }, { "IpFamily": "IPv6", "IpAddresses": [ "2600:9000:a400:4092:88f3:82e2:e5b2:e686", "2600:9000:a516:b4ef:157e:4cbd:7b48:20f1" ], "IpAddressFamily": "IPv6" } ], "DnsName": "a099f799900e5b10d.awsglobalaccelerator.com", 2 "Status": "IN_PROGRESS", "CreatedTime": "2023-11-13T15:46:40+00:00", "LastModifiedTime": "2023-11-13T15:46:42+00:00", "DualStackDnsName": "ac86191ca5121e885.dualstack.awsglobalaccelerator.com" 3 } }
aws globalaccelerator create-listener --accelerator-arn 'arn:aws:globalaccelerator::606671647913:accelerator/e35a94dd-391f-4e3e-9a3d-d5ad22a78c71' --port-ranges '[{"FromPort":443,"ToPort":443}]' --protocol TCP --region us-west-2
{ "Listener": { "ListenerArn": "arn:aws:globalaccelerator::606671647913:accelerator/e35a94dd-391f-4e3e-9a3d-d5ad22a78c71/listener/1f396d40", "PortRanges": [ { "FromPort": 443, "ToPort": 443 } ], "Protocol": "TCP", "ClientAffinity": "NONE" } }
CLUSTER_1_ENDPOINT_ARN=USD(aws elbv2 describe-load-balancers --query "LoadBalancers[?DNSName=='abab80a363ce8479ea9c4349d116bce2-6b65e8b4272fa4b5.elb.eu-west-1.amazonaws.com'].LoadBalancerArn" \ 1 --region eu-west-1 \ 2 --output text ) CLUSTER_2_ENDPOINT_ARN=USD(aws elbv2 describe-load-balancers --query "LoadBalancers[?DNSName=='a1c76566e3c334e4ab7b762d9f8dcbcf-985941f9c8d108d4.elb.eu-west-1.amazonaws.com'].LoadBalancerArn" \ 3 --region eu-west-1 \ 4 --output text ) ENDPOINTS='[ { "EndpointId": "'USD{CLUSTER_1_ENDPOINT_ARN}'", "Weight": 128, "ClientIPPreservationEnabled": false }, { "EndpointId": "'USD{CLUSTER_2_ENDPOINT_ARN}'", "Weight": 128, "ClientIPPreservationEnabled": false } ]' aws globalaccelerator create-endpoint-group --listener-arn 'arn:aws:globalaccelerator::606671647913:accelerator/e35a94dd-391f-4e3e-9a3d-d5ad22a78c71/listener/1f396d40' \ 5 --traffic-dial-percentage 100 --endpoint-configurations USD{ENDPOINTS} --endpoint-group-region eu-west-1 \ 6 --region us-west-2
{ "EndpointGroup": { "EndpointGroupArn": "arn:aws:globalaccelerator::606671647913:accelerator/e35a94dd-391f-4e3e-9a3d-d5ad22a78c71/listener/1f396d40/endpoint-group/2581af0dc700", "EndpointGroupRegion": "eu-west-1", "EndpointDescriptions": [ { "EndpointId": "arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/abab80a363ce8479ea9c4349d116bce2/6b65e8b4272fa4b5", "Weight": 128, "HealthState": "HEALTHY", "ClientIPPreservationEnabled": false }, { "EndpointId": "arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/a1c76566e3c334e4ab7b762d9f8dcbcf/985941f9c8d108d4", "Weight": 128, "HealthState": "HEALTHY", "ClientIPPreservationEnabled": false } ], "TrafficDialPercentage": 100.0, "HealthCheckPort": 443, "HealthCheckProtocol": "TCP", "HealthCheckPath": "undefined", "HealthCheckIntervalSeconds": 30, "ThresholdCount": 3 } }
apiVersion: k8s.keycloak.org/v2alpha1 kind: Keycloak metadata: name: keycloak spec: hostname: hostname: USDHOSTNAME 1 ingress: enabled: false 2
apply -f - << EOF apiVersion: v1 kind: ConfigMap metadata: name: user-workload-monitoring-config namespace: openshift-user-workload-monitoring data: config.yaml: | alertmanager: enabled: true enableAlertmanagerConfig: true EOF -n openshift-user-workload-monitoring rollout status --watch statefulset.apps/alertmanager-user-workload
aws secretsmanager create-secret --name webhook-password \ 1 --secret-string changeme \ 2 --region eu-west-1 3
FUNCTION_NAME= 1 ROLE_ARN=USD(aws iam create-role --role-name USD{FUNCTION_NAME} --assume-role-policy-document '{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Service": "lambda.amazonaws.com" }, "Action": "sts:AssumeRole" } ] }' --query 'Role.Arn' --region eu-west-1 \ 2 --output text )
POLICY_ARN=USD(aws iam create-policy --policy-name LambdaSecretManager --policy-document '{ "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Action": [ "secretsmanager:GetSecretValue" ], "Resource": "*" } ] }' --query 'Policy.Arn' --output text ) aws iam attach-role-policy --role-name USD{FUNCTION_NAME} --policy-arn USD{POLICY_ARN}
aws iam attach-role-policy --role-name USD{FUNCTION_NAME} --policy-arn arn:aws:iam::aws:policy/ElasticLoadBalancingReadOnly
aws iam attach-role-policy --role-name USD{FUNCTION_NAME} --policy-arn arn:aws:iam::aws:policy/GlobalAcceleratorFullAccess
LAMBDA_ZIP=/tmp/lambda.zip cat << EOF > /tmp/lambda.py from urllib.error import HTTPError import boto3 import jmespath import json import os import urllib3 from base64 import b64decode from urllib.parse import unquote Prevent unverified HTTPS connection warning urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) class MissingEnvironmentVariable(Exception): pass class MissingSiteUrl(Exception): pass def env(name): if name in os.environ: return os.environ[name] raise MissingEnvironmentVariable(f"Environment Variable '{name}' must be set") def handle_site_offline(labels): a_client = boto3.client('globalaccelerator', region_name='us-west-2') acceleratorDNS = labels['accelerator'] accelerator = jmespath.search(f"Accelerators[?(DnsName=='{acceleratorDNS}'|| DualStackDnsName=='{acceleratorDNS}')]", a_client.list_accelerators()) if not accelerator: print(f"Ignoring SiteOffline alert as accelerator with DnsName '{acceleratorDNS}' not found") return accelerator_arn = accelerator[0]['AcceleratorArn'] listener_arn = a_client.list_listeners(AcceleratorArn=accelerator_arn)['Listeners'][0]['ListenerArn'] endpoint_group = a_client.list_endpoint_groups(ListenerArn=listener_arn)['EndpointGroups'][0] endpoints = endpoint_group['EndpointDescriptions'] # Only update accelerator endpoints if two entries exist if len(endpoints) > 1: # If the reporter endpoint is not healthy then do nothing for now # A Lambda will eventually be triggered by the other offline site for this reporter reporter = labels['reporter'] reporter_endpoint = [e for e in endpoints if endpoint_belongs_to_site(e, reporter)][0] if reporter_endpoint['HealthState'] == 'UNHEALTHY': print(f"Ignoring SiteOffline alert as reporter '{reporter}' endpoint is marked UNHEALTHY") return offline_site = labels['site'] endpoints = [e for e in endpoints if not endpoint_belongs_to_site(e, offline_site)] del reporter_endpoint['HealthState'] a_client.update_endpoint_group( EndpointGroupArn=endpoint_group['EndpointGroupArn'], EndpointConfigurations=endpoints ) print(f"Removed site={offline_site} from Accelerator EndpointGroup") take_infinispan_site_offline(reporter, offline_site) print(f"Backup site={offline_site} caches taken offline") else: print("Ignoring SiteOffline alert only one Endpoint defined in the EndpointGroup") def endpoint_belongs_to_site(endpoint, site): lb_arn = endpoint['EndpointId'] region = lb_arn.split(':')[3] client = boto3.client('elbv2', region_name=region) tags = client.describe_tags(ResourceArns=[lb_arn])['TagDescriptions'][0]['Tags'] for tag in tags: if tag['Key'] == 'site': return tag['Value'] == site return false def take_infinispan_site_offline(reporter, offlinesite): endpoints = json.loads(INFINISPAN_SITE_ENDPOINTS) if reporter not in endpoints: raise MissingSiteUrl(f"Missing URL for site '{reporter}' in 'INFINISPAN_SITE_ENDPOINTS' json") endpoint = endpoints[reporter] password = get_secret(INFINISPAN_USER_SECRET) url = f"https://{endpoint}/rest/v2/container/x-site/backups/{offlinesite}?action=take-offline" http = urllib3.PoolManager(cert_reqs='CERT_NONE') headers = urllib3.make_headers(basic_auth=f"{INFINISPAN_USER}:{password}") try: rsp = http.request("POST", url, headers=headers) if rsp.status >= 400: raise HTTPError(f"Unexpected response status '%d' when taking site offline", rsp.status) rsp.release_conn() except HTTPError as e: print(f"HTTP error encountered: {e}") def get_secret(secret_name): session = boto3.session.Session() client = session.client( service_name='secretsmanager', region_name=SECRETS_REGION ) return client.get_secret_value(SecretId=secret_name)['SecretString'] def decode_basic_auth_header(encoded_str): split = encoded_str.strip().split(' ') if len(split) == 2: if split[0].strip().lower() == 'basic': try: username, password = b64decode(split[1]).decode().split(':', 1) except: raise DecodeError else: raise DecodeError else: raise DecodeError return unquote(username), unquote(password) def handler(event, context): print(json.dumps(event)) authorization = event['headers'].get('authorization') if authorization is None: print("'Authorization' header missing from request") return { "statusCode": 401 } expectedPass = get_secret(WEBHOOK_USER_SECRET) username, password = decode_basic_auth_header(authorization) if username != WEBHOOK_USER and password != expectedPass: print('Invalid username/password combination') return { "statusCode": 403 } body = event.get('body') if body is None: raise Exception('Empty request body') body = json.loads(body) print(json.dumps(body)) if body['status'] != 'firing': print("Ignoring alert as status is not 'firing', status was: '%s'" % body['status']) return { "statusCode": 204 } for alert in body['alerts']: labels = alert['labels'] if labels['alertname'] == 'SiteOffline': handle_site_offline(labels) return { "statusCode": 204 } INFINISPAN_USER = env('INFINISPAN_USER') INFINISPAN_USER_SECRET = env('INFINISPAN_USER_SECRET') INFINISPAN_SITE_ENDPOINTS = env('INFINISPAN_SITE_ENDPOINTS') SECRETS_REGION = env('SECRETS_REGION') WEBHOOK_USER = env('WEBHOOK_USER') WEBHOOK_USER_SECRET = env('WEBHOOK_USER_SECRET') EOF zip -FS --junk-paths USD{LAMBDA_ZIP} /tmp/lambda.py
aws lambda create-function --function-name USD{FUNCTION_NAME} --zip-file fileb://USD{LAMBDA_ZIP} --handler lambda.handler --runtime python3.12 --role USD{ROLE_ARN} --region eu-west-1 1
aws lambda create-function-url-config --function-name USD{FUNCTION_NAME} --auth-type NONE --region eu-west-1 1
aws lambda add-permission --action "lambda:InvokeFunctionUrl" --function-name USD{FUNCTION_NAME} --principal "*" --statement-id FunctionURLAllowPublicAccess --function-url-auth-type NONE --region eu-west-1 1
-n USD{NAMESPACE} get route infinispan-external -o jsonpath='{.status.ingress[].host}' 1
ACCELERATOR_NAME= 1 LAMBDA_REGION= 2 CLUSTER_1_NAME= 3 CLUSTER_1_ISPN_ENDPOINT= 4 CLUSTER_2_NAME= 5 CLUSTER_2_ISPN_ENDPOINT= 6 INFINISPAN_USER= 7 INFINISPAN_USER_SECRET= 8 WEBHOOK_USER= 9 WEBHOOK_USER_SECRET= 10 INFINISPAN_SITE_ENDPOINTS=USD(echo "{\"USD{CLUSTER_NAME_1}\":\"USD{CLUSTER_1_ISPN_ENDPOINT}\",\"USD{CLUSTER_2_NAME}\":\"USD{CLUSTER_2_ISPN_ENDPOINT\"}" | jq tostring) aws lambda update-function-configuration --function-name USD{ACCELERATOR_NAME} --region USD{LAMBDA_REGION} --environment "{ \"Variables\": { \"INFINISPAN_USER\" : \"USD{INFINISPAN_USER}\", \"INFINISPAN_USER_SECRET\" : \"USD{INFINISPAN_USER_SECRET}\", \"INFINISPAN_SITE_ENDPOINTS\" : USD{INFINISPAN_SITE_ENDPOINTS}, \"WEBHOOK_USER\" : \"USD{WEBHOOK_USER}\", \"WEBHOOK_USER_SECRET\" : \"USD{WEBHOOK_USER_SECERT}\", \"SECRETS_REGION\" : \"eu-central-1\" } }"
aws lambda get-function-url-config --function-name USD{FUNCTION_NAME} --query "FunctionUrl" --region eu-west-1 \ 1 --output text
https://tjqr2vgc664b6noj6vugprakoq0oausj.lambda-url.eu-west-1.on.aws
NAMESPACE= # The namespace containing your deployments apply -n USD{NAMESPACE} -f - << EOF apiVersion: v1 kind: Secret type: kubernetes.io/basic-auth metadata: name: webhook-credentials stringData: username: 'keycloak' 1 password: 'changme' 2 --- apiVersion: monitoring.coreos.com/v1beta1 kind: AlertmanagerConfig metadata: name: example-routing spec: route: receiver: default groupBy: - accelerator groupInterval: 90s groupWait: 60s matchers: - matchType: = name: alertname value: SiteOffline receivers: - name: default webhookConfigs: - url: 'https://tjqr2vgc664b6noj6vugprakoq0oausj.lambda-url.eu-west-1.on.aws/' 3 httpConfig: basicAuth: username: key: username name: webhook-credentials password: key: password name: webhook-credentials tlsConfig: insecureSkipVerify: true --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: name: xsite-status spec: groups: - name: xsite-status rules: - alert: SiteOffline expr: 'min by (namespace, site) (vendor_jgroups_site_view_status{namespace="default",site="site-b"}) == 0' 4 labels: severity: critical reporter: site-a 5 accelerator: a3da6a6cbd4e27b02.awsglobalaccelerator.com 6
-n openshift-operators scale --replicas=0 deployment/infinispan-operator-controller-manager 1 -n openshift-operators rollout status -w deployment/infinispan-operator-controller-manager -n USD{NAMESPACE} scale --replicas=0 deployment/infinispan-router 2 -n USD{NAMESPACE} rollout status -w deployment/infinispan-router
-n openshift-operators scale --replicas=1 deployment/infinispan-operator-controller-manager -n openshift-operators rollout status -w deployment/infinispan-operator-controller-manager -n USD{NAMESPACE} scale --replicas=1 deployment/infinispan-router 1 -n USD{NAMESPACE} rollout status -w deployment/infinispan-router
-n keycloak exec -it pods/infinispan-0 -- ./bin/cli.sh --trustall --connect https://127.0.0.1:11222
Username: developer Password: [infinispan-0-29897@ISPN//containers/default]>
site take-offline --all-caches --site=site-a
authenticationSessions: ok
work: ok
loginFailures: ok
actionTokens: ok
site status --all-caches --site=site-a
status: offline
clearcache actionTokens clearcache authenticationSessions clearcache loginFailures clearcache work
site bring-online --all-caches --site=site-a
authenticationSessions: ok
work: ok
loginFailures: ok
actionTokens: ok
site status --all-caches --site=site-a
status: online
-n keycloak exec -it pods/infinispan-0 -- ./bin/cli.sh --trustall --connect https://127.0.0.1:11222
Username: developer Password: [infinispan-0-29897@ISPN//containers/default]>
site push-site-state --all-caches --site=site-b
authenticationSessions: ok
work: ok
loginFailures: ok
actionTokens: ok
site status --all-caches --site=site-b
status: online
site push-site-status --cache=actionTokens site push-site-status --cache=authenticationSessions site push-site-status --cache=loginFailures site push-site-status --cache=work
{ "site-b" : "OK" } { "site-b" : "OK" } { "site-b" : "OK" } { "site-b" : "OK" }
site push-site-state --cache=<cache-name> --site=site-b
site clear-push-site-status --cache=actionTokens site clear-push-site-status --cache=authenticationSessions site clear-push-site-status --cache=loginFailures site clear-push-site-status --cache=work
"ok" "ok" "ok" "ok"
NAMESPACE= 1 REGION= 2 HOSTNAME=USD(oc -n USDNAMESPACE get svc accelerator-loadbalancer --template="{{range .status.loadBalancer.ingress}}{{.hostname}}{{end}}") aws elbv2 describe-load-balancers --query "LoadBalancers[?DNSName=='USD{HOSTNAME}'].LoadBalancerArn" --region USD{REGION} --output text
arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/a49e56e51e16843b9a3bc686327c907b/9b786f80ed4eba3d
ACCELERATOR_NAME= 1 ACCELERATOR_ARN=USD(aws globalaccelerator list-accelerators --query "Accelerators[?Name=='USD{ACCELERATOR_NAME}'].AcceleratorArn" --region us-west-2 \ 2 --output text ) LISTENER_ARN=USD(aws globalaccelerator list-listeners --accelerator-arn USD{ACCELERATOR_ARN} --query "Listeners[*].ListenerArn" --region us-west-2 --output text ) aws globalaccelerator list-endpoint-groups --listener-arn USD{LISTENER_ARN} --region us-west-2
{ "EndpointGroups": [ { "EndpointGroupArn": "arn:aws:globalaccelerator::606671647913:accelerator/d280fc09-3057-4ab6-9330-6cbf1f450748/listener/8769072f/endpoint-group/a30b64ec1700", "EndpointGroupRegion": "eu-west-1", "EndpointDescriptions": [ { "EndpointId": "arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/a49e56e51e16843b9a3bc686327c907b/9b786f80ed4eba3d", "Weight": 128, "HealthState": "HEALTHY", "ClientIPPreservationEnabled": false }, { "EndpointId": "arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/a3c75f239541c4a6e9c48cf8d48d602f/5ba333e87019ccf0", "Weight": 128, "HealthState": "HEALTHY", "ClientIPPreservationEnabled": false } ], "TrafficDialPercentage": 100.0, "HealthCheckPort": 443, "HealthCheckProtocol": "TCP", "HealthCheckIntervalSeconds": 30, "ThresholdCount": 3 } ] }
aws globalaccelerator update-endpoint-group --endpoint-group-arn arn:aws:globalaccelerator::606671647913:accelerator/d280fc09-3057-4ab6-9330-6cbf1f450748/listener/8769072f/endpoint-group/a30b64ec1700 --region us-west-2 --endpoint-configurations ' [ { "EndpointId": "arn:aws:elasticloadbalancing:eu-west-1:606671647913:loadbalancer/net/a49e56e51e16843b9a3bc686327c907b/9b786f80ed4eba3d", "Weight": 128, "ClientIPPreservationEnabled": false } ] '