index_v2/OnCall/docstore.json

{"docstore/metadata": {"d6496709-cfc5-4485-93e6-2c6905c9d395": {"doc_hash": "ba829374ee458613e165ee84864cb14c77e58f29c0834dbd4e5f912038f0bb0b"}, "7bc79ea6-68b3-4bf5-8112-31e1aa58d448": {"doc_hash": "9235c45f05466bd25cf44b08c688648d350ecc8155d3298737e6da74e901d63c"}, "57018674-bf81-4ad7-940b-47bc8f22a82b": {"doc_hash": "8d63fb8bf8df55df4f5197459fd3a64b87c236b6064aebed0c594689dcc8c818"}, "04678987-bdb0-4922-a081-c6066b190cff": {"doc_hash": "f3f6588a774bb7d11930e89fc7dfdcfaa1cd3b790c7dbe529caa506839d8b53a"}, "130a58d8-6265-4783-b0a6-056344dacdb8": {"doc_hash": "21c74d0f1661068e36910b2edb2dfc0a7023e3e0824a1886d0ef24f1084dd561"}, "bd4335e6-72c5-40f0-ac46-bfd30bc3b26f": {"doc_hash": "8a28314bb5126884074d743b913cdfbee6aeecb9683fb8fe916f6b87e89ef6ee"}, "0bd2d94e-126a-4644-8989-aa46455d3788": {"doc_hash": "73588ffe6973fa49b0c3077faa9f86c139ad0f8c454e53fc65f9515cea160d28"}, "d09c18d3-ac4b-4719-a3f1-9ca73015e9c2": {"doc_hash": "393d4ef40635d51192007750b39e813be462e2977f1394d8268d2ed934bd1efe"}, "ce7f46b5-953b-4cb8-b961-183b46bc7645": {"doc_hash": "c71e2be5d1b16eae6c3cc025a36805db56dc72f75f370e5a627febd746eb4bee"}, "597ca512-6f36-4807-b986-4a1d5acc33ca": {"doc_hash": "b5f9583474e44ec59425e0b688b0be07ff599cf8e029af55625eb07ebef2ac90"}, "1ba7a349-4a28-4423-a013-05d8defe7065": {"doc_hash": "02e672753bc4d8fea49be0aa8911cd09b3f8c9c155c24d2699605170446679ee"}, "c48683dc-f64e-4689-bcba-15ef3e12b199": {"doc_hash": "063df790c6149c4ce5236602034cfc02e6f6f1b2a622dddf43af4d3a38df36ef"}, "6ca3f6f6-9ccc-4ee4-9af0-15bb8ed19623": {"doc_hash": "4de7f29babc2fb2eb3a09f34596f49c2a755a6f39c82a3347d9e84dfcd24b952"}, "2db441ba-005f-47fc-ab87-96ebe48bb427": {"doc_hash": "8d026d684544b78161af0013116f2adc29a261dcfb565b682a1aef00ec0e2672"}, "468db9da-7f2c-458a-8e4b-a209e0ccb1ad": {"doc_hash": "282f7a4e8d6c8a5fbdad62c017649e2fa9a8bee44a08e744fe98dac3e1c3ca6c"}, "00e68c43-8f2c-4ed5-b5ec-e0c71b8f3819": {"doc_hash": "1a86000e046e02e65e8b90d724c44e1c30b0211a2b13674ec716d3f64c5c6686"}, "61eb3840-5d98-4260-8e52-318ba943edcf": {"doc_hash": "9c016adfebd20b478a08aa2f5f267dc63ee4bb47a6583cb8e8677832abc3b1e8"}, "03a7050b-f0d8-4a33-b9e8-d4f82439cca0": {"doc_hash": "1e7ddc74e8d4464205a6644eb228753e21d7a80ddaa201ec55bfbb3ebdde4fcd", "ref_doc_id": "d6496709-cfc5-4485-93e6-2c6905c9d395"}, "33630944-a8b9-4d7e-8d3f-0381d2c4e586": {"doc_hash": "bd205fbaf9edc0054ec276d8b090a4d142de9aa42f4a6b0966d4a3b662a46327", "ref_doc_id": "d6496709-cfc5-4485-93e6-2c6905c9d395"}, "69adae88-db95-4f48-81df-ac19ca8077e3": {"doc_hash": "04291c6d919c46bce3de41cfd59054e6f6749fbebe20fe95c507d9cc7725c4d5", "ref_doc_id": "7bc79ea6-68b3-4bf5-8112-31e1aa58d448"}, "21d0b6b7-28cf-437e-aef6-522813d4e0a1": {"doc_hash": "442829f5592cb598491ad09a9bf802b5ffd05c35f0acd8ed778933a106b6fbfd", "ref_doc_id": "7bc79ea6-68b3-4bf5-8112-31e1aa58d448"}, "792e2728-bdc4-4e54-ad72-55e359056616": {"doc_hash": "468aa6ba5b142eaf477fc77a16ac3860eb0a697cb185d98efe5952334915c1d9", "ref_doc_id": "57018674-bf81-4ad7-940b-47bc8f22a82b"}, "6f1adeda-e4d0-4b96-b2f1-57def8904cb6": {"doc_hash": "ac81899b7dbe7fc3f0421bc53a447ed09c476c48ace64bf033962da90442f5c8", "ref_doc_id": "57018674-bf81-4ad7-940b-47bc8f22a82b"}, "07aa42dc-7b17-4b9f-848a-181c4ec7e77f": {"doc_hash": "e293beddafaeba9c2e7920144bfa4f83273e02ec83f6018e14e50352775b4066", "ref_doc_id": "04678987-bdb0-4922-a081-c6066b190cff"}, "d14ad41a-0e46-4fb9-b979-8973ba836de4": {"doc_hash": "6522631f31a4eb34a2c2b544744221bb4786da92958151c2bdcbf5885c718333", "ref_doc_id": "04678987-bdb0-4922-a081-c6066b190cff"}, "e5834cdd-0e9d-484d-a7e8-721644c8e119": {"doc_hash": "1d176f65dedef8d5ad0687715f9c339683b887090d0401a901c6c6978b0feb9a", "ref_doc_id": "130a58d8-6265-4783-b0a6-056344dacdb8"}, "4cc6e453-d8cb-4316-9f00-1ad8fc6152ad": {"doc_hash": "421638b67a9017a993430a131cb489f7c2e389add8bbd170c41fb9342cfbea24", "ref_doc_id": "130a58d8-6265-4783-b0a6-056344dacdb8"}, "975881aa-a45e-4b2c-83a8-bd6932e4a819": {"doc_hash": "8a28314bb5126884074d743b913cdfbee6aeecb9683fb8fe916f6b87e89ef6ee", "ref_doc_id": "bd4335e6-72c5-40f0-ac46-bfd30bc3b26f"}, "26d56212-a665-414f-803a-387a807bffa0": {"doc_hash": "d7ac5b9c0ab787ad4760d17bb0aadddcf494ad86227d369624429af714871f64", "ref_doc_id": "0bd2d94e-126a-4644-8989-aa46455d3788"}, "ca972543-cc22-4e27-a596-6492703c69ce": {"doc_hash": "f423df9dcf495ebde76dc87f0ed83ee90510a32f59cf6074b61c5a2f50ec8489", "ref_doc_id": "0bd2d94e-126a-4644-8989-aa46455d3788"}, "02048f03-be3c-4ebf-9383-5e85fd3356e1": {"doc_hash": "3ec06fd5ac2c5df8dca93e0230688bddf501838001a915f4a72d6fc1af4a78e1", "ref_doc_id": "d09c18d3-ac4b-4719-a3f1-9ca73015e9c2"}, "9cf96cfc-0673-4806-b906-e4dab4f6aa0e": {"doc_hash": "f847cda6d3bfbd372510c419f3c441db5dd3b844749ac665292b54549c535324", "ref_doc_id": "d09c18d3-ac4b-4719-a3f1-9ca73015e9c2"}, "9b16e0e5-8641-4e16-a4f7-67130cdd3a59": {"doc_hash": "cfaaada74b10d4e9a4bab7716a674baed2bc3b9ea87aa0d38103027d8d0aae3c", "ref_doc_id": "ce7f46b5-953b-4cb8-b961-183b46bc7645"}, "7f0dd2c8-c705-447e-b1f0-55556cf61ad4": {"doc_hash": "22889ef4aaa4e83fb7aa65672e44a170330a352aedc9af9484a65dfa8cd19cf9", "ref_doc_id": "ce7f46b5-953b-4cb8-b961-183b46bc7645"}, "7ce57955-6637-4a9a-85e0-3b8d16306141": {"doc_hash": "ea26818540474a01a0f073875cf1ecb90cce2bf2d3197374a49cf7a5adaf243a", "ref_doc_id": "597ca512-6f36-4807-b986-4a1d5acc33ca"}, "ffdf9f90-fc3a-4461-a76f-81e03fe84ba5": {"doc_hash": "b4fd660279f17945d5caced3f06297ba5931d73d5f168cba77ea325e7b016211", "ref_doc_id": "597ca512-6f36-4807-b986-4a1d5acc33ca"}, "626bf733-83e0-4372-99f9-1dbf3c28ba09": {"doc_hash": "250979f309d921145598693fd5749628bea415943ab79ac333d4ae961514b527", "ref_doc_id": "1ba7a349-4a28-4423-a013-05d8defe7065"}, "d5075358-1c6e-4cea-9e52-089839227b0b": {"doc_hash": "75592305e804625729dfa9f91bc1405c7e4ec65b537c842e77bea06b52c6fc2c", "ref_doc_id": "1ba7a349-4a28-4423-a013-05d8defe7065"}, "c50c0de2-ac18-4cd7-b256-ada4437b2bda": {"doc_hash": "7b73a33c11e41231fde88bd963b2c4f4e02766e0c9a3cf5261dec06934c7d16d", "ref_doc_id": "c48683dc-f64e-4689-bcba-15ef3e12b199"}, "01024302-ea9b-4c9e-a828-a1993457fb64": {"doc_hash": "1c2c983e8edfc01984ac6fc3739201892f191ec1bfd9893c7762166f24f7da78", "ref_doc_id": "c48683dc-f64e-4689-bcba-15ef3e12b199"}, "097a8045-83e3-4b75-a50f-f1d632d4b401": {"doc_hash": "7aba5260560ca7df5c005486b024256bd3f55ca1957aadc5720744d03ef88f46", "ref_doc_id": "6ca3f6f6-9ccc-4ee4-9af0-15bb8ed19623"}, "323c8c4a-7364-4ad8-82c9-ed3deb9eca73": {"doc_hash": "35b3c7e033df386e946b443f55ed94de7a012f3b704dcddff4667103d4e47c2b", "ref_doc_id": "6ca3f6f6-9ccc-4ee4-9af0-15bb8ed19623"}, "076e6fd8-44dc-4bac-89f1-37a0041f2c11": {"doc_hash": "8d026d684544b78161af0013116f2adc29a261dcfb565b682a1aef00ec0e2672", "ref_doc_id": "2db441ba-005f-47fc-ab87-96ebe48bb427"}, "6b452ef4-8877-4949-bb83-18edbabbfb58": {"doc_hash": "c9214a49365cb50c3b7b3a4bcf53baa21cf17048f981d7a5617b54e6e1d35fc4", "ref_doc_id": "468db9da-7f2c-458a-8e4b-a209e0ccb1ad"}, "7bd9db26-5333-49d3-acfe-b320ad60a235": {"doc_hash": "41a06d83a069f023f4bb9e5c9b5bcce6010e31811dc5604185831759e9fc6c6e", "ref_doc_id": "468db9da-7f2c-458a-8e4b-a209e0ccb1ad"}, "279ac7d5-3dfe-4ac0-90e5-1e530831058a": {"doc_hash": "7f3c775f3b4f4c6752d1971216828989007def89a9878a50ce3d002c3ef9e9a6", "ref_doc_id": "00e68c43-8f2c-4ed5-b5ec-e0c71b8f3819"}, "ffbb83c4-370e-4692-826e-a86bbef48dc1": {"doc_hash": "44ed1835b0e365ac7b4163d53ac0a53217d44cf34507b1cf0dc0a842371e9d14", "ref_doc_id": "00e68c43-8f2c-4ed5-b5ec-e0c71b8f3819"}, "57762e71-1462-4735-8d07-d94672b8c357": {"doc_hash": "9c016adfebd20b478a08aa2f5f267dc63ee4bb47a6583cb8e8677832abc3b1e8", "ref_doc_id": "61eb3840-5d98-4260-8e52-318ba943edcf"}}, "docstore/data": {"03a7050b-f0d8-4a33-b9e8-d4f82439cca0": {"__data__": {"text": "On-call Log\nOpen on-call bugs/tasks  \nJIRA  List\nOn-call Incident Log Key Summary T Created Updated Due Assignee Reporter P Status Resolution\nCOLL-2428ON-CALL: invalid_refresh_token error for Tenant 12956\n(crowdstrike)15/May/23 11:29 PM 15/May/23 11:31 PM Harinandan\nChintamreddyParanthaman\nKarthikeyanTO DO Unresolved\nCOLL-2308Find rootcause for 404 while brocessing slack event\nduring cross brokering24/Apr/23 10:57 AM 24/Apr/23 10:58 AM Harinandan\nChintamreddyAnil Ganivada TO DO Unresolved\nCOLL-2294Debug 500 response from sentiment collector endpoint\n\"/sentiment-collector/api/messages\"19/Apr/23 12:14 AM 19/Apr/23 12:16 AM Harinandan\nChintamreddyAnil Ganivada TO DO Unresolved\nDISC-2807[Investigate] 500 error due connection reset by peer for\nvalidate user call11/Apr/23 10:51 PM 11/Apr/23 10:58 PM ritu vaidya Pradeep TO DO Unresolved\nCOLL-2264Alert for : [Prod] [Collab] /slack/slash/addressBook took\nmore than 3 seconds11/Apr/23 10:25 PM 18/Apr/23 3:28 AM Harinandan\nChintamreddyGuruprasad\nShenoyTO DO Unresolved\nCOLL-2084Alert for : [Prod] [Collab] Failed brokering message to\nSlack14/Mar/23 11:24 PM 03/May/23 11:59 PM Harinandan\nChintamreddyShivam\nBhosaleTO DO Unresolved\nCOLL-1992[On-Call] Failure to extend pulse for messages deleted\non Slack, but not on Axm.28/Feb/23 12:42 PM 28/Feb/23 4:15 PM Harinandan\nChintamreddyRama\nTaranigantyTO DO Unresolved\nCOLL-1904[On-call] [Investigate] Failed to refresh token due to\nHttpMessageNotReadableException14/Feb/23 12:40 PM 14/Feb/23 12:41 PM Harinandan\nChintamreddyRama\nTaranigantyTO DO Unresolved\nCOLL-1898[On-call] [Investigate] Failed brokering message to Slack\nwith a 40413/Feb/23 12:17 PM 23/Feb/23 10:30 AM Sagarika\nNangiaRama\nTaranigantyTO DO Unresolved\nDAT-697 [On-call] [Investigate] Is there a need to tune the API\ngranularity or # of connections in the Hikari pool to avoid\nthe drop in IDLE connections26/Oct/22 3:07 PM 13/Feb/23 12:20 PM Satya Vummidi Rama\nTaranigantyTO DO Unresolved\nCOLL-1289[on-call] [Document] If we see a Slack Operation we\ncannot handle, there needs to be a process to follow to\ntriage it01/Oct/22 4:31 PM 13/Feb/23 12:20 PM Harinandan\nChintamreddyRama\nTaranigantyTO DO Unresolved\nDISC-1422[On-call] Prod ES yellow status alarm: Increase the\nnumber of data points or duration to consider before\ncreating an alarm19/Sep/22 9:53 PM 13/Feb/23 12:20 PM Pradeep Pradeep TO DO Unresolved\nCOLL-928 Add Details for runboook entry for COLLAB-FAILED-\nMSG-TO-TEAMS-OR-SLACK01/Jul/22 9:18 AM 13/Feb/23 12:20 PM Harinandan\nChintamreddyManoj\nInukolunuTO DO Unresolved\n13 issues\nJun 7, 2023 @Satya Vummidi Threshold Violated: GLOBAL-JVM-HEAP-THRESHOLD It looks like a false positive spike. We may need to update the trigger \ncriteria to tone it down a little bit.", "doc_id": "03a7050b-f0d8-4a33-b9e8-d4f82439cca0", "embedding": null, "doc_hash": "1e7ddc74e8d4464205a6644eb228753e21d7a80ddaa201ec55bfbb3ebdde4fcd", "extra_info": {"page_label": "1", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 2740, "_node_type": "1"}, "relationships": {"1": "d6496709-cfc5-4485-93e6-2c6905c9d395", "3": "33630944-a8b9-4d7e-8d3f-0381d2c4e586"}}, "__type__": "1"}, "33630944-a8b9-4d7e-8d3f-0381d2c4e586": {"__data__": {"text": "We may need to update the trigger \ncriteria to tone it down a little bit. Started a discussion in Slack me\nssage from Satya Vummidi in #reg-alerts-prod Date Responder Notes Related JIRAs/identified actions\n", "doc_id": "33630944-a8b9-4d7e-8d3f-0381d2c4e586", "embedding": null, "doc_hash": "bd205fbaf9edc0054ec276d8b090a4d142de9aa42f4a6b0966d4a3b662a46327", "extra_info": {"page_label": "1", "file_name": "OnCall.pdf"}, "node_info": {"start": 2667, "end": 2873, "_node_type": "1"}, "relationships": {"1": "d6496709-cfc5-4485-93e6-2c6905c9d395", "2": "03a7050b-f0d8-4a33-b9e8-d4f82439cca0"}}, "__type__": "1"}, "69adae88-db95-4f48-81df-ac19ca8077e3": {"__data__": {"text": "Jun 6, 2023 @Prashant Kumar Jha [Prod] [Disc] Report Partial Generation Failed DISC-2723: [ON-CALL] [Partial Report Generation] Report genera\ntion API failure: unauthorizedTO DO \n31 May 2023 @Gaurav Dahiya [Prod] [Collab] Failed brokering message to collab COLL-2528: throw 404 from TM and 401 from consuming service\ns if service token is not foundTO DO \nSlack thread - Slack message from Opsgenie in #opsgenie-alerts \n29 May 2023 @Prabhu Pant [Prod] [Collab] Unable to install app in teams Raised during testing in production\n27 May 2023 @Prabhu Pant ALARM: \"cdk-rds-alarms-production-\naslv2commonproductionrdsalarmslowfreeablem...\" in US West \n(Oregon)Raised due to low primary memory in the RDS instance at that time. \nSlack message from seshan in #on-call \n27 May 2023 @Prabhu Pant [LightStep] Critical Threshold Violated: GLOBAL-MAX-FILES\n[LightStep] Resolved: GLOBAL-MAX-FILESPossibly raised from cloud-config-server during the deployment as \nGitlab was not accessible and the service was deployed twice. \nVerified that cloud-config server is working in prod and alert wasn\u2019t \nre-raised later on Slack message from Anil Kumar Ganivada in #o\npsgenie-alerts \n27 May 2023 @Prabhu Pant Alert for : [Prod][Plat] Service Restarting Repeatedly This alert was raised because a platform service was restarted to \ncheck automation tests Slack message from Logz Alerts in #critica\nl-alerts-prod \n26 May 2023 @Prabhu Pant CloudWatch alarm - cdk-rds-alarms-production-\naslv2discoveryproductionrdsalarmslowfreeablememoryalarmC53\nE0730-A4HRWXNQRI1COpen a ticket with AWS to increase the resources on this cluster - \nSlack message from Slackbot in #reg-alerts-prod \nMay 22, 2023 @Anzar Slack message from Logz Alerts in #critical-alerts-prod \n@Mohith disabled the alert for the sys test tenant: 1932\nCOLL-2243: Sys test triggering alertsTO DO \nMay 18, 2023 @Paranthaman Karthikeyan [Logz.io: Cloud Observability & Security Powered by Open Source \n] Alert for : [Prod] [Collab] CRUD failed for Survey InstanceLooks the product instance 19026 is no longer existing or is deleted \nbut pulse manage survey instance has a reference to it. \nThe tenant 1932 is not a real customer. Closing the ticket.\n1\n2\n3\n4\n5\n6\n7\n8TaskContext(customerTenantId=17686, vendorTenantId=176\nteamsOnboardingV2=false, genericTenantId=17687, pvtId=\nUser {\n    emailId: tony.stark@zw8yb.onmicrosoft.com\n    persona: 2000\n    name: user1\n}]),\n1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13An alert has been triggered due to an event in your Ax\nHigh severityHigh severity\n[Prod] [Collab] CRUD failed for Survey Instance\n[Int] [Collab] CRUD failed\nMay 22, 2023, 3:57 AM to May 22, 2023, 4:12 AM (UTC)\nAlert event samples\nSample 1 event out of 1:\n[ {\n \"date\" : 1.684728670693344E9,\n \"exception\" : {\n   \"exception_class\" : \"org.springframework.web.reacti\n   \"exception_message\" : \"404 Not Found from GET http:\n   \"stacktrace\" :", "doc_id": "69adae88-db95-4f48-81df-ac19ca8077e3", "embedding": null, "doc_hash": "04291c6d919c46bce3de41cfd59054e6f6749fbebe20fe95c507d9cc7725c4d5", "extra_info": {"page_label": "2", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 2865, "_node_type": "1"}, "relationships": {"1": "7bc79ea6-68b3-4bf5-8112-31e1aa58d448", "3": "21d0b6b7-28cf-437e-aef6-522813d4e0a1"}}, "__type__": "1"}, "21d0b6b7-28cf-437e-aef6-522813d4e0a1": {"__data__": {"text": ": \"404 Not Found from GET http:\n   \"stacktrace\" : \"org.springframework.web.reactive.fu\n1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n21\n22\n23\n24\n25\n26\n27\n28\n29Description : [Int] [Collab] CRUD failed\nEvent Samples : Sample 1 event out of 1:\n[ {\n\"date\" : 1.684383028945186E9,\n\"exception\" : {\n\"exception_class\" : \"org.springframework.web.reactive.\n\"exception_message\" : \"404 Not Found from GET http://c\n\"stacktrace\" : \"org.springframework.web.reactive.funct\n},\n\"logzio-signature\" : -1076969265,\n\"message_obj\" : {\n\"msg\" : \"Error reading pulse survey instance by id=147\n\"tenantId\" : 1932\n},\n\"source\" : \"stdout\",\n\"_logzio_logceptions\" : [ \"490f4bf7c48f4e57171fb9dbd5f\n\"type\" : \"fargate\",\n\"file\" : \"PulseSurveyInstancesApiService.java\",\n\"env_name\" : \"Production\",\n\"LogSize\" : 6759,\n\"line_number\" : 204,\n\"@version\" : 1,\n\"logger_name\" : \"com.axm.collaboration.pulse.manager.s\n\"class\" : \"com.axm.collaboration.pulse.manager.service\n\"source_host\" : \"ip-10-55-10-115.us-west-2.compute.int\n\"method\" : \"getPulseSurveyInstanceById\",\n\"level\" : \"ERROR\",\n\"mdc\" : {\n\"AWS-XRAY-TRACE-ID\" : \"1-6465a534-bb2fc63923b8da7bd31f", "doc_id": "21d0b6b7-28cf-437e-aef6-522813d4e0a1", "embedding": null, "doc_hash": "442829f5592cb598491ad09a9bf802b5ffd05c35f0acd8ed778933a106b6fbfd", "extra_info": {"page_label": "2", "file_name": "OnCall.pdf"}, "node_info": {"start": 2816, "end": 3923, "_node_type": "1"}, "relationships": {"1": "7bc79ea6-68b3-4bf5-8112-31e1aa58d448", "2": "69adae88-db95-4f48-81df-ac19ca8077e3"}}, "__type__": "1"}, "792e2728-bdc4-4e54-ad72-55e359056616": {"__data__": {"text": "May 17, 2023 @Shivam Parashar (Unlicensed) Repeated issue as below.\n[Logz.io: Cloud Observability & Security Powered by Open Source \n] Alert for : [Prod] [Collab] Get Access Token errors\n[invalid_token_response] An error occurred while \nattempting to retrieve the OAuth 2.0 Access Token \nResponse: Error while extracting response for type [class \norg.springframework.security.oauth2.core.endpoint.OAuth2Ac\ncessTokenResponse] and content type \n[application/json;charset=utf-8]; nested exception is \norg.springframework.http.converter.HttpMessageNotReadableE\nxception: An error occurred reading the OAuth 2.0 Access \nToken Response: Could not convert {ok=false, \nerror=invalid_refresh_token, warning=superfluous_charset, \nresponse_metadata={warnings=[superfluous_charset]}} to \nOAuth2AccessTokenResponse; nested exception is \norg.springframework.http.converter.HttpMessageConversionEx\nception: Could not convert {ok=false, \nerror=invalid_refresh_token, warning=superfluous_charset, \nresponse_metadata={warnings=[superfluous_charset]}} to \nOAuth2AccessTokenResponse(Same as below) \nTenant: 12,956 (Crowdstrike)\nTracking it here:COLL-2428: ON-CALL: invalid_refresh_token err\nor for Tenant 12956 (crowdstrike)TO DO \nMay 16, 2023 @Paranthaman Karthikeyan [Logz.io: Cloud Observability & Security Powered by Open Source \n] Alert for : [Prod] [Collab] Get Access Token errors\n[invalid_token_response] An error occurred while \nattempting to retrieve the OAuth 2.0 Access Token \nResponse: Error while extracting response for type [class \norg.springframework.security.oauth2.core.endpoint.OAuth2Ac\ncessTokenResponse] and content type \n[application/json;charset=utf-8]; nested exception is \norg.springframework.http.converter.HttpMessageNotReadableE\nxception: An error occurred reading the OAuth 2.0 Access \nToken Response: Could not convert {ok=false, \nerror=invalid_refresh_token, warning=superfluous_charset, \nresponse_metadata={warnings=[superfluous_charset]}} to \nOAuth2AccessTokenResponse; nested exception is \norg.springframework.http.converter.HttpMessageConversionEx\nception: Could not convert {ok=false, \nerror=invalid_refresh_token, warning=superfluous_charset, \nresponse_metadata={warnings=[superfluous_charset]}} to \nOAuth2AccessTokenResponseTenant: 12,956 (Crowdstrike)\nCan be tracked via COLL-2428: ON-CALL: invalid_refresh_token \nerror for Tenant 12956 (crowdstrike)TO DO \nMay 11, 2023 @Shivam Bhosale ALARM: \"stepfunction-alarms-production-\nStatemachinetenancytenantdeletion112E...\" in US West (Oregon)Already ticket exists.\nPLAT-1999: [Prod] Cleanup issue for system testsTO DO \nMay 8, 2023 @Sanjiv Ranjan stepfunction-alarms-production-Statemachinetenancytenantdeletion Generated by System test. \nPLAT-1999: [Prod] Cleanup issue for system testsTO DO \nMay 8, 2023 @Sanjiv Ranjan [Prod] [Disc] Report Generation Failed - \n\"message\" : \"Updated status as FAILED\",\n\"requestId\" : \"b631ae1f-126f-5ee3-b167-f2cd6b77f3b1\",\n\"responseData\" : {\n\"reportFormat\" : \"plain_text\",\n\"reportId\" : 185,\n\"reportUrl\" : \"1932/report_details_lKA1MaECut_1683519206621.pdf\",\n\"status\" : \"FAILED\"System", "doc_id": "792e2728-bdc4-4e54-ad72-55e359056616", "embedding": null, "doc_hash": "468aa6ba5b142eaf477fc77a16ac3860eb0a697cb185d98efe5952334915c1d9", "extra_info": {"page_label": "3", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 3081, "_node_type": "1"}, "relationships": {"1": "57018674-bf81-4ad7-940b-47bc8f22a82b", "3": "6f1adeda-e4d0-4b96-b2f1-57def8904cb6"}}, "__type__": "1"}, "6f1adeda-e4d0-4b96-b2f1-57def8904cb6": {"__data__": {"text": ": \"FAILED\"System test(1932) - Fix not deployed in prod yet.\nDISC-2836: Reporting Lambda: TimeoutError: Navigation timeout \nof 30000 msDONE \nUI-1945: Timeout while exporting  qualitative comments from UI \nDONE \nApr 26, 2023 @Syed Ummar Farooqh [Prod] [Plat] ERROR logs > 100 in 15mins\n<a \nhref=https://pinstripe.atlassian.net/wiki/spaces/PRODUCT/pages/2772\n82829/Run+books#GLOBAL-ERROR-LOGS>Run book entry</a>\nApr 26, 2023, 3:27 PM to Apr 26, 2023, 3:42 PM (UTC)\nAlert event samplestenantId: 11923 (mattel)\nIssue: COLL-2297: Handle or supress error log for pulse survey fil\nter failureDONE \nSlack thread: Slack message from Anil Kumar Ganivada in #on-ca\nll \n30\n31\n32\n33\n34\n35\n36\n37\n38\n39\n40\n41\n42\n43\"requestId\" : \"98d497bd-c77c-4019-94d0-3ec164cdbb0e\",\n\"span_id\" : \"8444a8a663ee5401\",\n\"trace_flags\" : \"01\",\n\"trace_id\" : \"6465a534bb2fc63923b8da7bd31f4d84\"\n},\n\"ecs_task_definition\" : \"pulsemanagerstackproductionpu\n\"tags\" : [ \"_logz_http_bulk_json_8070\" ],\n\"ecs_task_arn\" : \"arn:aws:ecs:us-west-2:643306803378:t\n\"@timestamp\" : \"2023-05-18T04:10:28.943+0000\",\n\"container_name\" : \"pulse-manager-task-container\",\n\"thread_name\" : \"http-nio-8080-exec-38\",\n\"ecs_cluster\" : \"pulse-manager-stack-production-pulsem\n\"container_id\" : \"7d2ae71648394ee0a8b6b763858fc2a1-585\n} ] \n", "doc_id": "6f1adeda-e4d0-4b96-b2f1-57def8904cb6", "embedding": null, "doc_hash": "ac81899b7dbe7fc3f0421bc53a447ed09c476c48ace64bf033962da90442f5c8", "extra_info": {"page_label": "3", "file_name": "OnCall.pdf"}, "node_info": {"start": 3065, "end": 4328, "_node_type": "1"}, "relationships": {"1": "57018674-bf81-4ad7-940b-47bc8f22a82b", "2": "792e2728-bdc4-4e54-ad72-55e359056616"}}, "__type__": "1"}, "07aa42dc-7b17-4b9f-848a-181c4ec7e77f": {"__data__": {"text": "The following have met the condition:\n[ {\n \"container_name\" : \"pulse-manager-task-container\",\n \"message_obj.tenantId\" : \"11923\",\n \"count\" : 102.0\n} ]\nApr 24, 2023 @Anil Ganivada #549: [Logz.io] Alert for : [Prod] [Collab] Failed brokering \nmessage to Slack\nDescription : #COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-\nSLACK\nEvent Samples : Sample 2 events out of 2:\n[ {\n \"date\" : 1.682352822403735E9,\n \"exception\" : {\n   \"exception_class\" : \n\"com.axm.collaboration.message.broker.exception.MessageBrokerEx\nception\",\n   \"exception_message\" : \"Exception while replying to the activity in \nteams.\",\n   \"stacktrace\" : \n\"com.axm.collaboration.message.broker.exception.MessageBrokerEx\nception: Exception while replying to the activity in teams.\\n\\tat \ncom.axm.collaboration.message.broker.teams.channel.TeamsBotServi\nce.replyMessage(TeamsBotService.java:298)\\n\\tat \ncom.axm.collaboration.message.broker.teams.service.TeamsMessagi\nngService.postReaction(TeamsMessagingService.java:154)\\n\\tat \ncom.axm.collaboration.message.broker.slack.handler.SlackReactionH\nandler.handleEvent(SlackReactionHandler.java:43)\\n\\tat \ncom.axm.collaboration.message.broker.slack.handler.SlackReactionH\nandler.handleEvent(SlackReactionHandler.java:17)\\n\\tat \ncom.axm.collaboration.message.broker.slack.Brokering failed while finding mappings for 2 reactions \n\u201cack-splunk\u201d => \nhttps://app.logz.io/#/goto/d7d81770f45a1ce54f75899b0f361fc8?\nswitchToAccountId=389045\n\u201cthanksplunk\u201d => \nhttps://app.logz.io/#/goto/c4064a89b23405c36abd8bd944ee4352?\nswitchToAccountId=389045\nbased on the thread Slack message from Nayan Srivastava in #o\nn-call  we are already sending message when we are unable to find \nmapped emoji. However, we were not able to decode the 404 \nresponse from bot added COLL-2308: Find rootcause for 404 whil\ne brocessing slack event during cross brokeringTO DO  to \ninvestigate this issue further\nApr 20, 2023 @Anil Ganivada [Prod] [Plat] ERROR logs > 100 in 15mins\n<a \nhref=https://pinstripe.atlassian.net/wiki/spaces/PRODUCT/pages/2772\n82829/Run+books#GLOBAL-ERROR-LOGS>Run book entry</a>\nApr 20, 2023, 12:54 PM to Apr 20, 2023, 1:09 PM (UTC)\nAlert event samples\nThe following have met the condition:\n[ {\n \"container_name\" : \"pulse-manager-task-container\",\n \"message_obj.tenantId\" : \"8677\",\n \"count\" : 124.0\n} ]COLL-2297: Handle or supress error log for pulse survey filter fail\nureDONE \n \nslack thread: Slack message from Anil Kumar Ganivada in #on-ca\nll \nApr 18, 2023 @Anil Ganivada #COLLAB-ERROR-RESPONSES-TO-TEAMS-OR-SLACK\nApr 18, 2023, 4:57 PM to Apr 18, 2023, 5:12 PM (UTC)\nAlert event samples\nThe following have met the condition:\n[ {\n \"message_obj.RequestURI\" : \"/sentiment-collector/api/messages\",\n \"count\" :", "doc_id": "07aa42dc-7b17-4b9f-848a-181c4ec7e77f", "embedding": null, "doc_hash": "e293beddafaeba9c2e7920144bfa4f83273e02ec83f6018e14e50352775b4066", "extra_info": {"page_label": "4", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 2693, "_node_type": "1"}, "relationships": {"1": "04678987-bdb0-4922-a081-c6066b190cff", "3": "d14ad41a-0e46-4fb9-b979-8973ba836de4"}}, "__type__": "1"}, "d14ad41a-0e46-4fb9-b979-8973ba836de4": {"__data__": {"text": ": \"/sentiment-collector/api/messages\",\n \"count\" : 2.0\n} ]COLL-2294: Debug 500 response from sentiment collector endpoi\nnt \"/sentiment-collector/api/messages\"TO DO \n \nSlack thread: Slack message from Anil Kumar Ganivada in #on-ca\nll \nApr 18, 2023 @Anil Ganivada Critical Threshold Violated: COLLAB-SLACK-INCOMING-\nREQUEST-RESPONSE-TIME\nProject axiamatic-prod-1067bb71\nAlert COLLAB-SLACK-INCOMING-REQUEST-RESPONSE-TIME\nDescription\nIf we take more than 3 seconds we will start seeing errors in slack bot\nExpression\nCritical Threshold Violated: alert 'COLLAB-SLACK-INCOMING-\nREQUEST-RESPONSE-TIME' is above 3 (value is 4.82)Subsequent operation on same path succeeded \nhttps://app.logz.io/#/goto/7d5e8312cc8bd5fe89c71625455c06\ne6?switchToAccountId=429760 \nmajor bottleneck was due to the call GET /message-\nbroker/api/v1/broker/pvt-lookup/slack?\nteamId=T02649E326S&channelId=C053A8RTCP9 which took about \n4.3 secs  updating findings in COLL-2264\n", "doc_id": "d14ad41a-0e46-4fb9-b979-8973ba836de4", "embedding": null, "doc_hash": "6522631f31a4eb34a2c2b544744221bb4786da92958151c2bdcbf5885c718333", "extra_info": {"page_label": "4", "file_name": "OnCall.pdf"}, "node_info": {"start": 2644, "end": 3586, "_node_type": "1"}, "relationships": {"1": "04678987-bdb0-4922-a081-c6066b190cff", "2": "07aa42dc-7b17-4b9f-848a-181c4ec7e77f"}}, "__type__": "1"}, "e5834cdd-0e9d-484d-a7e8-721644c8e119": {"__data__": {"text": "Apr 18, 2023 @Sanjiv Ranjan ERROR logs > 100 in 15mins. {\n\"container_name\" : \"pulse-manager-task-container\",\n\"message_obj.tenantId\" : \"8677\",\n\"count\" : 124.0\n} ]Already toned down log level to WARN - Fix is available in INT .\nApr 17, 2023 @Pradeep #539: ALARM: \"stepfunction-alarms-production-\nproductionTEAMSCONNECTstepfunctionala...\" in US West (Oregon)\nconnect-api failed for system test tenant due to pvt getting deleted. \nthis could be due parallel execution of tests.TEST-739: [on-call]  Investigate step function failure in prod in test \ntenant 3210DONE \nApr 14, 2023 @Pradeep 404 error on connected-products API in actions tab in web\nSlack message from Aryan Nair in #on-call \nThis was due to a stale entry in MB , we can use the new table in \ncore-data which has pvt-vvt mapping instead. Created a tacker for \nAPPs to move the APIDISC-2827: [on-call] Move connected products call from message \nbroker to core-data DONE \nApr 12, 2023 @Guruprasad Shenoy COLLAB-ERROR-RESPONSES-TO-TEAMS-OR-SLACK\nSlack message from Logz Alerts in #reg-alerts-prod COLL-1249: [On-call] [Investigate]  /sentiment-collector/api/messa\nges is throwing JSONParseException and returning 401DONE \nCOLL-2269: Create 3rdparty repo for https://github.com/microsoft/\nbotbuilder-java to leverage latest fixes pushed to itDONE \nApr 12, 2023 @Guruprasad Shenoy COLLAB-SLACK-INCOMING-REQUEST-RESPONSE-TIME\nActual Value\n3.14\nAttributes\napp:sentiment-collector, env:production, uri:/slack/slash/addressBook\nSlack message from Lightstep Alerts in #critical-alerts-prod \nRCA : TBDThe command worked on subsequent attempt. \nFiled Jira for tracking\nCOLL-2264: Alert for : [Prod] [Collab] /slack/slash/addressBook to\nok more than 3 secondsTO DO \nApr 11, 2023 @Pradeep Remove unnecessary check pvtAssociations.pvt.id!=null which \nprobably is causing issues\nslack: Slack message from Rama Taraniganty in #on-call DISC-2808: [on-call] Remove unnecessary check pvtAssociation\ns.pvt.id!=nullDONE \nApr 11, 2023 @Pradeep There was a failure while adding users. Call failed on url \n'/dashboard-app/api/v1/async/verify/pvt-\nuser/status/validateUsers:b4687e0d-e407-42c4-a391-\n2f3e710bf495'\nThis seems to be due to connection reset issues that we have seen \nwith reports as well. \ntraceid - 64357c9def4d953829939cadf8851e7e\nslack: Slack message from Anil Kumar Ganivada in #on-call Filed Jira for investigating further - DISC-2807: [Investigate] 500 e\nrror due connection reset by peer for validate user callTO DO \nApr 11, 2023 @Pradeep Exception on onTeamsUserEvent Already filed - COLL-1739: Production Exception: No value prese\nntTO DO \nApr 4, 2023 @Harinandan Chintamreddy [Prod] [Disc] Report Partial Generation Failed\nSlack message from Logz Alerts in #critical-alerts-prod \nRCA: TBD \nApr 4, 2023 @Harinandan Chintamreddy [Prod] [Collab] CRUD failed for Survey Instance\nApr 4, 2023, 1:32 PM to Apr 4, 2023, 1:47 PM (UTC)\n \"axm.user\" : \"system_user@axiamatic.com\",\nNo instances found for query filter", "doc_id": "e5834cdd-0e9d-484d-a7e8-721644c8e119", "embedding": null, "doc_hash": "1d176f65dedef8d5ad0687715f9c339683b887090d0401a901c6c6978b0feb9a", "extra_info": {"page_label": "5", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 2960, "_node_type": "1"}, "relationships": {"1": "130a58d8-6265-4783-b0a6-056344dacdb8", "3": "4cc6e453-d8cb-4316-9f00-1ad8fc6152ad"}}, "__type__": "1"}, "4cc6e453-d8cb-4316-9f00-1ad8fc6152ad": {"__data__": {"text": ": \"system_user@axiamatic.com\",\nNo instances found for query filter \npulseSurvey.productInstanceId==3695;pulseSurvey.tenantId==8677\nSlack message from Logz Alerts in #critical-alerts-prod \n \n[Prod] [Collab] CRUD failed for Survey Instance\nApr 4, 2023, 2:58 PM to Apr 4, 2023, 3:13 PM (UTC)\n \"axm.user\" : \"system_user@axiamatic.com\",\nNo instances found for query filter \npulseSurvey.productInstanceId==3695;pulseSurvey.tenantId==8677\nSlack message from Logz Alerts in #critical-alerts-prod \n \n[Prod] [Collab] CRUD failed for Survey Instance\nApr 4, 2023, 4:05 PM to Apr 4, 2023, 4:20 PM (UTC)\n\"axm.user\" : \"abenjamin@ea.com\", \n", "doc_id": "4cc6e453-d8cb-4316-9f00-1ad8fc6152ad", "embedding": null, "doc_hash": "421638b67a9017a993430a131cb489f7c2e389add8bbd170c41fb9342cfbea24", "extra_info": {"page_label": "5", "file_name": "OnCall.pdf"}, "node_info": {"start": 2894, "end": 3518, "_node_type": "1"}, "relationships": {"1": "130a58d8-6265-4783-b0a6-056344dacdb8", "2": "e5834cdd-0e9d-484d-a7e8-721644c8e119"}}, "__type__": "1"}, "975881aa-a45e-4b2c-83a8-bd6932e4a819": {"__data__": {"text": "No instances found for query filter \npulseSurvey.productInstanceId==8928;endedAt==null;pulseSurvey.ten\nantId==8677\nSlack message from Logz Alerts in #critical-alerts-prod \n \nRCA: This log line should not be alerted on there is a discussion in \nslack regarding this Slack message from Rama Taraniganty in #op\nsgenie-alerts \nEssentially this was a change that app team requested to return 404 \ninstead of empty response and this response is handled as expected\nSlack message from Kiran Marshall in #apps-ui \n \nThe first two occurrences were when we were internally accessing EA \ntenant but @Anil Ganivada confirmed they did not notice any issues \nin the UI\nThe last access was by a user from EA which needs investigation\nApr 4, 2023 @Harinandan Chintamreddy [Prod] [Collab] CRUD failed for Survey Instance\nApr 4, 2023, 12:26 PM to Apr 4, 2023, 12:41 PM (UTC)\nSlack message from Logz Alerts in #critical-alerts-prod \nRCA: This happened when a sys test created tracker to resolve the \nissue\n COLL-2243: Sys test triggering alertsTO DO \nApr 4, 2023 @Harinandan Chintamreddy #521: [Logz.io] Alert for : [PROD] [PLAT] user invitation with \npermissions failed\nSlack message from Logz Alerts in #critical-alerts-prod \n \nRCA: Per @Anil Ganivada this is most likely system test regression \nissue due to change in payload as this request came from user agent \npython \nApr 4, 2023 @Harinandan Chintamreddy #520: [Logz.io] Alert for : [PROD] [PLAT] Update or replace \npermissions failed for user\nRCA: Per @Anil Ganivada this is most likely system test regression \nissue due to change in payload as this request came from user agent \npython also this failure is the same as next alert \nApr 4, 2023 @Harinandan Chintamreddy [Prod] [VMS] DATA-SCORING-SQL-DATA-ACCESS-ERROR\nInvestigation is being tracked on this thread\nSlack message from Opsgenie in #opsgenie-alerts \nRCA: TBD@ritu vaidya confirmed the fix is in 0.9.10, but we are looking to \nbackport a HF to 0.9.9\nApps :  DISC-2619: Dashboard App : Timeline Filters : Fix the impl \nas it is making O(N) downstream api callsDONE \nUI : UI-1720: Frontend : Update API for timeline filters for Dashbo\nard & Vendor AppDONE \nApr 4, 2023 @Harinandan Chintamreddy [Prod] [Collab] CRUD failed for Survey Instance\nSlack message from Logz Alerts in #critical-alerts-prod \n \nThis error occurred for EA tenant ID 8677\n \nRCA: Per @ritu vaidya This is an intermittent/transient issue when \npulse app was still 0.9.8 and pulse-manager was 0.9.9 \nFrom prod-deployments\n#pulse-manager-service was deployed at 4:50 PM\nSlack message from Deployer in #prod-deployments \n#pulsesurvey-app-service was deployed at 5:29 PM\nSlack message from Deployer in #prod-deployments \n \nAnd the issue was seen at 17:10:31.357 \nApr 4, 2023 @Harinandan Chintamreddy [Prod][Plat] Service Restarting Repeatedly\n \"container_name\" : \"reporting-task-container\",\nSlack message from Logz Alerts in #critical-alerts-prod \nRCA: This was a resource issue  per @Anil Ganivada whcih should \nnow be resolved\n  \n", "doc_id": "975881aa-a45e-4b2c-83a8-bd6932e4a819", "embedding": null, "doc_hash": "8a28314bb5126884074d743b913cdfbee6aeecb9683fb8fe916f6b87e89ef6ee", "extra_info": {"page_label": "6", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 2995, "_node_type": "1"}, "relationships": {"1": "bd4335e6-72c5-40f0-ac46-bfd30bc3b26f"}}, "__type__": "1"}, "26d56212-a665-414f-803a-387a807bffa0": {"__data__": {"text": "Apr 4, 2023 @Harinandan Chintamreddy [Prod][Plat] Service Restarting Repeatedly\n \"container_name\" : \"score-provider-task-container\",\nSlack message from Logz Alerts in #critical-alerts-prod \n \nRCA: This looks like issue in prod deployment \nApr 3, 2023 @ritu vaidya #520: [Logz.io] Alert for : [PROD] [PLAT] Update or replace \npermissions failed for userPLAT-1880: [On call] : Runtime Exception while updating the user \npermissionTO DO \nMar 28, 2023 @Nayan Srivastava  [Prod] [Disc] Report Partial Generation Failed DISC-2723: [ON-CALL] [Partial Report Generation] Report genera\ntion API failure: unauthorizedTO DO \nMar 28, 2023 @ritu vaidya [Prod] [Collab] Unable to create team and channel in teams\nexception_message\" : \"Error code: BadRequest\\nError message: \nFailed to execute Templates backend request \nCreateTeamFromTemplateRequest. Request Url: \nhttps://teams.microsoft.com/fabric/amer/templates/api/team,Need to add teams name length validation : COLL-2106:   Micros\noft - Teams name length ValidationTO DO \nMar 21, 2023 @Gaurav Dahiya [PROD][PLAT] ONBOARDING_ERROR_IN_INFO_CALL Closing this after discussion on slack - Slack message from Logz \nAlerts in #critical-alerts-prod \nMar 21, 2023 @Sanjiv Ranjan ALARM: \"stepfunction-alarms-production-\nproductionSLACKCONNECTstepfunctionalaSlack channel creation is failing with invalid chars .     Tracking ticket \n: COLL-2113: Slack channel creation is failing with invalid chars \nTO DO \nMar 20, 2023 @Sanjiv Ranjan [Prod] [Collab] Unable to create team and channel in teams\nexception_message\" : \"Error code: BadRequest\\nError message: \nFailed to execute Templates backend request \nCreateTeamFromTemplateRequest. Request Url: \nhttps://teams.microsoft.com/fabric/amer/templates/api/team,Need to add teams name length validation : COLL-2106:   Micros\noft - Teams name length ValidationTO DO \nMar 19, 2023 @Sanjiv Ranjan Alert for : [Prod][Collab] Error responses sent to slack or teams 401 : POST /sentiment-collector/api/messages\n \nMar 14, 2023 @Shivam Bhosale Alert for : [Prod] [Collab] Failed brokering message to Slack COLL-2084: Alert for : [Prod] [Collab] Failed brokering message to \nSlackTO DO \nThis happned due to corrupted data. Collab team is looking into this \nissue.\nMar 14, 2023 @Paranthaman Karthikeyan [Prod] [Disc] Report Partial Generation Failed Seems a 500 (Internal server error) when pulsesurvey-\napp/api/v1/pulse-survey-tracking/pulse-survey-\ninstances/dimensions/performance?\nproductInstanceIdQuery=8928&pulseSurveyInstanceIdList=10290&p\nulseSurveyInstanceIdList=10291 was called. \nTagged apps team to confirm if it was an issue overall the message \nsay report generation completed. Conversations recorded as part of \nslack thread:\nSlack message from Logz Alerts in #critical-alerts-prod \nThe below trackers are created:\n DISC-2626: [ON_CALL]: Pulse App: Null pointer exception encou\nntered at the time of report generation for EADUPLICATE \nDAT-903: Discrepancy between api response - when called from r\neporting service vs UI DONE \nMar 14, 2023 @Paranthaman Karthikeyan [Prod] [Plat] ERROR logs > 100 in 15mins This was due to sentiment-collector. EA closed their pulse. Since we \nhad run into issue previously and", "doc_id": "26d56212-a665-414f-803a-387a807bffa0", "embedding": null, "doc_hash": "d7ac5b9c0ab787ad4760d17bb0aadddcf494ad86227d369624429af714871f64", "extra_info": {"page_label": "7", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 3189, "_node_type": "1"}, "relationships": {"1": "0bd2d94e-126a-4644-8989-aa46455d3788", "3": "ca972543-cc22-4e27-a596-6492703c69ce"}}, "__type__": "1"}, "ca972543-cc22-4e27-a596-6492703c69ce": {"__data__": {"text": "EA closed their pulse. Since we \nhad run into issue previously and had deleted the pulse messages \nthere were issues of message_not_found (74) and we also hit rate \nlimiting 3 times. The expiry message was successfully delivered to \n187 users.\nMar 7, 2023 @Syed Ummar Farooqh \n@Rama Taraniganty [Prod] [Disc] Report Partial Generation Failed Issue with pulse-survey-app, an API call made by reporting to render \nsome data. There is a temprary connection reset which happens \nwhich results in one of the charts/components not being loaded in \nthe report, this must be further investigated.\n(Short-term) DISC-2564: Pulsesurvey app: failed: Connection res\net by peerQA-READY \n(Long term) DISC-2593: [On-call] [Arch] Need to find a way to inc\norporate retries in OpenAPI generated code to prevent reports from \nhaving empty sectionsNOT REQUIRED \nRegenerating should resolve this temporary issue\n", "doc_id": "ca972543-cc22-4e27-a596-6492703c69ce", "embedding": null, "doc_hash": "f423df9dcf495ebde76dc87f0ed83ee90510a32f59cf6074b61c5a2f50ec8489", "extra_info": {"page_label": "7", "file_name": "OnCall.pdf"}, "node_info": {"start": 3123, "end": 4014, "_node_type": "1"}, "relationships": {"1": "0bd2d94e-126a-4644-8989-aa46455d3788", "2": "26d56212-a665-414f-803a-387a807bffa0"}}, "__type__": "1"}, "02048f03-be3c-4ebf-9383-5e85fd3356e1": {"__data__": {"text": "Mar 1, 2023 @Syed Ummar Farooqh [Prod] [Collab] Failed brokering message to Slack COLL-1872: [ON-CALL] : Silently Ignore slack event with subtype \n\"channel_join\" and \"channel_leave\"DONE \nFeb 28, 2023 @Syed Ummar Farooqh [Prod] [Collab] Failed brokering message to Slack COLL-1872: [ON-CALL] : Silently Ignore slack event with subtype \n\"channel_join\" and \"channel_leave\"DONE \nFeb 27,2023 @Anil Ganivada [Prod] [Collab] Failed brokering message to Slack  COLL-1872: [ON-CALL] : Silently Ignore slack event with subtype \n\"channel_join\" and \"channel_leave\"DONE \nwill be fixed as part of 0.9.8\nFeb 24,2023: 17:40 IST @Anzar [Prod] [Collab] Failed brokering message to Slack  COLL-1872: [ON-CALL] : Silently Ignore slack event with subtype \n\"channel_join\" and \"channel_leave\"DONE \nwill be fixed as part of 0.9.8\nFeb 24,2023 @Anil Ganivada [Prod] [Collab] Failed brokering message to Slack  COLL-1872: [ON-CALL] : Silently Ignore slack event with subtype \n\"channel_join\" and \"channel_leave\"DONE \nwill be fixed as part of 0.9.8\nFeb 24,2023 @Anil Ganivada Opsgenie Alert: ALARM: \"stepfunction-alarms-production-\nproductionSLACKCONNECTstepfunctionala...\"Race condition issue on system_test account COLL-1966: Slack c\nhannel creation failed due to race conditionDONE \nFeb 23,2023 @Anil Ganivada Tenancy failed to fetch customers of a vendor Related to perft test calls to tenancy\nFeb 23,2023 @Anil Ganivada Enable API Access errors https://app.logz.io/#/goto/82ab076e3d68bc7005d2b55d9df4a1\n01?switchToAccountId=389045 \nmost likely related to perf test since service was waiting for DB \nconnection\nFeb 23,2023 @Anil Ganivada GLOBAL-HIKARI-IDLE-CONNECTIONS Related to perft test calls to tenancy\nFeb 23,2023 @Anil Ganivada [Prod] [Collab] Failed brokering message to Slack COLL-1872: [ON-CALL] : Silently Ignore slack event with subtype \n\"channel_join\" and \"channel_leave\"DONE \nwill be fixed as part of 0.9.8\nFeb 23,2023 @Anil Ganivada [Prod][Collab] Error responses sent to slack or teams https://app.logz.io/#/goto/b48070f66b0cb7592576c31c2d0a1496?\nswitchToAccountId=389045\nCOLL-1249: [On-call] [Investigate]  /sentiment-collector/api/messa\nges is throwing JSONParseException and returning 401DONE \nthis is a known issue - The fix is in the bot SDK 4.15.0-SNAPSHOT \nwhich is not released yet currently we use 4.14.x in our code.\nSlack message from Harinandan Chintamreddy in #on-call \nFeb 22,2023 @Anil Ganivada [Prod] [Collab] Failed brokering message to Slack COLL-1872: [ON-CALL] : Silently Ignore slack event with subtype \n\"channel_join\" and \"channel_leave\"DONE \nwill be fixed as part of 0.9.8\nFeb 21, 2023 @Anil Ganivada [PROD] [PLAT] - Tenancy failed to fetch customers of a vendor This has happened once in prod so far it seems like an exception \ntrigerring only for getAllCustomerLinkedToVendor in tenancy. This \nis not a", "doc_id": "02048f03-be3c-4ebf-9383-5e85fd3356e1", "embedding": null, "doc_hash": "3ec06fd5ac2c5df8dca93e0230688bddf501838001a915f4a72d6fc1af4a78e1", "extra_info": {"page_label": "8", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 2818, "_node_type": "1"}, "relationships": {"1": "d09c18d3-ac4b-4719-a3f1-9ca73015e9c2", "3": "9cf96cfc-0673-4806-b906-e4dab4f6aa0e"}}, "__type__": "1"}, "9cf96cfc-0673-4806-b906-e4dab4f6aa0e": {"__data__": {"text": "only for getAllCustomerLinkedToVendor in tenancy. This \nis not a major error since the API call itself is a safety measure to \nhandle vendor assignments but we need to figure out the root cause \nand fix the issue - added PLAT-1712: Handle \"Cannot ask for requ\nest attribute - request is not active anymore!\" when tenancy makes a \ncall to core dataDONE  to handle this\nFeb 20, 2023 @Pradeep [Prod] [Collab] CRUD failed for Survey Instance Being discussed here - This has happened couple of times. Still \nunder investigation\nFeb 20, 2023 @Pradeep [Prod] [Collab] App not published to AppCatalog\n#448: ALARM: \"stepfunction-alarms-production-\nproductionTEAMSCONNECTstepfunctionala...\" in US West (Oregon)This was due to a new Teams Account where we did not upload the \nprod app. App is uploaded now by bhavana. Regression to be rerun \nto verify.\nSlack message from bhavana in #opsgenie-alerts \nFeb 20, 2023 @Pradeep [Prod] [Collab] Failed brokering message to Slack COLL-1872: [ON-CALL] : Silently Ignore slack event with subtype \n\"channel_join\" and \"channel_leave\"DONE \nCOLL-1901: [On-call] [Investigate] : Getting handler not found for \nFailed brokering message to SlackDUPLICATE \nFeb 16, 2023 @Pradeep \n@Rama Taraniganty [PROD] [PLAT] PERMISSIONS-ERROR-ON-RESOURCE PLAT-1642: Product admin while trying to edit priority - 500 error i\ns thrownDONE \nSlack message from Anil Kumar Ganivada in #critical-alerts-prod \nTemporary fix is to update auth0 user\u2019s metadata add the following \nline in axm_permissions array:\n<tenant-id>::customer::<tenant-id>:productInstance:\n<product-instance-id>::readWrite\nFor example:\n", "doc_id": "9cf96cfc-0673-4806-b906-e4dab4f6aa0e", "embedding": null, "doc_hash": "f847cda6d3bfbd372510c419f3c441db5dd3b844749ac665292b54549c535324", "extra_info": {"page_label": "8", "file_name": "OnCall.pdf"}, "node_info": {"start": 2754, "end": 4363, "_node_type": "1"}, "relationships": {"1": "d09c18d3-ac4b-4719-a3f1-9ca73015e9c2", "2": "02048f03-be3c-4ebf-9383-5e85fd3356e1"}}, "__type__": "1"}, "9b16e0e5-8641-4e16-a4f7-67130cdd3a59": {"__data__": {"text": "14759::customer::14759:productInstance:13808::readWrite\nAdditional details:\nSlack message from Anil Kumar Ganivada in #on-call \nB\nFeb 15, 2023 \nFeb 17, 2023 @ritu vaidya \n@Pradeep \n@ritu vaidya #414: [Logz.io] Alert for : [Prod] [Collab] Get Access Token errors\n#420: [Logz.io] Alert for : [Prod] [Collab] Get Access Token errors\n#430: [Logz.io] Alert for : [Prod] [Collab] Get Access Token errors\n#436: [Logz.io] Alert for : [Prod] [Collab] Get Access Token errors\n#437: [Logz.io] Alert for : [Prod] [Collab] Get Access Token errorsTenant: 3209  (anu-slack)\nresponse_metadata={warnings=[superfluous_charset]\nCOLL-1904: [On-call] [Investigate] Failed to refresh token due to \nHttpMessageNotReadableExceptionTO DO \nFeb 14, 2023 \nFeb 15, 2023 (multiple \ntimes)@Rama Taraniganty \n@ritu vaidya #413: [Logz.io] Alert for : [Prod] [Collab] Get Access Token errors\n#419: [Logz.io] Alert for : [Prod] [Collab] Get Access Token errors\n \n[invalid_token_response] An error occurred while \nattempting to retrieve the OAuth 2.0 Access Token \nResponse: Error while extracting response for type [class \norg.springframework.security.oauth2.core.endpoint.OAuth2Ac\ncessTokenResponse] and content type \n[application/json;charset=utf-8]; nested exception is \norg.springframework.http.converter.HttpMessageNotReadableE\nxception: An error occurred reading the OAuth 2.0 Access \nToken Response: Could not convert {ok=false, \nerror=invalid_refresh_token, warning=superfluous_charset, \nresponse_metadata={warnings=[superfluous_charset]}} to \nOAuth2AccessTokenResponse; nested exception is \norg.springframework.http.converter.HttpMessageConversionEx\nception: Could not convert {ok=false, \nerror=invalid_refresh_token, warning=superfluous_charset, \nresponse_metadata={warnings=[superfluous_charset]}} to \nOAuth2AccessTokenResponseTenant: 3209  (anu-slack)\nresponse_metadata={warnings=[superfluous_charset]\nCOLL-1904: [On-call] [Investigate] Failed to refresh token due to \nHttpMessageNotReadableExceptionTO DO \nMdc trace id: 63ebedf85fc7236ae35314f85fc0721e\nMaybe we could log more?\nFeb 14, 2023 \nFeb 15, 2023 @ritu vaidya #411: [Logz.io] Alert for : [Prod] [Collab] Failed brokering message to \nSlack\njava.lang.IllegalArgumentException: Could not find a \nhandler for event at \ncom.axm.collaboration.message.broker.slack.handler.SlackEv\nentHandlerRegistry.handleEvent(SlackEventHandlerRegistry.j\nava:36) at \ncom.axm.collaboration.message.broker.slack.processor.event\n.SlackEventProcessor.processEvent(SlackEventProcessor.java\n:78) at \ncom.axm.collaboration.message.broker.slack.processor.event\n.ClassicSlackEventProcessor.processEvent(ClassicSlackEvent\nProcessor.java:39) at \ncom.axm.collaboration.message.broker.event.common.MessageD\neliveryController.processSlackEvent(MessageDeliveryControl\nler.java:156) at \ncom.axm.collaboration.message.broker.event.common.MessageD\neliveryController.processData(MessageDeliveryController.ja\nva:77) at \njdk.internal.reflect.GeneratedMethodAccessor838.invoke(Unk\nnown Source)COLL-1901:", "doc_id": "9b16e0e5-8641-4e16-a4f7-67130cdd3a59", "embedding": null, "doc_hash": "cfaaada74b10d4e9a4bab7716a674baed2bc3b9ea87aa0d38103027d8d0aae3c", "extra_info": {"page_label": "9", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 2995, "_node_type": "1"}, "relationships": {"1": "ce7f46b5-953b-4cb8-b961-183b46bc7645", "3": "7f0dd2c8-c705-447e-b1f0-55556cf61ad4"}}, "__type__": "1"}, "7f0dd2c8-c705-447e-b1f0-55556cf61ad4": {"__data__": {"text": "Source)COLL-1901: [On-call] [Investigate] : Getting handler not found for \nFailed brokering message to SlackDUPLICATE \nSlack message from Opsgenie in #opsgenie-alerts \n", "doc_id": "7f0dd2c8-c705-447e-b1f0-55556cf61ad4", "embedding": null, "doc_hash": "22889ef4aaa4e83fb7aa65672e44a170330a352aedc9af9484a65dfa8cd19cf9", "extra_info": {"page_label": "9", "file_name": "OnCall.pdf"}, "node_info": {"start": 2978, "end": 3146, "_node_type": "1"}, "relationships": {"1": "ce7f46b5-953b-4cb8-b961-183b46bc7645", "2": "9b16e0e5-8641-4e16-a4f7-67130cdd3a59"}}, "__type__": "1"}, "7ce57955-6637-4a9a-85e0-3b8d16306141": {"__data__": {"text": "Feb 14, 2023 \nFeb 15, 2023 @ritu vaidya #410: ALARM: \"stepfunction-alarms-production-\nproductionTEAMSCONNECTstepfunctionala...\" in US West (Oregon)\nUnderlying issue is same as \u201cApp not published to AppCatalog\u201d \nFeb 14, 2023 \nFeb 15, 2023 @ritu vaidya  #409: [Logz.io] Alert for : [Prod] [Collab] App not published to \nAppCatalog\nError in log : \nResponse body={\u201cerror\u201d:\n{\u201ccode\u201d:\u201cInvalidAuthenticationToken\u201d,\u201cmessage\u201d:\u201cAccess token has \nexpired or is not yet valid.\u201c,\u201dinnerError\u201d:{\u201cdate\u201d:\u201c2023-02-\n14T05:13:29\",\u201crequest-id\u201d:\u201cac16bf7f-bfd9-43a2-9782-\n6ab85f2982ba\u201d,\u201cclient-request-id\u201d:\u201cac16bf7f-bfd9-43a2-9782-\n6ab85f2982ba\u201d}}}, Token expired for tenant fetching new token \nuser=nullSlack message from Ritu Vaidya in #opsgenie-alerts \nSummarizing the discussion here :\n1. There was no \u201cdev\u201d activity/api-calls on this account for > 90 days \nand hence the license expired.\n2. Options :\nDisable this test from suite till we have another account in \nplace\nIgnore this alert going fwd for test tenant\n1. add a new dev account and use the credentials for this test in \nProd/Stage or INT\n2. we need to make sure that some brokering happens on these \naccount otherwise they will be marked as no dev activity and will \nexpire\nLogged TEST-584: [On-call] [Investigate] : Update the system \ntest with valid MS dev account detailsDONE \nFeb 13, 2023 @Rama Taraniganty [Logz.io] Alert for : [Prod] [Collab] Failed brokering message to Slack\nCOLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACK\nFound this in the exception message:https://pinstripe.app.opsgenie.com/alert/detail/0abe40dc-245a-4760-\naeed-6f13e17dbf38-1676318247656/details\nCOLL-1898: [On-call] [Investigate] Failed brokering message to Sl\nack with a 404 TO DO \n13 Feb 2023 @Guruprasad Shenoy Add users to PVT failed for EA when getting user info from slack DISC-2386: Bulk Add user to PVT is failing - SlackDONE \n9 Feb 2023 @Satya Vummidi [LightStep] Critical Threshold Violated: GLOBAL-HIKARI-IDLE-\nCONNECTIONSThis issue happened to Tenancy service. It happened for a very short \ntime but it recovered. \nStarted a slack thread Slack message from Satya Vummidi in #on\n-call \n9 Feb 2023 @Prabhu Pant COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACK\nhttps://app.logz.io/#/dashboard/osd/discover/?_a=(columns:!\n(message),filters:!(),index:'logzioCustomerIndex*',interval:auto,query:\n(language:lucene,query:'container_name: message-broker-task-\ncontainer AND level: ERROR AND file: \nSlackEventProcessor.java'),sort:!(!('@timestamp',desc)))&_g=\n(refreshInterval:(display:Off,section:0,value:0),time:(from:'2023-02-\n09T04:12:00.000Z',mode:absolute,to:'2023-02-\n09T04:22:00.000Z'))&discoverTab=logz-exceptions-\ntab&accountIds=429760&switchToAccountId=389045Already being tracked here COLL-1722:", "doc_id": "7ce57955-6637-4a9a-85e0-3b8d16306141", "embedding": null, "doc_hash": "ea26818540474a01a0f073875cf1ecb90cce2bf2d3197374a49cf7a5adaf243a", "extra_info": {"page_label": "10", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 2723, "_node_type": "1"}, "relationships": {"1": "597ca512-6f36-4807-b986-4a1d5acc33ca", "3": "ffdf9f90-fc3a-4461-a76f-81e03fe84ba5"}}, "__type__": "1"}, "ffdf9f90-fc3a-4461-a76f-81e03fe84ba5": {"__data__": {"text": "being tracked here COLL-1722: [ON-CALL]: [Prod] [Colla\nb] Failed brokering message to Slack (Deleting a Slack message )\nDONE \n08 Feb 2023 @Satya Vummidi [Logz.io: Cloud Observability & Security Powered by Open Source \n] Alert for : [Prod] [Collab] Failed brokering message to TeamsAfter investigating this log by querying MS Teams using internal \ntoken, @Harinandan Chintamreddy confirmed it is a welcome \nmessage and safe to ignore it. \nCreated tracker COLL-1845: Build a diagnostic endpoint to compa\nre messages brokered between PVT and VVT and force push a mes\nsage after bug fixesIN PROGRESS \n7 Feb 2023 @Prabhu Pant ALARM: stepfunction-alarms-production-\nproductionTEAMSCONNECTstepfunctionalarmshighexecutionfail\nurealarmA8548853-YWFDBLZBJCAM\nThis also came up during testing tenant 15423 in production\nhttps://app.logz.io/#/dashboard/osd/discover/?_a=(columns:!\n(message),filters:!(('$state':(store:appState),meta:\n(alias:!n,disabled:!f,index:'logzioCustomerIndex*',key:lambda_function\n_name,negate:!f,params:(query:apps-workflow-stack-apps--Raised during testing tenant 15423 in production Slack message f\nrom Anuradha Raman in #on-call \n1\n2\n3\n4\n5\n6\n7\n8Caused by: java.util.concurrent.ExecutionException: com\nat java.base/java.util.concurrent.CompletableFuture\nat java.base/java.util.concurrent.CompletableFuture\nat com.axm.collaboration.message.broker.teams.chann\nat com.axm.collaboration.message.broker.teams.event\nat com.axm.collaboration.message.broker.teams.event\nat java.base/java.util.Optional.ifPresentOrElse(Opt\nat com.axm.collaboration.message.broker.teams.event\n1\n2\n3\n4com.axm.collaboration.message.broker.exception.Mess\nat com.axm.collaboration.message.broker.slack.data.\nat java.base/java.util.Optional.orElseThrow(Optiona\nat com.axm.collaboration.message.broker.slack.data.\n", "doc_id": "ffdf9f90-fc3a-4461-a76f-81e03fe84ba5", "embedding": null, "doc_hash": "b4fd660279f17945d5caced3f06297ba5931d73d5f168cba77ea325e7b016211", "extra_info": {"page_label": "10", "file_name": "OnCall.pdf"}, "node_info": {"start": 2694, "end": 4489, "_node_type": "1"}, "relationships": {"1": "597ca512-6f36-4807-b986-4a1d5acc33ca", "2": "7ce57955-6637-4a9a-85e0-3b8d16306141"}}, "__type__": "1"}, "626bf733-83e0-4372-99f9-1dbf3c28ba09": {"__data__": {"text": "functionappslambdaworkfl-QFv6SE3wgNc3),type:phrase),query:\n(match_phrase:(lambda_function_name:apps-workflow-stack-apps--\nfunctionappslambdaworkfl-\nQFv6SE3wgNc3)))),index:'logzioCustomerIndex*',interval:auto,query:\n(language:lucene,query:''),sort:!())&_g=(filters:!(),refreshInterval:\n(pause:!t,value:0),time:(from:now-2h,to:now))&discoverTab=logz-logs-\ntab&accountIds=429760&switchToAccountId=429760\n7 Feb 2023 @Prabhu Pant [Prod] [Collab] Unable to install app in teams\nThis came up during testing tenant 15423 in production Raised during testing tenant 15423 in production Slack message f\nrom Anuradha Raman in #on-call \nFeb 5, 2023 @Harinandan Chintamreddy #COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACK\nIt looks like we do not have a handler setup to handle some event type \nbecause which the NPE occurred. Created a tracker\n \nSlack message from Logz Alerts in #critical-alerts-prod COLL-1843: [ON-CALL] NPE when handling a slack web hook ev\nentDONE \nFeb 2, 2023 ,Feb 3, \n2023 @Harinandan Chintamreddy #COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACK\nThere seems to be an issue when processing delete message events. \n@Sanket Jain is investigating this issue\n \nSlack message from Logz Alerts in #critical-alerts-prod \nSlack message from Logz Alerts in #critical-alerts-prod COLL-1598: Fix log messages to avoid alerting when a delete acti\non is attempted on non-brokered messageDUPLICATE \nCOLL-1722: [ON-CALL]: [Prod] [Collab] Failed brokering messag\ne to Slack (Deleting a Slack message )DONE \nJan 23, 2023 @Satya Vummidi P1 alert related to \u201cUnable to create team and channel in teams\u201d for \nthe tenant 12958 (connect_customer). I\u2019m assuming it is your internal \ntest.\nSlack message from Logz Alerts in #critical-alerts-prod \nhttps://pinstripe.app.opsgenie.com/alert/detail/d734c000-8cdb-4ea4-\na227-54da7db432e9-1674511759532/detailsRCA from @Anzar \nRCA: Graph API to retrieve the primary channel ( general \nchannel) from the newly created team failed with a 404. But \nthe channel is present in the team when checked manually. I \nguess we need to add retry logic to this API as this could be \nsome refresh issue.\nCOLL-1764: [On-call] RCA for Alert: Failing to create tea\nm and channel in TeamsDONE \nJan 19, 2023 @Nayan Srivastava Description : #COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-\nSLACK\nEvent Samples : Sample 1 event out of 1:\n[ {\n\"date\" : 1.674034475368389E9,\n\"exception\" : {\n\"exception_class\" : \n\"com.axm.collaboration.message.broker.exception.MessageBrokerEx\nception\",\n\"exception_message\" : \"MessageId=1674034468.750409 not found\",\n\"stacktrace\" : \n\"com.axm.collaboration.message.broker.exception.MessageBrokerEx\nception: MessageId=1674034468.750409 not found\\n\\tat \ncom.axm.collaboration.message.broker.slack.data.SlackDataManager.\nlambda$getMappedMessageId$5(SlackDataManager.java:252)\\n\\tat", "doc_id": "626bf733-83e0-4372-99f9-1dbf3c28ba09", "embedding": null, "doc_hash": "250979f309d921145598693fd5749628bea415943ab79ac333d4ae961514b527", "extra_info": {"page_label": "11", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 2802, "_node_type": "1"}, "relationships": {"1": "1ba7a349-4a28-4423-a013-05d8defe7065", "3": "d5075358-1c6e-4cea-9e52-089839227b0b"}}, "__type__": "1"}, "d5075358-1c6e-4cea-9e52-089839227b0b": {"__data__": {"text": "\njava.base/java.util.Optional.orElseThrow(Optional.java:408)\\n\\tat \ncom.axm.collaboration.message.broker.slack.data.SlackDataManager.\ngetMappedMe\n \nhttps://pinstripe.app.opsgenie.com/alert/detail/24d99cd5-1d68-4791-\nb424-a4b31e462e86-1674034714268/detailsCOLL-1722: [ON-CALL]: [Prod] [Collab] Failed brokering messag\ne to Slack (Deleting a Slack message )DONE \nJan 18, 2023 @Shivam Parashar (Unlicensed) [Logz.io] Alert for : [Prod] [Collab] Enable API Access errors\nEvent Samples : The following have met the condition:\n[ {\n\"message_obj.service\" : \"jira\",\n\"method\" : \"getOauth2TokenWithAuthorizationCode\",\n\"count\" : 1.0\n} ]\nLink to LogZThis was a part of testing on prod.\nAPI to get Oauth2 token for Jira service, but authorization failed.\nHappened for tenant Id: \n14,041\nAuthorization token not found\nInitiated a discussion here : Slack message from Shivam Parasha\nr in #on-call \nJan 18, 2023 @Shivam Parashar (Unlicensed) [Logz.io] Alert for : [Prod] [Collab] Enable API Access errors\nEvent Samples : The following have met the condition:\n[ {\n\"message_obj.service\" : \"jira\",\n\"method\" : \"getOauth2TokenWithAuthorizationCode\",\n\"count\" : 1.0\n} ]This was a part of testing on prod.\nThis is API to get Oauth2 token for Jira service, but authorization \nfailed.\nHappened for tenant Id: \n5,777\nAuthorization failed error=access_denied \n1\n2askContext=TaskContext(customerTenantId=15423, \nvendorTenantId=null, teamsOnboardingV2=true, genericTen\n", "doc_id": "d5075358-1c6e-4cea-9e52-089839227b0b", "embedding": null, "doc_hash": "75592305e804625729dfa9f91bc1405c7e4ec65b537c842e77bea06b52c6fc2c", "extra_info": {"page_label": "11", "file_name": "OnCall.pdf"}, "node_info": {"start": 2803, "end": 4241, "_node_type": "1"}, "relationships": {"1": "1ba7a349-4a28-4423-a013-05d8defe7065", "2": "626bf733-83e0-4372-99f9-1dbf3c28ba09"}}, "__type__": "1"}, "c50c0de2-ac18-4cd7-b256-ada4437b2bda": {"__data__": {"text": "Link to logZerrorDescription=User did not authorize the request \nerrorUri=null\nInitiated a discussion here : Slack message from Shivam Parasha\nr in #on-call \nJan 18, 2023 @Shivam Parashar (Unlicensed) [LightStep] Warning Threshold Violated: COLLAB-SLACK-\nINCOMING-REQUEST-RESPONSE-TIME\nProject Name : axiamatic-prod-1067bb71Took more than 3 seconds to respond to the slack event.\nThere is already an action item in collab to fix this.\nSlack message from Lightstep Alerts in #critical-alerts-prod \nJan 18, 2023 @Shivam Parashar (Unlicensed) Not a severe behaviour according to AWS. Instance will turn back \ngreen shortly. Issue already being tracked here DISC-1422: [On-c\nall] Prod ES yellow status alarm: Increase the number of data points \nor duration to consider before creating an alarmTO DO \nJan 16, 2023 @Satya Vummidi We got below alarm from AWS. \nAlarmName: stepfunction-alarms-production-\nproductionTEAMSCONNECTstepfunctionalarmshighexecutionfailu\nrealarmA8548853-YWFDBLZBJCAMThis alarm is very cryptic, requested in Slack to provide more details \nfor this alarm. \nAt the same time there is an on-boarding issue with Teams \npermissions for tenant ID 14979, probably both are related.\nJan 14 , 2023 @seshan Critical Threshold Violated: GLOBAL-HIKARI-IDLE-CONNECTIONS\nThere was a failover - that happened in aurora common db cluster - \ntriggering this issue - looks like some network issue in connecting to \nstorage . This resulted in multiple servicealerts - global hikari idle \nconnection went down\nEvents:\nLogs:\n Marking the issue as closed\nJan 13,2023 @seshan [Prod] [Collab] Enable API Access errors \nAuthorization failed error=access_denied errorDescription=The user \nhas denied access to the scope(s) requested by the client application. \nerrorUri=null - Based on previous on call log , @Paran you mentioned \nthat we need additional scope and i see that we have already \ndeployed the changes - but now its failed bcoz the requested scope is \ndenied - Below are the details of tenantMarking to as closed - based on discussion\n Slack message from Logz Alerts in #reg-alerts-prod \nJan 13, 2023 @seshan [Prod] [Collab] CRUD failed for Survey Instance : Unable to find \ncom.axm.collaboration.pulse.manager.entity.PulseSurvey with id\nhappening for tenant 14797\ntenant is already deleted via tenant deletion flow ( but there is \nexception for the same - no tenant type with webonly value found)\nas per discussion , tenant deletion is not implemented for PMS\nbut still we are not able to find pulse instance by id\n COLL-1733: ON-CALL: PulseSurveyInstancesApiService throws \n\"Error reading pulse survey instance by id\"  DONE \nCOLL-1734: ON-CALL: Message Broker tenant deletion throws e\nxception - No tenantType with value webonly found!!!DONE \n \n \nJan 12, 2023 @seshan COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACK - Unable \nto enrich Event=SlackEvent\nSlack message from Logz Alerts in #critical-alerts-prod COLL-1724 ON-CALL: NullPointerException while enrich Slack user \nname\nCOLL-1725 ON-CALL: Cleanup of stale entries in message-broker\n1\n2ALARM: \"es-plat-evtmgmt-production-Alarmsclusterstatusy\nUS West (Oregon)\n1\n2\n3\n4January 14, 2023, 12:14 (UTC+05:30)\nStarted cross AZ failover to DB instance: aslv2-common-\nJanuary 14,", "doc_id": "c50c0de2-ac18-4cd7-b256-ada4437b2bda", "embedding": null, "doc_hash": "7b73a33c11e41231fde88bd963b2c4f4e02766e0c9a3cf5261dec06934c7d16d", "extra_info": {"page_label": "12", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 3229, "_node_type": "1"}, "relationships": {"1": "c48683dc-f64e-4689-bcba-15ef3e12b199", "3": "01024302-ea9b-4c9e-a828-a1993457fb64"}}, "__type__": "1"}, "01024302-ea9b-4c9e-a828-a1993457fb64": {"__data__": {"text": "cross AZ failover to DB instance: aslv2-common-\nJanuary 14, 2023, 12:14 (UTC+05:30)\nCompleted failover to DB instance: aslv2-common-product\n1\n2\n3\n4\n5\n6\n72023-01-14T12:13:51.000+05:30\n2023-01-14 06:43:51 UTC::@:[29169]:LOG:  Storage initia\n2023-01-14T12:13:51.000+05:30\n2023-01-14T12:13:51.000+05:302023-01-14 06:43:51 UTC\n2023-01-14T12:13:51.000+05:302023-01-14 06:43:51 UTC\n1\n2\n3\n4\n5[ {\n \"message_obj.service\" : \"slack\",\n \"method\" : \"getOauth2TokenWithAuthorizationCode\",\n \"count\" : 1.0\n} ]\n112944,1,axiamatic_customer,axiamatic.com\n1\n2\n3\n4tenant_id,Count,name,domain\n13728,2,splunk,splunk.com\n7025,1,axm_onboarding,axm_onboarding\n14443,2,NA,NA", "doc_id": "01024302-ea9b-4c9e-a828-a1993457fb64", "embedding": null, "doc_hash": "1c2c983e8edfc01984ac6fc3739201892f191ec1bfd9893c7762166f24f7da78", "extra_info": {"page_label": "12", "file_name": "OnCall.pdf"}, "node_info": {"start": 3170, "end": 3815, "_node_type": "1"}, "relationships": {"1": "c48683dc-f64e-4689-bcba-15ef3e12b199", "2": "c50c0de2-ac18-4cd7-b256-ada4437b2bda"}}, "__type__": "1"}, "097a8045-83e3-4b75-a50f-f1d632d4b401": {"__data__": {"text": "Unable to enrich - is happening for  splunk and axm_onboarding \n(active tenants) - The rest of tenants are invalid /deleted tenants . \nThere are two question \n1) Is this error normal for active tenants\n2)For deleted tenants - how to remove data from message broker so \nthat it does not perform this action - @anil- i think 14xxx series are \ntenants that failed during cleanup will trigger cleanup of these tenants \ntoday\n \n \nJan 12, 2023 @seshan COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACK - Failed \nto deliver message to slack- MessageId not Found\nSlack message from Logz Alerts in #critical-alerts-prod COLL-1722: [ON-CALL]: [Prod] [Collab] Failed brokering messag\ne to Slack (Deleting a Slack message )DONE \nJan 11, 2023 @seshan [Prod] [Plat] ERROR logs > 100 in 15mins Message broker Container  \nmultiple errors \nSlack message from Logz Alerts in #critical-alerts-prod \nTop 10 contributors by tenant (both testing and  real customer)\nTop contributors - actual customer\nIdentitified  tests tenants - 3572 , 4882 - which pump in lots of error s \nand triggered tenant deletion for the same\nThere are two main buckets of errors after removing test tenants\n COLL-1689: [On-call][Investigate] Figure why the error counts are \ngoing up every 6 hoursDUPLICATE \nCOLL-1690: [On-Call] skip logging error if service is not supported \nfor brokeringDONE \nJan 11, 2023 @Ashwani Kumar #358: ALARM: \"stepfunction-alarms-production-\nStatemachinetenancytenantdeletion112E...\" in US West (Oregon)\nStepfunction failure alarm triggered.This was because of failure in discovery tenant deletion- HF was \nbeing tested.\nWorkflow started passing after the hotfix was completed.\nNo further action needed.\nJan 10, 2023 @Ashwani Kumar [Prod] [Plat] ERROR\nErrors in core-data, the rule decoding was failing DISC-2137: [On-call] Reporting Lambda : Rule parsing failing in c\nore because of incorrect rule definition DONE", "doc_id": "097a8045-83e3-4b75-a50f-f1d632d4b401", "embedding": null, "doc_hash": "7aba5260560ca7df5c005486b024256bd3f55ca1957aadc5720744d03ef88f46", "extra_info": {"page_label": "13", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 1892, "_node_type": "1"}, "relationships": {"1": "6ca3f6f6-9ccc-4ee4-9af0-15bb8ed19623", "3": "323c8c4a-7364-4ad8-82c9-ed3deb9eca73"}}, "__type__": "1"}, "323c8c4a-7364-4ad8-82c9-ed3deb9eca73": {"__data__": {"text": ": Rule parsing failing in c\nore because of incorrect rule definition DONE \n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15\n16\n17\n18\n19\n20\n21\n22\n23\n24\n25\n2614446,2,NA,NA\n14515,1,NA,NA\n14528,1,NA,NA\n14540,1,NA,NA\n14554,1,NA,NA\n14555,1,NA,NA\n14579,1,NA,NA\n14592,1,NA,NA\n14604,1,NA,NA\n14613,1,NA,NA\n14618,1,NA,NA\n14639,1,NA,NA\n14651,1,NA,NA\n14667,1,NA,NA\n14679,1,NA,NA\n14689,1,NA,NA\n14705,1,NA,NA\n14716,1,NA,NA\n14731,1,NA,NA\n14743,1,NA,NA\n14760,1,NA,NA\n14774,1,NA,NA\n1\n2\n3\n4\n5\n6\n7\n8\n9\n10tenant_idCountnamedomain\n35722070anu-prod-teamanu-prod-team\n13728169splunksplunk.com\n8677162electronic_artselectronic_arts\n1932143production_system_tests_teamsproduct\n4882108anu_prod_dndanu_prod_dnd\n1332098apps_sys_test_integrationvgn21.onmic\n437981armaan_collabcustomer1_aryanarmaan_\n1403768test_customer_app_1mprz.onmicrosoft.com\n1152855armisarmis\n1\n2\n3\n4\n5\n6\n7tenant_idCountnamedomain\n13728169splunksplunk.com\n8677162electronic_artselectronic_arts\n1295654crowdstrikecrowdstrike.com\n1299154medigateclaroty.com\n1108135axiamatic_ea_connectaxiamatic_ea_connec\n487527las_vegas_sands_corplas_vegas_sands_cor\n1\n2\n3\n4\n5\n6Bucket 1:\n8677,electronic_arts- \"Unable to get userid for user\" \nBucket 2: Almost all the other tenants has below catego\n12956,crowdstrike\n13728,splunk - \"unable to sync users for pvt class\",\"un", "doc_id": "323c8c4a-7364-4ad8-82c9-ed3deb9eca73", "embedding": null, "doc_hash": "35b3c7e033df386e946b443f55ed94de7a012f3b704dcddff4667103d4e47c2b", "extra_info": {"page_label": "13", "file_name": "OnCall.pdf"}, "node_info": {"start": 1819, "end": 3092, "_node_type": "1"}, "relationships": {"1": "6ca3f6f6-9ccc-4ee4-9af0-15bb8ed19623", "2": "097a8045-83e3-4b75-a50f-f1d632d4b401"}}, "__type__": "1"}, "076e6fd8-44dc-4bac-89f1-37a0041f2c11": {"__data__": {"text": "\"message_obj\": { \"msg\": \"Failed to parse given rule \n14037::customer::14037::productInstance::9408::readWrite . \nso skipping the rule\" },\n Invalid rule configuration being done with extra colon.\nJan 8, 2023 @Sanjiv Ranjan [Collab] CRUD failed for Survey Instance - \nCould not open JPA EntityManager for transaction; nested exception \nis org.hibernate.exception.JDBCConnectionException: Unable to \nacquire JDBC ConnectionCollab DB under maintenance \nJan 8, 2023 @Sanjiv Ranjan Critical Threshold Violated: GLOBAL-HIKARI-IDLE-\nCONNECTIONSCollab DB under maintenance \n \nJan 6, 2023 @Sanjiv Ranjan Critical Threshold Violated: GLOBAL-HIKARI-IDLE-CONNECTIONS\nLabels\napp:discoveryBeacuse primary and secondary node switch happened.\nJan 4, 2023 @Rama Taraniganty [Logz.io] Alert for : [Prod] [Plat] ERROR logs > 100 in 15mins\nMBLooks like every 6 hours there at 100 errors across various tenants. \nThis has been going on in Stage for the last couple of weeks.\n \nSlack message from Rama Taraniganty in #merge-requests \nCOLL-1689: [On-call][Investigate] Figure why the error counts are \ngoing up every 6 hoursDUPLICATE \nJan 4, 2023 @Paranthaman Karthikeyan [Logz.io] Alert for : [Prod] [Plat] ERROR logs > 100 in 15mins This again was due to missing scopes in slack bot. Fixed after \ndeploying \nhttps://gitlab.com/axiamatic-main/axiamatic-cfg/production-\nconfigs/-/merge_requests/13\nJan 4, 2023 @Paranthaman Karthikeyan [Logz.io] Alert for : [Prod] [Collab] Get Access Token errors This was due to missing scopes to slack bot that was needed for \nbidirectional flow.\n", "doc_id": "076e6fd8-44dc-4bac-89f1-37a0041f2c11", "embedding": null, "doc_hash": "8d026d684544b78161af0013116f2adc29a261dcfb565b682a1aef00ec0e2672", "extra_info": {"page_label": "14", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 1558, "_node_type": "1"}, "relationships": {"1": "2db441ba-005f-47fc-ab87-96ebe48bb427"}}, "__type__": "1"}, "6b452ef4-8877-4949-bb83-18edbabbfb58": {"__data__": {"text": "https://gitlab.com/axiamatic-main/axiamatic-cfg/production-\nconfigs/-/merge_requests/13\nJan 4, 2023 @Paranthaman Karthikeyan [Logz.io] Alert for : [Prod] [Collab] Failed brokering message to Slack\n COLL-1688: [ON_CALL]: message brokering: Reply in slack-slack \nis brokenDONE \nJan 4, 2023 @Paranthaman Karthikeyan [Logz.io] Alert for : [Prod][Plat] Service Restarting Repeatedly This was triggered during PROD deployment due to resource crunch. \nAction taken to update the resources https://gitlab.com/axiamatic-\nmain/axm-deployment/base-\ninfra/-/commit/e5f0a8ffac64af43504926bd10ec292f34fde438\nJan 4, 2023 @Sanjiv Ranjan [Prod][Plat] Service Restarting Repeatedly\n{\n\"container_name\" : \"core-data-task-container\",\n\"count\" : 3.0\n}1. Restart was triggered by scheduled deployment to prod(No Action \nneeded).\nDec 26, 2022 @Syed Ummar Farooqh COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACK This issues is fixed for 0.9.6. (Error Processing message edit \nevent)\nDec 24, 2022\nJan 26, 2023@Satya Vummidi While watching Avatar: The way of waters in screenX theater, I got this \nalert. As soon as I came back, I investigated the logs, it looks like \nthere is an outage from SendGrid. Our health check failing and keep \nrestarting the service.\nSendgrid API I think we don\u2019t want to add external service health part of our service \nhealth check. What is the point in keep restarting based on this \nhealth check? If it is a recoverable situation, we need to log error, \ntrigger alert. Keep pending email delivery in queue and recover as \nsoon as the external service is up.\nPLAT-1521: [On-call] Don't use send grid health as health indicato\nr for notification service.DONE \nDec 22, 2022 @Syed Ummar Farooqh COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACK This issue is fixed\nCOLL-1615: [on-call][prod] Error processing edit event from slack\nDUPLICATE \nDec 20, 2022 @Syed Ummar Farooqh 1. Warning Threshold Violated: GLOBAL-HIKARI-IDLE-\nCONNECTIONS\n2. Resolved: GLOBAL-HIKARI-IDLE-CONNECTIONSObserved a temporary spike for idle connections in tenancy-service, \nclosing the alert now. Created a tracker for this\nPLAT-1507: [on-call] [Prod] [tenancy-service] investigate - GLOBA\nL-HIKARI-IDLE-CONNECTIONSDONE \nDec 19, 2022 @Anzar [Prod] [Collab] Get Access Token errors The above error is from the tenant 13572, which is a cleaned up \nSplunk tenant. The event was triggered on opening the slack home \ntab and failed with token not found.\nThe tenant was deleted on 22nd Nov, Sentiment collector data \ncleanup code was merged post that date hence stale data has \ncaused this error. \nDec 19, 2022 @Anil Ganivada COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACK alerts triggered on test tenants, closing the alert for nowCOLL-159\n8: Fix log messages to avoid alerting when a delete action is attempt\ned on non-brokered messageDUPLICATE COLL-1561: On Call : G\netting [Collab] Failed brokering message to Slack alert in Prod\nDUPLICATE \nshould fix the issue.\nDec 16, 2022 @Anil Ganivada", "doc_id": "6b452ef4-8877-4949-bb83-18edbabbfb58", "embedding": null, "doc_hash": "c9214a49365cb50c3b7b3a4bcf53baa21cf17048f981d7a5617b54e6e1d35fc4", "extra_info": {"page_label": "15", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 2954, "_node_type": "1"}, "relationships": {"1": "468db9da-7f2c-458a-8e4b-a209e0ccb1ad", "3": "7bd9db26-5333-49d3-acfe-b320ad60a235"}}, "__type__": "1"}, "7bd9db26-5333-49d3-acfe-b320ad60a235": {"__data__": {"text": "\nshould fix the issue.\nDec 16, 2022 @Anil Ganivada COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACK Alert triggerred on test tenants, fix is in 0.9.6\nhttps://app.logz.io/#/goto/c1fe3d22d61574e9c48fd62f66dc33\nd9?switchToAccountId=389045\n Slack message from Anzar MK in #critical-alerts-prod \nDec 16, 2022 @Anil Ganivada [Prod][Plat] Service Restarting Repeatedly Jira service restarting post deployment due to resourcing issue  \nPLAT-1501: Update all apps and jira service size to medium insta\nnceDONE\nDec 15, 2022 @Rama Taraniganty Support issue: Tenant delete + recreate has a few issues that need to \nbe addressedPLAT-1494: [On-call] Tenant creation/deletion issues - need to rev\nisit transactional supportDONE \nPLAT-1495: [On-call] [Investigate] publisher mapping didn't get re\nmoved when a tenant is deleted in core-dataDONE \nSlack message from Rama Taraniganty in #on-call \n1\n2\n3\n4\n5\n6\n7Description : #COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-S\nEvent Samples : Sample 1 event out of 1:\n[ {\n\"date\" : 1.672829539424851E9,\n\"exception\" : {\n\"exception_class\" : \"java.lang.NullPointerException\",\n\"stacktrace\" : \"java.lang.NullPointerException\\n\\tat co\n", "doc_id": "7bd9db26-5333-49d3-acfe-b320ad60a235", "embedding": null, "doc_hash": "41a06d83a069f023f4bb9e5c9b5bcce6010e31811dc5604185831759e9fc6c6e", "extra_info": {"page_label": "15", "file_name": "OnCall.pdf"}, "node_info": {"start": 2904, "end": 4049, "_node_type": "1"}, "relationships": {"1": "468db9da-7f2c-458a-8e4b-a209e0ccb1ad", "2": "6b452ef4-8877-4949-bb83-18edbabbfb58"}}, "__type__": "1"}, "279ac7d5-3dfe-4ac0-90e5-1e530831058a": {"__data__": {"text": "Dec 15, 2022 @Anil Ganivada COLLAB-SLACK-INCOMING-REQUEST-RESPONSE-TIME 2 alerts triggered on filtered events, root cause still to be analyzed on \nintermittent delay in response from backend for slack events beyond \n3 secs\nhttps://app.logz.io/#/goto/cfa2a2de9a6dc1acfe95aa6fcd8e074c?\nswitchToAccountId=389045\nhttps://app.logz.io/#/goto/cfa2a2de9a6dc1acfe95aa6fcd8e074c?\nswitchToAccountId=389045\nDec 15, 2022 @Anil Ganivada [Prod] [Collab] Failed brokering message to Slack alert triggerred on a ST tenant, attempt to delete a non-brokered \nmessage is harmless Slack message from Anzar MK in #critical-al\nerts-prod \nDec 14, 2022 @Rama Taraniganty [Logz.io] Alert for : [Prod][Collab] Error responses sent to slack \nor teams\nLooks like the payload is malformed:  =%7B%7D=https://app.logz.io/#/goto/a7166f7ad99ed1cd417d143be02854\nbb?switchToAccountId=389045\nCOLL-1589: [On-call] Handle junk message body in SlackSlashC\nommandWebhook::taskSlashCommandDONE \n12 Dec 2022 @ritu vaidya COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACK Slack message from Logz Alerts in #critical-alerts-prod \nClosing it based on the slack thread.\nDec 9, 2022 @Rama Taraniganty COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-TEAMS There is no run book entry to make sense of this. Looks like there\u2019s \nno reference to the error in this page either:\nSlack message from Rama Taraniganty in #on-call \n9 Dec 2022 @Pradeep @Anil Ganivada [Prod] [Tenancy] Tenant Registration Failed Expected error due to tests by @Anuradha Raman  on prod\nSlack message from Logz Alerts in #critical-alerts-prod \nThis alert triggerred because we were expecting test_vendor tenant \nto have been remove from our db but it was present because in \nprocess of re-creating customer scenario in prod, tenant id 12944 \nwas deleted instead of 12954 . 12944 is not a customer account but \nan internal account that we use to send invitations to vendors for \nonboarding. Luckily no data got removed from core data but the org \nwas deleted from auth0 so I used these steps to recover the instance \nin auth0 How to recover a deleted org in auth0 . After recovering \nthe instance I used anair@axiamatic.com user to link to this org and \nassigned him role of orgadmin. For step-15:\nUpdate org id in tenants table in tenancy schema for the \nrespective tenant id\nI executed following statements on prod tenancy table\npostgresdbproduction=> select * from tenant where id = \n12944;\n  id   |             name             |    domain     |        \ntime_created        |        time_updated        | is_deleted | \nmode | props | status |\n      org_id\n-------+------------------------------+---------------+-----------------------\n-----+----------------------------+------------+------+-------+--------+--\n--------------------\n 12944 | axiamatic_customer (removed) | axiamatic.com | \n2022-11-08 09:43:23.845+00 | 2022-12-09 09:44:00.627+00 \n| t          |      |     ", "doc_id": "279ac7d5-3dfe-4ac0-90e5-1e530831058a", "embedding": null, "doc_hash": "7f3c775f3b4f4c6752d1971216828989007def89a9878a50ce3d002c3ef9e9a6", "extra_info": {"page_label": "16", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 2881, "_node_type": "1"}, "relationships": {"1": "00e68c43-8f2c-4ed5-b5ec-e0c71b8f3819", "3": "ffbb83c4-370e-4692-826e-a86bbef48dc1"}}, "__type__": "1"}, "ffbb83c4-370e-4692-826e-a86bbef48dc1": {"__data__": {"text": "        |      |       |      0 | o\nrg_hbnLyrra973ZNT4x\n(1 row)postgresdbproduction=> update tenant set org_id = \n'org_KllDkpdeJhnOCWfz' where id = 12944 and domain like \n'axiamatic.com';\nUPDATE 1\npostgresdbproduction=> select * from tenant where id = \n12944;\n  id   |             name             |    domain     |        \ntime_created        |        time_updated        | is_deleted | \nmode | props | status |\n      org_id\n-------+------------------------------+---------------+-----------------------\n-----+----------------------------+------------+------+-------+--------+--\n--------------------\n 12944 | axiamatic_customer (removed) | axiamatic.com | \n2022-11-08 09:43:23.845+00 | 2022-12-09 09:44:00.627+00 \n| t          |      |       |      0 | o\nrg_KllDkpdeJhnOCWfz\n(1 row)postgresdbproduction=> update tenant set name = \n'axiamatic_customer' where id = 12944 and domain like \n", "doc_id": "ffbb83c4-370e-4692-826e-a86bbef48dc1", "embedding": null, "doc_hash": "44ed1835b0e365ac7b4163d53ac0a53217d44cf34507b1cf0dc0a842371e9d14", "extra_info": {"page_label": "16", "file_name": "OnCall.pdf"}, "node_info": {"start": 2860, "end": 3747, "_node_type": "1"}, "relationships": {"1": "00e68c43-8f2c-4ed5-b5ec-e0c71b8f3819", "2": "279ac7d5-3dfe-4ac0-90e5-1e530831058a"}}, "__type__": "1"}, "57762e71-1462-4735-8d07-d94672b8c357": {"__data__": {"text": "1.  \n 'axiamatic.com';\nUPDATE 1\npostgresdbproduction=> exit\nafter this I tried to login as anair@axiamatic.com -> \naxiamatic_customer and was able to see all previous integrations on \nthis tenant including splunk and tried to access this customer data \nfrom a vendor account and I see that things are working as expected.\n9 Dec 2022 @Pradeep api-gw-ids-production-Alarmshighfailureratealarm @Anuradha Raman  was verifying in production and while \nonboarding there were lot of failures on user-recommendations even \nwhen MB is not enabled. This caused lot of 500 error responses.\nAlready a Tracker exists.- DISC-1921: User suggestions should n\not be called when none of the MB is not enabled DONE \n8 Dec 2022 @Pradeep [Prod] [Collab] CRUD failed for Survey Instance Discussed in slack thread Slack message from Logz Alerts in #crit\nical-alerts-prod \nIt is due to Reetol poc that we did. \n8 Dec 2022 @ritu vaidya COLLAB-SLACK-INCOMING-REQUEST-RESPONSE-TIME Known issue , see 30 Nov log\n7 Dec 2022 @Pradeep COLLAB-SLACK-INCOMING-REQUEST-RESPONSE-TIME Known issue\n7 Dec 2022 @Pradeep [Prod] [Collab] Failed brokering message to Slack\n#COLLAB-MB-FAILED-DELIVERING-MESSAGE-TO-SLACKAlready fixed in 0.9.6 \nCOLL-1561: On Call : Getting [Collab] Failed brokering messa\nge to Slack alert in ProdDUPLICATE \nCOLL-1551: message-broker: editing of message should not s\ntoremappedmessagesDONE\n", "doc_id": "57762e71-1462-4735-8d07-d94672b8c357", "embedding": null, "doc_hash": "9c016adfebd20b478a08aa2f5f267dc63ee4bb47a6583cb8e8677832abc3b1e8", "extra_info": {"page_label": "17", "file_name": "OnCall.pdf"}, "node_info": {"start": 0, "end": 1378, "_node_type": "1"}, "relationships": {"1": "61eb3840-5d98-4260-8e52-318ba943edcf"}}, "__type__": "1"}}, "docstore/ref_doc_info": {"d6496709-cfc5-4485-93e6-2c6905c9d395": {"doc_ids": ["03a7050b-f0d8-4a33-b9e8-d4f82439cca0", "33630944-a8b9-4d7e-8d3f-0381d2c4e586"], "extra_info": {"page_label": "1", "file_name": "OnCall.pdf"}}, "7bc79ea6-68b3-4bf5-8112-31e1aa58d448": {"doc_ids": ["69adae88-db95-4f48-81df-ac19ca8077e3", "21d0b6b7-28cf-437e-aef6-522813d4e0a1"], "extra_info": {"page_label": "2", "file_name": "OnCall.pdf"}}, "57018674-bf81-4ad7-940b-47bc8f22a82b": {"doc_ids": ["792e2728-bdc4-4e54-ad72-55e359056616", "6f1adeda-e4d0-4b96-b2f1-57def8904cb6"], "extra_info": {"page_label": "3", "file_name": "OnCall.pdf"}}, "04678987-bdb0-4922-a081-c6066b190cff": {"doc_ids": ["07aa42dc-7b17-4b9f-848a-181c4ec7e77f", "d14ad41a-0e46-4fb9-b979-8973ba836de4"], "extra_info": {"page_label": "4", "file_name": "OnCall.pdf"}}, "130a58d8-6265-4783-b0a6-056344dacdb8": {"doc_ids": ["e5834cdd-0e9d-484d-a7e8-721644c8e119", "4cc6e453-d8cb-4316-9f00-1ad8fc6152ad"], "extra_info": {"page_label": "5", "file_name": "OnCall.pdf"}}, "bd4335e6-72c5-40f0-ac46-bfd30bc3b26f": {"doc_ids": ["975881aa-a45e-4b2c-83a8-bd6932e4a819"], "extra_info": {"page_label": "6", "file_name": "OnCall.pdf"}}, "0bd2d94e-126a-4644-8989-aa46455d3788": {"doc_ids": ["26d56212-a665-414f-803a-387a807bffa0", "ca972543-cc22-4e27-a596-6492703c69ce"], "extra_info": {"page_label": "7", "file_name": "OnCall.pdf"}}, "d09c18d3-ac4b-4719-a3f1-9ca73015e9c2": {"doc_ids": ["02048f03-be3c-4ebf-9383-5e85fd3356e1", "9cf96cfc-0673-4806-b906-e4dab4f6aa0e"], "extra_info": {"page_label": "8", "file_name": "OnCall.pdf"}}, "ce7f46b5-953b-4cb8-b961-183b46bc7645": {"doc_ids": ["9b16e0e5-8641-4e16-a4f7-67130cdd3a59", "7f0dd2c8-c705-447e-b1f0-55556cf61ad4"], "extra_info": {"page_label": "9", "file_name": "OnCall.pdf"}}, "597ca512-6f36-4807-b986-4a1d5acc33ca": {"doc_ids": ["7ce57955-6637-4a9a-85e0-3b8d16306141", "ffdf9f90-fc3a-4461-a76f-81e03fe84ba5"], "extra_info": {"page_label": "10", "file_name": "OnCall.pdf"}}, "1ba7a349-4a28-4423-a013-05d8defe7065": {"doc_ids": ["626bf733-83e0-4372-99f9-1dbf3c28ba09", "d5075358-1c6e-4cea-9e52-089839227b0b"], "extra_info": {"page_label": "11", "file_name": "OnCall.pdf"}}, "c48683dc-f64e-4689-bcba-15ef3e12b199": {"doc_ids": ["c50c0de2-ac18-4cd7-b256-ada4437b2bda", "01024302-ea9b-4c9e-a828-a1993457fb64"], "extra_info": {"page_label": "12", "file_name": "OnCall.pdf"}}, "6ca3f6f6-9ccc-4ee4-9af0-15bb8ed19623": {"doc_ids": ["097a8045-83e3-4b75-a50f-f1d632d4b401", "323c8c4a-7364-4ad8-82c9-ed3deb9eca73"], "extra_info": {"page_label": "13", "file_name": "OnCall.pdf"}}, "2db441ba-005f-47fc-ab87-96ebe48bb427": {"doc_ids": ["076e6fd8-44dc-4bac-89f1-37a0041f2c11"], "extra_info": {"page_label": "14", "file_name": "OnCall.pdf"}}, "468db9da-7f2c-458a-8e4b-a209e0ccb1ad": {"doc_ids": ["6b452ef4-8877-4949-bb83-18edbabbfb58", "7bd9db26-5333-49d3-acfe-b320ad60a235"], "extra_info": {"page_label": "15", "file_name": "OnCall.pdf"}}, "00e68c43-8f2c-4ed5-b5ec-e0c71b8f3819": {"doc_ids": ["279ac7d5-3dfe-4ac0-90e5-1e530831058a", "ffbb83c4-370e-4692-826e-a86bbef48dc1"], "extra_info": {"page_label": "16", "file_name": "OnCall.pdf"}}, "61eb3840-5d98-4260-8e52-318ba943edcf": {"doc_ids": ["57762e71-1462-4735-8d07-d94672b8c357"], "extra_info": {"page_label": "17", "file_name": "OnCall.pdf"}}}}