Package: detech-ai-db
Database access package for detech.aiThis is detech.ai's package to access Dynamodb & Timestream programatically.Importsimportdetech_query_pkg###############    DynamoDB Package    ##############################fromdetech_query_pkg.dynamodb_pkgimportdynamodb_queriesasdb_queriesfromdetech_query_pkg.dynamodb_pkg.utilsimportdynamodb_utilsasdb_utils#Start DynamoDB Clientdb_utils.create_dynamodb_client(aws_access_key_id=AWS_ACCESS_KEY_ID,aws_secret_access_key=AWS_SECRET_ACCESS_KEY,region_name=REGION_NAME)###############    Timestream Package    ##############################fromdetech_query_pkg.timestream_pkgimportts_queriesfromdetech_query_pkg.timestream_pkg.utilsimportts_utilsfromdetech_query_pkg.timestream_pkg.modelsimportmetrics_modelfromdetech_query_pkg.timestream_pkg.modelsimportmetrics_creator_utilsInitialize Clientdefcreate_dynamodb_client(aws_access_key_id,aws_secret_access_key,region_name)defcreate_timestream_write_client(aws_access_key_id,aws_secret_access_key)defcreate_timestream_query_client(aws_access_key_id,aws_secret_access_key)Functionstimestream_pkg (ts_queries.py)insert_metrics_from_metric_listdefinsert_metrics_from_metric_list(client,metric_list)# Inserts metrics in batch to timestream#metric_list must have the following fieldsmetric_list=[{'org_id','region_name','namespace','component_id','period','agent','metric_alignment','unit','description','metric_id','metric_name','value','timestamp'},{'org_id','region_name','namespace','component_id','period','agent','metric_alignment','unit','description','metric_id','metric_name','value','timestamp'},...]query_metricsdefquery_metrics(client,sql_query)# Performs an SQL query to timestream and transforms the output to a more desirable format#Outputquery_response={'metric_id':'qgrdy1bXGeKSmAtW58CD','agent':'AWS.CloudWatch','component_id':'AWS/ApplicationELB.app/component','period':'60','unit':'None','org_id':'Organization','metric_alignment':'Sum','namespace':'AWS/ApplicationELB','description':'The total number of concurrent TCP connections active from clients to the load balancer and from the load balancer to targets.','region_name':'eu-west-1','value':'64.0','metric_name':'ActiveConnectionCount','timestamp':'2020-10-12 14:28:00.000000000'}timestream_pkg.utils (ts_utils.py)prepare_metric_recordsdefprepare_metric_records(measure_name,measure_value,timestamp,dimensions)#The dimensions that need to be passed must be in the following formatdimensions=[{'Name':'org_id','Value':str(metric['org_id'])},{'Name':'region_name','Value':str(metric['region_name'])},{'Name':'namespace','Value':str(metric['namespace'])},{'Name':'component_id','Value':str(metric['component_id'])},{'Name':'period','Value':str(metric['period'])},{'Name':'agent','Value':str(metric['agent'])},{'Name':'metric_alignment','Value':str(metric['metric_alignment'])},{'Name':'unit','Value':str(metric['unit'])},{'Name':'description','Value':str(metric['description'])},{'Name':'metric_id','Value':str(metric['metric_id'])}]write_to_timestreamdefwrite_to_timestream(client,records,database_name,table_name)# Inserts metrics to timestream after they are in the correct formatquery_from_timestreamdefquery_from_timestream(client,sql_query)# Queries metrics from timestream with a given sql_querytimestream_pkg.models (metric_creator_utils.py & metrics_model.py)build_metric_model#from metric_creator_utils.pydefbuild_metric_model(metric_id,metric_name,org_id,component_id,namespace,metric_alignment,agent,dimensions,region_name=None,is_default=False,description=None,period=60,unit=None,samples=[])#Queries metrics from timestream with a given sql_queryMetricModel#from metrics_model.pyclassMetricModel(object):def__init__(self,id,name,org_id,component_id,namespace,alignment,region_name,data_center_id,agent,dimension_name=None,dimension_value=None,is_active=False,description=None,unit=None)defto_dict(self)#Queries metrics from timestream with a given sql_querydynamodb_pkginsert_alertdefinsert_alert(alert_id,metric_id,org_id,app_id,team_id,assigned_to,start_time,end_time,alert_description,is_acknowledged,anomalies_dict,related_prev_anomalies,service_graph,significance_score,dynamodb)#Exampleinsert_alert(alert_id="256828",metric_id=123,org_id='org_id',app_id='app_id',team_id='team_id',assigned_to='Jorge',\start_time='2020-09-03 12:00:00',end_time='2020-09-03 12:20:00',alert_description='Spike in costs',\is_acknowledged='True',anomalies_dict={},related_prev_anomalies={},service_graph={},significance_score='34.3')get_alert_item_by_keydefget_alert_item_by_key(anom_id,dynamodb)update_alert_with_related_anomaliesdefupdate_alert_with_related_anomalies(alert_id,start_time,corr_anoms_dict,related_prev_anomalies,dynamodb)terminate_alertdefterminate_alert(alert_id,start_time,end_timestamp,dynamodb)create_metricdefcreate_metric(metric_id,date_bucket,metric_name,provider,namespace,agent,org_id,app_id,alignment,groupby,dimensions,data_points_list,dynamodb)#Examplecreate_metric(metric_id="test1",date_bucket="2020-10-02",metric_name="error_rate",provider="aws",namespace="dynamodb",agent="CloudWatch",org_id="test",app_id="app1",alignment="Sum",dimensions=[{"Name":"TableName","Value":"alerts.config"}],last=1535530432,data_points_list=[{'val':55,'time':1535530430},{'val':56,'time':1535530432}],dynamodb=dynamodb)batch_insert_metric_objectsdefbatch_insert_metric_details_objects(list_of_metric_objects,dynamodb)#Inserts list of metrics objects in batch into Dynamodbbatch_insert_metric_objectsdefbatch_insert_metric_details_objects(list_of_metric_objects,dynamodb)#Inserts list of metrics objects in batch into Dynamodbbatch_insert_metric_objectsdefbatch_insert_component_info_objects(list_of_component_objects,dynamodb)#Inserts list of component objects in batch into Dynamodbget_metric_detailsdefget_metric_details(metric_id,dynamodb)#Fetches all the details for a specific metric_idget_metric_item_by_keydefget_metric_item_by_key(metric_id,curr_date,dynamodb)scan_metrics_by_encrypted_iddefscan_metrics_by_encrypted_id(anom_alarm_id,dynamodb)query_alerts_configs_by_keydefquery_alerts_configs_by_key(metric_id,dynamodb)insert_alert_configdefinsert_alert_config(metric_id,alert_title,severity,alert_type,alert_direction,description,duration,duration_unit,rule_dict,recipients_list,owner_dict,dynamodb)#Exampleinsert_alert_config(metric_id="metric1245",alert_title="Anomaly by Cluster",severity="critical",alert_type="anomaly",alert_direction="spikes/drops",description="Relevant to Play Store billing user journey",duration=12,duration_unit="hours",rule_dict={},recipients_list=[{"channel":"webhook","contact":"j.velez2210@gmail.com"},{"channel":"slack","contact":"j.velez2210@gmail.com"}],owner_dict={"user_id":"user12341","user_name":"João Tótó",})query_most_recent_metric_fetching_logdefquery_most_recent_metric_fetching_log(component_id,dynamodb)#Fetches the log with the highest timestamp, from all the logs between start & end tsinsert_api_request_logdefinsert_api_request_log(api_name,request_timestamp,response_status_code,request,response,dynamodb)# Exampleinsert_api_request_log(api_name='anomalarm_metrics',request_timestamp=1603466177,response_status_code='202',request={'key':'value'},response={'key':'value'},dynamodb=dynamodb)insert_new_anomalydefinsert_new_anomaly(id,timestamp,metric_id,value,dynamodb,is_dev_env=False):# Exampleinsert_new_anomaly(id="125123",timestamp=1599563224,metric_id="m412",value=123.44,dynamodb=dynamodb)update_anomaly_relationsdefupdate_anomaly_relations(id,timestamp,cross_correlations,possible_related_anomalies,possible_related_matches,dynamodb,is_dev_env=False):# Exampleupdate_anomaly_relations(id="125123",timestamp=1599563224,cross_correlations={"web-server-1.cpu0.iowait":{"coefficient":0.95752,"shifted":0,"shifted_coefficient":0.95752},},possible_related_anomalies={"256826":{"metric_id":"web-server-1.mysql.counters.handlerRead_key","timestamp":1599563164},},possible_related_matches={"169560":{"timestamp":1599563230,"fp id":8821,"layer id":"None","metric_id":"web-server-2.mariadb.localhost:3306.mysql.bytes_sent"}},dynamodb=dynamodb)insert_anomalarm_anomalies_webhook_logdefinsert_anomalarm_anomalies_webhook_log(anomalarm_id,timestamp,request,response,dynamodb,is_dev_env=False):# Exampleinsert_anomalarm_anomalies_webhook_log(anomalarm_id='256828',timestamp=1603466177,request={...},response={...},dynamodb=dynamodb)terminate_anomalydefterminate_anomaly(id,timestamp,end_timestamp,dynamodb,is_dev_env=False):# Exampleterminate_anomaly(id="125123",timestamp=1599563224,end_timestamp=1599663224,dynamodb=dynamodb)insert_error_logdefinsert_error_log(dynamodb,service_name,timestamp,msg,details,is_dev_env=False):# Exampleinsert_error_log(dynamodb=dynamodb,service_name="metric_to_db",timestamp=1599563224,msg="Error inserting value",details={'exception':'RejectedRecordsException','response':{...}})dynamodb_pkg.utilsput_itemdefput_item(item_dict,table_name,dynamodb)#Inserts json item into DynamoDB table#Exampleitem_dict={"attr":"value","attr2":"value2"}table_name="alerts"batch_insertdefbatch_insert(list_of_item_dicts,table_name,dynamodb)#Inserts a list of item_dicts in batch to dynamodbget_itemdefget_item(key_dict,table_name,dynamodb)#Retrieves item from DynamoDB table#Examplekey_dict={"prim_key"="value","sort_key"="value"}get_item_and_retrieve_specific_attributesdefget_item_and_retrieve_specific_attributes(key_dict,attr_list,table_name,dynamodb)#Retrieves item from DynamoDB table and retrieve specific attributes#Examplekey_dict={"prim_key":"value","sort_key":"value"}attr_list=['attr1','attr2']update_itemdefupdate_item(key_dict,update_expression,expression_attr_values,table_name,dynamodb)#Retrieves item from DynamoDB table#Examplekey_dict={"prim_key"="value","sort_key"="value"}update_expression="set service_graph=:i, metric_list=:l, significance_score=:s"expression_attr_values={':i':{'s1':['s2','s3']},':l':['124','123'],':s':Decimal(35.5)}#example to append to listUpdateExpression="SET some_attr = list_append(if_not_exists(some_attr, :empty_list), :i)",ExpressionAttributeValues={':i':[some_value],"empty_list":[]}update_item_conditionallydefupdate_item_conditionally(key_dict,condition_expression,update_expression,expression_attr_values,table_name,dynamodb)#Retrieves item from DynamoDB table#Examplekey_dict={"prim_key"="value","sort_key"="value"}update_expression="set service_graph=:i, metric_list=:l, significance_score=:s"expression_attr_values={':i':{'s1':['s2','s3']},':l':['124','123'],':s':Decimal(35.5)}condition_expression="significance_score <= :val"delete_item_conditionallydefdelete_item_conditionally(key_dict,condition_expression,expression_attr_values,table_name,dynamodb)#Examplecondition_expression="significance_score <= :val"expression_attr_values={":val":Decimal(50)}key_dict={'org_id':'Aptoide','start_time':'2020-09-03 12:00:00'}'''query_by_keydefquery_by_key(key_condition,table_name,dynamodb)#Queries from DynamoDB table by key condition#Examplekey_condition=Key('org_id').eq('Aptoide')query_and_project_by_key_conditiondefquery_and_project_by_key_condition(projection_expr,expr_attr_names,key_condition,table_name,dynamodb)#Queries from DynamoDB table by key condition and only returns some attrs#Examplekey_condition=Key('year').eq(year)&Key('title').between(title_range[0],title_range[1])projection_expr="#yr, title, info.genres, info.actors[0]"expr_attr_names={"#yr":"year"}scan_tabledefscan_table(scan_kwargs,table_name,dynamodb)#Scans entire table looking for items that match the filter expression#Examplescan_kwargs={'FilterExpression':Key('year').between(*year_range),'ProjectionExpression':"#yr, title, info.rating",'ExpressionAttributeNames':{"#yr":"year"}}query_by_key_min_maxdefquery_by_key_min_max(key_condition,table_name,is_min,dynamodb)#Queries from DynamoDB table by key condition#Examplekey_condition=Key('part_id').eq(partId)&Key('range_key').between(start,end)#orkey_condition=Key('part_id').eq(partId)get_all_items_in_tabledefget_all_items_in_table(table_name,dynamodb)increment_atomic_counterdefincrement_atomic_counter(counter_type,number_of_values,dynamodb)#Increments a counter and makes sure it is done atomically#Available counter types:#org_id#component_id#metric_id