cloudwatch,slack,nginxログを送る

# nginx to cloudwatchの設定
## AWS CloudWatch Logsエージェントのインストール
https://remotestance.com/blog/2433/

# cloudwatch to slack
blueprint使うと早い
slackのwebhookとchannelで十分
https://dev.classmethod.jp/cloud/aws/lambda-blueprint-slack-cloudwatch/
https://qiita.com/yahagin/items/2248287c5285cd1e9201


https://qiita.com/siro33950/items/71f52c5d1ef8b81fb7a0

https://qiita.com/yahagin/items/2248287c5285cd1e9201



https://qiita.com/onooooo/items/f59c69e30dc5b477f9fd

# ログ内容を出力するスクリプトhttp://htnosm.hatenablog.com/entry/2016/05/05/090000

# SNSログの形
{
    "AlarmName": "sample-error",
    "AlarmDescription": "sampleでエラーが発生しました。",
    "AWSAccountId": "xxxxxxxxxxxx",
    "NewStateValue": "ALARM",
    "NewStateReason": "Threshold Crossed: 1 datapoint [2.0 (29/11/17 01:09:00)] was greater than or equal to the threshold (1.0).",
    "StateChangeTime": "2017-11-29T01:10:32.907+0000",
    "Region": "Asia Pacific (Tokyo)",
    "OldStateValue": "OK",
    "Trigger": {
        "MetricName": "sample-metric",
        "Namespace": "LogMetrics",
        "StatisticType": "Statistic",
        "Statistic": "SUM",
        "Unit": null,
        "Dimensions": [],
        "Period": 60,
        "EvaluationPeriods": 1,
        "ComparisonOperator": "GreaterThanOrEqualToThreshold",
        "Threshold": 1,
        "TreatMissingData": "- TreatMissingData:                    NonBreaching",
        "EvaluateLowSampleCountPercentile": ""
    }
}

```

import boto3
import json
import logging
import os
import datetime
import calendar

from base64 import b64decode
from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError


# The base-64 encoded, encrypted key (CiphertextBlob) stored in the kmsEncryptedHookUrl environment variable
ENCRYPTED_HOOK_URL = os.environ['kmsEncryptedHookUrl']
# The Slack channel to send a message to stored in the slackChannel environment variable
SLACK_CHANNEL = os.environ['slackChannel']

HOOK_URL = "https://" + boto3.client('kms').decrypt(CiphertextBlob=b64decode(ENCRYPTED_HOOK_URL))['Plaintext'].decode('utf-8')

logger = logging.getLogger()
logger.setLevel(logging.INFO)

FILTER_PATTERN='{ $.log_level = "ALERT" }'
OUTPUT_LIMIT=5
TIME_FROM_MIN=5

def lambda_handler(event, context):
    logger.info("Event: " + str(event))
    message = json.loads(event['Records'][0]['Sns']['Message'])
    logger.info("Message: " + str(message))

    alarm_name = message['AlarmName']
    #old_state = message['OldStateValue']
    new_state = message['NewStateValue']
    reason = message['NewStateReason']
    metric = message['Trigger']['MetricName']
    metricNamespace = message['Trigger']['Namespace']

    timeto = datetime.datetime.strptime(message['StateChangeTime'][:19] ,'%Y-%m-%dT%H:%M:%S') + datetime.timedelta(minutes=1)
    u_to = calendar.timegm(timeto.utctimetuple()) * 1000
    timefrom = timeto - datetime.timedelta(minutes=TIME_FROM_MIN)
    u_from = calendar.timegm(timefrom.utctimetuple()) * 1000
    logs = boto3.client('logs')
    metricfilters = logs.describe_metric_filters(
            metricName = message['Trigger']['MetricName'] ,
            metricNamespace = message['Trigger']['Namespace']
    )
    response = logs.filter_log_events(
            logGroupName = metricfilters['metricFilters'][0]['logGroupName'] ,
            filterPattern = metricfilters['metricFilters'][0]['filterPattern'],
            startTime = u_from,
            endTime = u_to,
            limit = OUTPUT_LIMIT
        )

    log_events = response['events']
    for e in log_events:
        date = datetime.datetime.fromtimestamp(int(str(e['timestamp'])[:10])) + datetime.timedelta(hours=9)
        log_message = '{"Timestamp":"' + str(date) + '","Message":' + e['message'] + '}'

    slack_message = {
        'channel': SLACK_CHANNEL,
        'text': "%s state is now %s: %s\n%s" % (alarm_name, new_state, reason, log_message)
    }

    req = Request(HOOK_URL, json.dumps(slack_message).encode('utf-8'))
    try:
        response = urlopen(req)
        response.read()
        logger.info("Message posted to %s", slack_message['channel'])
    except HTTPError as e:
        logger.error("Request failed: %d %s", e.code, e.reason)
    except URLError as e:
        logger.error("Server connection failed: %s", e.reason)

```

# INSUFFICIENT DATAが出るとき
Errorのしきい値は>=1にする
default Valueを0にする
0でも1でもない値が大量に存在しているためOKに戻らない

# KMSで
クラウドウォッチログ読み取り権限を付与する
# cloudwatchのログの確認の仕方