Monitoring AWS CloudWatch with Zabbix

Posted: August 11, 2018 in Amazon Web Services (AWS)

This is combination of https://github.com/wawastein/zabbix-cloudwatch and https://github.com/omni-lchen/zabbix-cloudwatch with some modifications from my side (added LLD for Lambda,EBS and Application Load Balancer.

IAM user  has been created with following 2 IAM policies:

{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VisualEditor0",
"Effect": "Allow",
"Action": [
"sns:ListSubscriptionsByTopic",
"lambda:ListFunctions",
"sns:GetTopicAttributes",
"lambda:ListVersionsByFunction",
"lambda:ListAliases",
"sns:ListTopics",
"sns:GetPlatformApplicationAttributes",
"sns:ListSubscriptions",
"sns:GetSubscriptionAttributes",
"sns:CheckIfPhoneNumberIsOptedOut",
"sns:ListEndpointsByPlatformApplication",
"sns:ListPhoneNumbersOptedOut",
"sns:GetEndpointAttributes",
"lambda:ListEventSourceMappings",
"sns:ListPlatformApplications",
"sns:GetSMSAttributes"
],
"Resource": "*"
}
]
}
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VisualEditor0",
"Effect": "Allow",
"Action": [
"ec2:DescribeInstances",
"elasticmapreduce:ListBootstrapActions",
"logs:DescribeSubscriptionFilters",
"logs:DescribeMetricFilters",
"ec2:DescribeSnapshots",
"ec2:DescribeHostReservationOfferings",
"elasticmapreduce:ListInstanceGroups",
"elasticmapreduce:ListInstances",
"ec2:DescribeVolumeStatus",
"elasticmapreduce:ListSecurityConfigurations",
"ec2:DescribeScheduledInstanceAvailability",
"ec2:DescribeVolumes",
"rds:DownloadDBLogFilePortion",
"ec2:DescribeFpgaImageAttribute",
"ec2:DescribeExportTasks",
"logs:FilterLogEvents",
"ec2:DescribeKeyPairs",
"s3:GetIpConfiguration",
"logs:DescribeDestinations",
"ec2:DescribeReservedInstancesListings",
"elasticmapreduce:DescribeSecurityConfiguration",
"events:DescribeRule",
"s3:GetBucketWebsite",
"ec2:DescribeSpotFleetRequestHistory",
"ec2:DescribeVpcClassicLinkDnsSupport",
"ec2:DescribeSnapshotAttribute",
"elasticmapreduce:ListSteps",
"ec2:DescribeIdFormat",
"s3:GetBucketNotification",
"cloudwatch:GetMetricStatistics",
"s3:GetReplicationConfiguration",
"ec2:DescribeVolumeAttribute",
"events:TestEventPattern",
"ec2:DescribeImportSnapshotTasks",
"rds:DescribeReservedDBInstances",
"ec2:DescribeVpcEndpointServicePermissions",
"ec2:GetPasswordData",
"ec2:DescribeScheduledInstances",
"ec2:DescribeImageAttribute",
"cloudwatch:DescribeAlarms",
"ec2:DescribeReservedInstancesModifications",
"ec2:DescribeSubnets",
"logs:ListTagsLogGroup",
"ec2:DescribeMovingAddresses",
"s3:GetLifecycleConfiguration",
"s3:GetBucketTagging",
"s3:GetInventoryConfiguration",
"ec2:DescribeRegions",
"ec2:DescribeFlowLogs",
"ec2:DescribeVpcEndpointServices",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeVpcAttribute",
"cloudwatch:ListMetrics",
"rds:DescribeReservedDBInstancesOfferings",
"elasticmapreduce:DescribeStep",
"cloudwatch:DescribeAlarmHistory",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeNetworkInterfaceAttribute",
"rds:DescribeDBInstances",
"rds:DescribeEngineDefaultClusterParameters",
"ec2:DescribeVpcEndpointConnections",
"rds:DescribeEventCategories",
"ec2:DescribeInstanceStatus",
"rds:DescribeEvents",
"s3:ListBucketMultipartUploads",
"ec2:DescribeHostReservations",
"ec2:DescribeBundleTasks",
"logs:TestMetricFilter",
"ec2:DescribeIdentityIdFormat",
"ec2:DescribeClassicLinkInstances",
"s3:GetBucketVersioning",
"ec2:DescribeVpcEndpointConnectionNotifications",
"ec2:DescribeSecurityGroups",
"rds:DescribeDBSnapshotAttributes",
"ec2:DescribeFpgaImages",
"s3:ListAllMyBuckets",
"rds:ListTagsForResource",
"ec2:DescribeVpcs",
"s3:GetBucketCORS",
"s3:GetObjectVersion",
"ec2:DescribeStaleSecurityGroups",
"s3:GetObjectVersionTagging",
"ec2:DescribeVolumesModifications",
"ec2:GetHostReservationPurchasePreview",
"elasticloadbalancing:DescribeLoadBalancerPolicyTypes",
"rds:DescribeEngineDefaultParameters",
"ec2:DescribePlacementGroups",
"ec2:GetConsoleScreenshot",
"ec2:DescribeInternetGateways",
"s3:GetObjectAcl",
"elasticloadbalancing:DescribeLoadBalancers",
"ec2:GetLaunchTemplateData",
"events:ListRuleNamesByTarget",
"cloudwatch:DescribeAlarmsForMetric",
"ec2:DescribeSpotDatafeedSubscription",
"cloudwatch:ListDashboards",
"s3:GetObjectVersionAcl",
"logs:GetLogEvents",
"elasticloadbalancing:DescribeLoadBalancerPolicies",
"ec2:DescribeAccountAttributes",
"events:ListRules",
"ec2:DescribeNetworkInterfacePermissions",
"ec2:DescribeReservedInstances",
"elasticloadbalancing:DescribeInstanceHealth",
"ec2:DescribeNetworkAcls",
"ec2:DescribeRouteTables",
"events:ListTargetsByRule",
"ec2:DescribeEgressOnlyInternetGateways",
"cloudwatch:GetDashboard",
"ec2:DescribeLaunchTemplates",
"rds:DescribeDBSnapshots",
"elasticmapreduce:ViewEventsFromAllClustersInConsole",
"ec2:DescribeVpnConnections",
"ec2:DescribeVpcPeeringConnections",
"ec2:DescribeReservedInstancesOfferings",
"ec2:DescribeVpcEndpointServiceConfigurations",
"rds:DescribeDBSecurityGroups",
"ec2:DescribePrefixLists",
"ec2:GetReservedInstancesExchangeQuote",
"ec2:DescribeInstanceCreditSpecifications",
"ec2:DescribeVpcClassicLink",
"s3:ListMultipartUploadParts",
"elasticloadbalancing:DescribeLoadBalancerAttributes",
"events:DescribeEventBus",
"s3:GetObject",
"logs:DescribeExportTasks",
"rds:DescribeOrderableDBInstanceOptions",
"s3:GetAnalyticsConfiguration",
"s3:GetObjectVersionForReplication",
"ec2:DescribeVpcEndpoints",
"ec2:DescribeElasticGpus",
"rds:DescribeCertificates",
"ec2:DescribeVpnGateways",
"rds:DescribeOptionGroups",
"s3:ListBucketByTags",
"ec2:DescribeAddresses",
"rds:DescribeDBEngineVersions",
"rds:DescribeDBSubnetGroups",
"cloudwatch:GetMetricData",
"logs:DescribeLogStreams",
"ec2:DescribeInstanceAttribute",
"s3:ListBucketVersions",
"s3:GetBucketLogging",
"ec2:DescribeDhcpOptions",
"s3:GetAccelerateConfiguration",
"rds:DescribePendingMaintenanceActions",
"rds:DescribeDBParameterGroups",
"elasticmapreduce:DescribeCluster",
"s3:GetBucketPolicy",
"ec2:GetConsoleOutput",
"ec2:DescribeSpotPriceHistory",
"s3:GetObjectVersionTorrent",
"s3:GetEncryptionConfiguration",
"ec2:DescribeNetworkInterfaces",
"s3:GetBucketRequestPayment",
"s3:GetObjectTagging",
"elasticmapreduce:ListClusters",
"s3:GetMetricsConfiguration",
"rds:DescribeDBParameters",
"logs:DescribeResourcePolicies",
"rds:DescribeDBClusterSnapshotAttributes",
"rds:DescribeDBClusterParameters",
"rds:DescribeEventSubscriptions",
"logs:DescribeLogGroups",
"ec2:DescribeIamInstanceProfileAssociations",
"ec2:DescribeTags",
"elasticloadbalancing:DescribeTags",
"ec2:DescribeLaunchTemplateVersions",
"ec2:DescribeImportImageTasks",
"rds:DescribeDBLogFiles",
"ec2:DescribeNatGateways",
"s3:GetBucketAcl",
"ec2:DescribeCustomerGateways",
"ec2:DescribeSpotFleetRequests",
"ec2:DescribeHosts",
"ec2:DescribeImages",
"s3:GetObjectTorrent",
"ec2:DescribeSpotFleetInstances",
"ec2:DescribeSecurityGroupReferences",
"rds:DescribeDBClusterSnapshots",
"rds:DescribeOptionGroupOptions",
"rds:DownloadCompleteDBLogFile",
"s3:GetBucketLocation",
"ec2:DescribeConversionTasks",
"rds:DescribeDBClusters",
"rds:DescribeAccountAttributes",
"elasticmapreduce:DescribeJobFlows",
"rds:DescribeDBClusterParameterGroups"
],
"Resource": "*"
}
]
}

Prerequisites:

yum install epel-release
yum install python-pip
yum install jq
pip install boto
pip instal boto3

unzip cloudwatch  zip file and copy content:

aws.discovery, awsLLD.sh and cloudwatch.metric to /usr/lib/zabbix/externalscripts ,make sure files are executable (chmod +x )

Unzip scripts.zip content (it’s folder named scripts) and copy that folder to /usr/lib/zabbix (as in picture bellow-enter IAM user credentails in aws.conf file)

1.PNG

Copy content of cloudwatch_aws.zip (cloudwatch folder) to /opt/zabbix (create that folder if doesn’t exist)

Inside this folders there is file awscred, enter IAM user credentials (i was lazy to point credentials to same file 🙂 )

Make sure following files are set as executable

2.PNG

3.PNG

Test if it works:

/usr/lib/zabbix/scripts/aws_discovery.py --account default --region eu-west-1 --service s3
/usr/lib/zabbix/scripts/aws_discovery.py --account default --region eu-west-1 --service rds
[root@ip-172-31-27-77 scripts]# ./aws_discovery.py --account default --region eu-west-1 --service s3
{"data": [{"{#BUCKET_NAME}": "bucket1"}, {"{#BUCKET_NAME}": "bucket2"}]}

[root@ip-172-31-27-77 scripts]# /usr/lib/zabbix/scripts/aws_discovery.py --account default --region eu-west-1 --service rds
{"data": [{"{#RDS_ID}": "mydb", "{#STORAGE}": 111111}, {"{#RDS_ID}": "mytestore", "{#STORAGE}": 11111},]}

/opt/zabbix/cloudwatch/zabbix-cloudwatch/awsLLD.py -a 'default' -r 'eu-west-1' -q 'ApplicationELB' -c ''
/opt/zabbix/cloudwatch/zabbix-cloudwatch/awsLLD.py -a 'default' -r 'eu-west-1' -q 'EBS' -c ''
/opt/zabbix/cloudwatch/zabbix-cloudwatch/awsLLD.py -a 'default' -r 'eu-west-1' -q 'SNSTopics' -c ''
/opt/zabbix/cloudwatch/zabbix-cloudwatch/awsLLD.py -a 'default' -r 'eu-west-1' -q 'LambdaFunction' -c ''

./awsLLD.py -a 'default' -r 'eu-west-1' -q 'LambdaFunction' -c ''
{
"data": [
{
"{#AWS_REGION}": "eu-west-1",
"{#AWS_ACCOUNT}": "default",
"{#FUNCTION_INAME}": "myfunction",
"{#FUNCTION_NAME}": "myfunction"
}]
}

If something is wrong, probably some prerequisites are not installed properly or files/folder copied to wrong path or some scripts have no +x flag

Creating Zabbix hosts

Create Zabbix hosts for every AWS region where services resides

4.PNG

Attaching Zabbix templates

https://1drv.ms/u/s!AizscpxS0QM4hJ0d_JvivLGeu8nWxg

Create full clone of template for every region and attach it to hosts.

Every template has macros with AWS Zone, change it if needed

4.PNG

Create cron jobs for every resource you want to monitor for Application Load Balancer,EBS,SNS and Lambda

# Lambda monitoring


#--Ireland


*/15 * * * * /opt/zabbix/cloudwatch/zabbix-cloudwatch/cron.d/cron.Lambda.sh "mylambda" "Ireland" "localhost" "default" "eu-west-1" &>/dev/null




# SNS monitoring





#----London

*/10 * * * * /opt/zabbix/cloudwatch/zabbix-cloudwatch/cron.d/cron.SNS.sh " aws-config" "London" "localhost" "default" "eu-west-2" &>/dev/null
*



#Application Load Balancer-----------------------------------------------------------------

*/10 * * * * /opt/zabbix/cloudwatch/zabbix-cloudwatch/cron.d/cron.ApplicationELB.sh "app/loadbalancer/" "Ireland" "localhost" "default" "eu-west-1"


#EBS monitoring----------------------------------------------------------------------
*/12 * * * * /opt/zabbix/cloudwatch/zabbix-cloudwatch/cron.d/cron.EBS.sh "vol-11111111" "aws_north_virginia" "localhost" "default" "us-east-1" &>/dev/null
*

#Ireland


*/10 * * * * /opt/zabbix/cloudwatch/zabbix-cloudwatch/cron.d/cron.EBS.sh "vol-059d78926c41b79c4" "Ireland" "localhost" "default" "eu-west-1" &>/dev/null

 

Make sure all files in /opt/zabbix/cloudwatch/zabbix-cloudwatch/cron.d are executable

 

5.PNG

Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out /  Change )

Google+ photo

You are commenting using your Google+ account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

Connecting to %s