Archive for February, 2019

Get AWS IAM reports – Python script

Posted: February 26, 2019 in Linux, Scripts

Script bellow will run under Docker container and will get IAM user, group membership and IAM policies assigned to user. Script will create HTML file from CSV, will check if there is any diffrencies between old and new files, if there is, then it will write changes in separate file and will send HTML files as email body. Finally, it will connect to CloudTrail to check if IAM policies has changed in last 24 hours, if yes, it will send separate email to specific mailbox. Jira monitors that specific mailbox and ticket will be created automatically from it.

start.py

#!/usr/bin/python3

import boto3
import json
import csv
import sys
import os
import shutil
import pandas as pd
import subprocess
import smtplib
import argparse
import time
from bson import json_util
from shutil import copyfile
from os.path import basename
from smtplib import SMTP
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.header import Header
from email.utils import parseaddr, formataddr
from base64 import encodebytes

###########################

#Input variables

##############################

parser = argparse.ArgumentParser()
parser.add_argument('-aws_acccess_key_id', '-aws_access_key_id', dest='aws_access_key_id', help='AWS Access Key ID.')
parser.add_argument('-aws_secret_access_key_id', '-aws_secret+access_key_id', dest='aws_secret_access_key', help='AWS Secret access key.')
parser.add_argument('-html_body_recipient', '-html_body_recipient', dest='html_body_recipient', help='Recipient of email with HTML files as body.')
parser.add_argument('-jira_email', '-jira_email', dest='jira_email', help='JIRA mailbox for automatic ticket creation .')
parser.add_argument('-support_email', '-support_email', dest='support_email', help='Support mailbox.')
args = parser.parse_args()

aws_access_key_id = args.aws_access_key_id
aws_secret_access_key = args.aws_secret_access_key

html_body_recipient = args.html_body_recipient

jira_email = args.jira_email

support_email = args.support_email

os.environ['AWS_ACCESS_KEY_ID'] = aws_access_key_id

os.environ['AWS_SECRET_ACCESS_KEY'] = aws_secret_access_key

os.environ['AWS_DEFAULT_REGION'] = "eu-west-1"

def read_files(file_list):
            data = ''
            for filename in file_list:
                with open(filename) as file:
                    data += file.read()
            return data

def send_email(recipients=[jira_email],
         subject="AWS IAM Role Changes",
         body="Dear colleagues,\nplease see attached files for recent changes in report",
         zipfiles=['/media/company/company_role_assignemnt_changes.txt', '/media/company/cloudtrail.csv'],
         server="localhost",
         sender="Rundeck ",
         replyto="=?ISO-8859-1?Q?M=F8=F8=F8?= "): #: bool
    """Sends an e-mail"""
    to = ",".join(recipients)
    charset = "utf-8"
    # Testing if body can be encoded with the charset
    try:
        body.encode(charset)
    except UnicodeEncodeError:
        print("Could not encode " + body + " as " + charset + ".")
        return False

    # Split real name (which is optional) and email address parts
    sender_name, sender_addr = parseaddr(sender)
    replyto_name, replyto_addr = parseaddr(replyto)

    sender_name = str(Header(sender_name, charset))
    replyto_name = str(Header(replyto_name, charset))

    # Create the message ('plain' stands for Content-Type: text/plain)
    try:
        msgtext = MIMEText(body.encode(charset), 'plain', charset)
    except TypeError:
        print("MIMEText fail")
        return False

    msg = MIMEMultipart()

    msg['From'] = formataddr((sender_name, sender_addr))
    msg['To'] = to #formataddr((recipient_name, recipient_addr))
    msg['Reply-to'] = formataddr((replyto_name, replyto_addr))
    msg['Subject'] = Header(subject, charset)
    msg['CC'] = support_email
    msg.attach(msgtext)

    for zipfile in zipfiles:
        part = MIMEBase('application', "zip")
        b = open(zipfile, "rb").read()
        # Convert from bytes to a base64-encoded ascii string
        bs = encodebytes(b).decode()
        # Add the ascii-string to the payload
        part.set_payload(bs)
        # Tell the e-mail client that we're using base 64
        part.add_header('Content-Transfer-Encoding', 'base64')
        part.add_header('Content-Disposition', 'attachment; filename="%s"' %
                        os.path.basename(zipfile))
        msg.attach(part)

    s = SMTP()
    try:
        s.connect(server)
    except:
        print("Could not connect to smtp server: " + server)
        return False

        print("Sending the e-mail")
    s.sendmail(sender, recipients, msg.as_string())
    s.quit()
    return True

#def main():
    #send_email()

#if __name__ == "__main__":
    #main()

def send_html_body(header, body):
           msg = MIMEText(body, 'html')  # second parameter is MIME type
           msg['Subject'] = header['Subject']
           msg['From'] = header['From']
           msg['To'] = header['To']
           s = smtplib.SMTP('localhost')
           s.send_message(msg)
           s.quit()

###########################

#start postfix service

###########################

os.system("service postfix start")

filename="/media/company/changes.zip"

if os.path.exists(filename):
    os.remove(filename)

################################################################################

#Make a backup of yesterday's reports

####################################################################################

shutil.copy2('/media/company/company_users.csv', '/media/company/company_users_old.csv')
shutil.copy2('/media/company/company_groups.csv', '/media/company/company_groups_old.csv')
shutil.copy2('/media/company/company_users_policies.csv', '/media/company/company_users_policies_old.csv')
shutil.copy2('/media/company/company_group_policies.csv',  '/media/company/company_group_policies_old.csv')
shutil.copy2('/media/company/company_role_policies.csv', '/media/company/company_role_policies_old.csv')
shutil.copy2('/media/company/company_role_assignment.csv', '/media/company/company_role_assignment_old.csv')
shutil.copy2('/media/company/roles_assign.html', '/media/company/roles_assign_old.html')
shutil.copy2('/media/company/output.json', '/media/company/output_old.json')

#-----------------------------------------------------------

#This section creates CSV reports

#-------------------------------------------------------

#GET Users-------------------------------------------

def subprocess_cmd(command):
    process = subprocess.Popen(command,stdout=subprocess.PIPE, shell=True)
    proc_stdout = process.communicate()[0].strip()
    print (proc_stdout)

subprocess_cmd("aws iam get-account-authorization-details > /media/company/output.json;aws cloudtrail lookup-events --start-time `date -d 'yesterday' '+%m-%d-%Y'` --end-time  `date -d 'today' '+%m-%d-%Y'`>/media/company/changes.json")

with open('/media/company/output.json') as file:
        data = json.load(file)

with open('/media/company/company_users.csv', 'wt') as file:
        file.write('Users\n')
        writer=csv.writer(file)
        for element in data['UserDetailList']:
         if 'UserName' in element.keys():
            s = element['UserName']
         file.write(s + '\n')

#Get Groups------------------------------------------------------

client = boto3.client('iam')

response = client.list_groups(

)

with open('/media/company/company_groups.csv', 'wt') as file:
        file.write('Groups\n')
        writer=csv.writer(file)
        for element in response['Groups']:
         file.write(user['GroupName']+ '\n')

#Get Users with associated policy--------------------------------------------------------

with open('/media/company/company_user_policies.csv', 'wt') as file:
    file.write('User,Policy\n')
    for element in data['UserDetailList']:
        if 'UserName' in element.keys():
            s = element['UserName']
        for policy in element['AttachedManagedPolicies']:
            c = s + ',' + policy['PolicyName']
            file.write(c + '\n')

#Get Groups with associated policies------------------------------------------------------------------

with open('/media/company/company_group_policies.csv', 'wt') as file:
       file.write('Group,Policies\n')
       for element in data['GroupDetailList']:
         s = element['GroupName']
         for policy in element['AttachedManagedPolicies']:

           file.write(s + "," + policy['PolicyName'] + '\n')

#Roles assigned to policies-----------------------------------------------------------------------------------------------------------------

with open('/media/company/company_role_policies.csv', 'wt') as file:
    file.write('Role,Policy\n')
    for element in data['RoleDetailList']:
     if 'RoleName' in element.keys():
         s= element['RoleName']
     for policy in element['AttachedManagedPolicies']:
         c = s + ',' + policy['PolicyName']
         file.write (c + '\n')

#Get IAM policies-----------------------------------------------------------------------------------------------------------

def get_user_group_service(element):
    s = ''
    for e in element['AssumeRolePolicyDocument']['Statement']:
        p = e['Principal']
        if 'Federated' in p:
            s += p['Federated']
        if 'Service' in p:
            obj = p['Service']
            if  type(obj) is str:
                s += obj  # element is string
            else:
                obj.sort()
                s += ''.join(obj) # element is array of strings

        if 'AWS' in p:
            s += p['AWS']
    return s

def get_policies(element):
    list = []
    if 'PolicyName' in element.keys():
        list.append(element['PolicyName'])
    for policy in element['AttachedManagedPolicies']:
        list.append(policy['PolicyName'])
    if len(element['RolePolicyList']) > 0:
        list.append(element['RolePolicyList'][0]['PolicyName'])
    return '--'.join(list)

def main():
        with open('/media/company/company_role_assignment.csv', 'wt') as file:
           file.write('Role,Policy,User/Group/Service\n')
           for element in data['RoleDetailList']:
              s = element['RoleName'] + ',' + get_policies(element) + ',' + get_user_group_service(element)
              file.write(s + '\n')

main()

##########################################################################################################################################

##Get cloudwatch events (if any)

with open('/media/company/changes.json') as file:
 data = json.load(file)

for element in data['Events']:
 for resource in element['Resources']:
  if 'Username' in element:
   with open('/media/company/cloudtrail.csv', 'wt') as file:
    file.write('ResourceType,ResourceName,EventName,UserName\n')
    file.write(resource['ResourceType'] + ',' + resource['ResourceName'] + ',' + element['EventName'] + ',' + element['Username'] + '\n')

###################################################################################################

#Check if there are diferences between old and new reports

######################################################################################################

def compare_files(file1, file2, result_file):
    fname1 = file1
    fname2 = file2
    result = result_file
    output_string = ""
    # Open file for reading in text mode (default mode)
    f1 = open(fname1)
    f2 = open(fname2)
    # Print confirmation
    #print("-----------------------------------")
    #print("Comparing files ", " > " + fname1, "  sign
        elif f1_line != '':
            print ("Line changed:Line-%d" % line_no + "-"+ f1_line)
            output_string += "Line changed:Line-%d" % line_no + "-" + f1_line +"\n"

        ########### If a line does not exist on file1 then mark the output with + sign
        if f1_line == '' and f2_line != '':
            print ("Line removed:Line-%d" % line_no + "-"+ f1_line)
            output_string += "Line removed:Line-%d" % line_no + "-" + f1_line +"\n"
          # otherwise output the line on file2 and mark it with < sign
         #elif f2_line != '':
            #print(&quot;<span id="mce_SELREST_start" style="overflow:hidden;line-height:0;"></span>&lt;&quot;, &quot;Line-%d&quot; %  line_no, f2_line)

         # Print a blank line
         #print()

    #Read the next line from the file
      f1_line = f1.readline()
      f2_line = f2.readline()
      #Increment line counter
      line_no += 1

    # Close the files
    f1.close()
    f2.close()
    #return output_string
    f = open(result, &quot;w&quot;)
    f.write(str(output_string))
    f.close

compare_files(&quot;/media/company/company_users.csv&quot;, &quot;/media/company/company_users_old.csv&quot;, &quot;/media/company/company_users_changes.txt&quot;)

compare_files(&quot;/media/company/company_groups.csv&quot;, &quot;/media/company/company_groups_old.csv&#039;, &#039;/media/company/company_groups_changes.txt&quot;)

compare_files(&quot;/media/company/company_users_policies.csv&quot;, &quot;/media/company/company_users_policies_old.csv&#039;, &#039;/media/company/company_users_policies_changes.txt&quot;)

compare_files(&quot;/media/company/company_group_policies.csv&quot;, &quot;/media/company/company_group_policies_old.csv&#039;, &#039;/media/company/company_group_policies_changes.txt&quot;)

compare_files(&quot;/media/company/company_role_policies.csv&quot;, &quot;/media/company/company_role_policies_old.csv&#039;, &#039;/media/company/company_role_policies_changes.txt&quot;)

compare_files(&quot;/media/company/company_role_assignment.csv&quot;, &quot;/media/company/company_role_assignment_old.csv&quot;, &quot;/media/company/company_role_assignemnt_changes.txt&quot;)

############################################################################################

#Look for changes and if any, collect it to changes.txt file

####################################################################################################

files = [&quot;/media/company/company_group_policies_changes.txt&quot;, &quot;/media/company/company_groups_changes.txt&quot;, &quot;/media/company/company_role_assignemnt_changes.txt&quot;, &quot;/media/company/company_role_policies_changes.txt&quot;, &quot;/media/company/company_users_changes.txt&quot;, &quot;/media/company/company_users_policies_changes.txt&quot;];

with open(&quot;/media/company/changes.txt&quot;,&quot;w&quot;) as file:

  for filename in files:
    with open(filename, &quot;r&quot;) as f:
        contents = f.read()
        output = &quot;&quot;
        if contents:
            output = &quot;FileName:&quot; + os.path.basename(filename) + &quot; &quot; + contents
            file.write(str(output + &#039;\n&#039; ))

#######################################################################################################

#Create HTML from CSV files

###################################################################################################

def html_from_csv(input_file, output_file):
    df = pd.read_csv(input_file)
    pd.set_option(&#039;display.max_colwidth&#039;, -1)
    pd.DataFrame({&#039;a&#039;: [1, 2]}).to_html()
    output = df.to_html(index=False)
    f = open(output_file, &#039;w&#039;)
    f.write(output)
    f.close


html_from_csv(&#039;/media//company/company_group_policies.csv&#039;, &#039;/media/company/group_policy.html&#039;)

html_from_csv(&#039;/media/company/company_groups.csv&#039;, &#039;/media/company/groups.html&#039;)

html_from_csv(&#039;/media/company/company_role_assignment.csv&#039;, &#039;/media/company/roles_assign.html&#039;)

html_from_csv(&#039;/media/company/company_role_policies.csv&#039;, &#039;/media/company/roles.html&#039;)

html_from_csv(&#039;/media/company/company_users_policies.csv&#039;, &#039;/media/company/user_policy.html&#039;)

html_from_csv(&#039;/media/company/company_users.csv&#039;, &#039;/media/company/users.html&#039;)

#########################################################

#Add caption  to HTML files

#################################################################

def html_caption(html_file, caption):
    append_copy = open(html_file, &quot;r&quot;)
    original_text = append_copy.read()
    append_copy.close()
    append_copy = open(html_file, &quot;w&quot;)
    append_copy.write(&quot;
<span id="mce_SELREST_end" style="overflow:hidden;line-height:0;"></span>\n<b>{caption}</b>

\n
\n".format(caption=caption))
    append_copy.write(original_text)
    append_copy.close()


html_caption('/media/company/groups.html', 'IAM Groups')

html_caption('/media/company/group_policy.html', 'IAM Group policy')

html_caption('/media/company/roles_assign.html', 'IAM Role Assignment')

html_caption('/media/company/roles.html', 'IAM Roles')

html_caption('/media/company/user_policy.html', 'IAM User policies')

################################################################################################

#if changes.txt file is not empty, convert it to HTML

###################################################################################################

with open('/media/company/changes.txt') as friendsfile:
    first = friendsfile.read(1)
    if not first:
        print('no changes')

    else:

        with open('/media/company/changes.txt') as fin, open('/media/company/change.txt', 'w') as fout:
           for line in fin:
               fout.write(line.replace(',', ''))
        contents = open("/media/company/changes.txt","r")
        with open("/media/company/changes.html", "w") as e:
             for lines in contents.readlines():
                 e.write("</pre>
<pre>" + lines + "</pre>
<pre>\n")
with open('/media/company/changes.txt') as friendsfile:
 first = friendsfile.read(1)
 if not first:
  #print ('file is empty)
  file_list = ['/media/company/no_changes.html','/media/company/users.html','/media/company/user_policy.html','/media/company/groups.html','/media/company/group_policy.html', '/media/company/roles.html', '/media/company/roles_assign.html']
  data = read_files(file_list)
  header = {'To': html_body_recipient, 'Subject': ' AWS IAM Reports' , 'From': 'svc@company.com'}
  send_html_body(header, data)
  time.sleep(10)
else:
  file_list = ['/media/company/changes.html', '/media/company/users.html',/media/company/user_policy.html','/media/company/groups.html','/media/company/group_policy.html', '/media/company/roles.html', '/media/company/roles_assign.html']
  data = read_files(file_list) header = {'To': html_body_recipient, 'Subject': ' AWS IAM Reports' , 'From': 'svc@company.com'} send_html_body(header, data)
  time.sleep(10)
#Create JIRA ticket if role assignment report has changes
with open('/media/company/company_role_assignemnt_changes.txt') as friendsfile: first = friendsfile.read(1)
if not first:
 print ('roleassignment is empty')
else:
 send_email()
 time.sleep(10)

requirements.txt:

requests
python-dateutil
json_tricks
boto3
pymongo
pandas
awscli

Dockerfile

FROM ubuntu:latest
WORKDIR /home
COPY . .

RUN echo "postfix postfix/mailname string rundeck.company.com" | debconf-set-selections && echo "postfix postfix/main_mailer_type string 'Internet Site'" | debconf-set-selections && apt-get update -y && apt-get install postfix sasl2-bin mailutils vim python3-pip -y && pip3 install --no-cache-dir -r requirements.txt && sed -i s/START=no/START=yes/g /etc/default/saslauthd && echo "[smtp.office365.com]:587 svc@company.com:pass" > /etc/postfix/sasl_passwd && echo "/.+/ svc@company.com" > /etc/postfix/sender_canonical && sed -i 's/inet_protocols = all/inet_protocols = ipv4/g' /etc/postfix/main.cf && sed -i 's/relayhost = /relayhost = [smtp.office365.com]:587/g' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_sasl_auth_enable = yes' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_sasl_security_options = noanonymous' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_tls_security_level = may' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a sender_canonical_maps = regexp:/etc/postfix/sender_canonical' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_tls_CAfile = /etc/postfix/cacert.pem' /etc/postfix/main.cf &&  mv /home/cacert.pem /etc/postfix/ && postmap hash:/etc/postfix/sasl_passwd && postmap hash:/etc/postfix/sender_canonical

ENTRYPOINT ["./start.py"]

Finally, we’ll map host folder to docker container so we can keep track of yesterday’s reports

docker run -it -v /home/centos/docker:/media test /bin/bash

Advertisements

Docker – create Postfix container

Posted: February 21, 2019 in docker, Linux

In this post we’ll create Docker image with postfix installed and configured as Office 365 relay host, pip3 will be also installes as well as Python request.

cacert.pem file contains Microsoft certificates for secure connection. Content of that file is output of following command:

openssl s_client -showcerts -starttls smtp -crlf -connect smtp.office365.com:587

start.sh is simple bash script which starts postfix and saslauthd services and sends test email. In order to prevent closing docker container before sending email i added sleep command at the end

#!/bin/bash

service postfix start && service saslauthd start
echo "sending email..."
echo "this is the body" | mail -s "this is the subject" "dvucanovic@example.com"
sleep 20

requirements.txt contains one string: requests, it will be used by pip3 command

Docker file

FROM: ubuntu:latest

WORKDIR /home

COPY . .

RUN echo "postfix postfix/mailname string rundeck.example.com" | debconf-set-selections && echo "postfix postfix/main_mailer_type string 'Internet Site'" | debconf-set-selections && apt-get update -y && apt-get install postfix sasl2-bin mailutils python3-pip -y && pip3 install --no-cache-dir -r requirements.txt && sed -i s/START=no/START=yes/g /etc/default/saslauthd && echo "[smtp.office365.com]:587 svc-user@example.com:password" > /etc/postfix/sasl_passwd && echo "/.+/ ssvc-user@example.com" > /etc/postfix/sender_canonical && sed -i 's/inet_protocols = all/inet_protocols = ipv4/g' /etc/postfix/main.cf && sed -i 's/relayhost = /relayhost = [smtp.office365.com]:587/g' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_sasl_auth_enable = yes' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_sasl_security_options = noanonymous' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_tls_security_level = may' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a sender_canonical_maps = regexp:/etc/postfix/sender_canonical' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_tls_CAfile = /etc/postfix/cacert.pem' /etc/postfix/main.cf && sed -i 's#128#& 172.17.0.0/16#' /etc/postfix/main.cf && mv /home/cacert.pem /etc/postfix/ && postmap hash:/etc/postfix/sasl_passwd && postmap hash:/etc/postfix/sender_canonical

ENTRYPOINT ["./start.sh"]

Put all 3 files in same directory and run

docker build . -t some_tag

 

In previous posts we created subtasks using Python and bash, in this one we’ll use python to do following:

  • Create task which will have variable in name
  • Script will searh if that task already exists and it’s status (resolved/opened)
  • if there is no task with status DONE, it will create it
  • it will check if there is sub tasks for that task, if there is no any it will create subtask
  • if task already exists (with status different than DONE), it will check if there is subtasks, if no it will create it
  • scripts uses client variable as parameter which will be used to build task/sub task name
#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import json
import requests
import os
import urllib2
import argparse
from getpass import getpass
from json_tricks import dump,dumps

username = 'user'
password = 'Pass'

client = 'Newbie'

query = "Managed Services On-boarding " + client

def create_subtask(summary, key, line, user, password):

    headers = {"Content-Type": "application/json"}
    data = {"fields": {"project": {"key": key},"parent": {"key": line},"summary": summary,
   "issuetype": {"name": "Sub-task"}}}
    response = requests.post("https://jira.corp.company.com/rest/api/latest/issue/",
    headers=headers,data=json.dumps(data),auth=(user, password))
    out= response.json()
    print out

def jql_query(query, username, password):

    headers = {
   'Content-Type': 'application/json',}
    params = (
    ('jql', 'project="Managed Services" AND summary~"'+query+'" AND issuetype="Task" AND status!="DONE"'),
    )
    response = requests.get('https://jira.corp.company.com/rest/api/2/search', headers=headers, params=params, auth=(username, password))
    data =  response.json()
    return data

data = jql_query(query, username, password)

if data["total"] == 0:

   headers = {"Content-Type": "application/json"}
   data = {"fields":{"labels":["SERVICE_MANAGEMENT"],"reporter":{"name":"user"},"project":{"key":"MS"},"summary":"Managed Services On-boarding {client}".format(**locals()),"description":"Managed service onboarding task for {client} client".format(**locals()),"issuetype":{"name":"Task"}}}
   response = requests.post("https://jira.corp.company.com/rest/api/latest/issue/",
   headers=headers, data=json.dumps(data), auth=(username, password))
   out= response.json()
   print out
   data = jql_query(query, username, password)
   for issue in data['issues']:
      if len(issue['fields']['subtasks']) == 0:
         line = issue['key']
         create_subtask(client + ":CRM – Set up Client kick off meeting.", "MS", line, username, password)

else:
   data = jql_query(query, username, password)
   for issue in data['issues']:
      if len(issue['fields']['subtasks']) == 0:
         line = issue['key']
         create_subtask(client + ":CRM – Set up Client kick off meeting.", "MS", line, username, password)

Changes compared with previous version:

  • Script will search all regions
  • added time when instance is launched
  • added option to terminate all EBS volumes associated with instance
  • uses Simple email service
  • uses Lambda environmental variables

6-1.png

import smtplib
import boto3
import collections
import datetime
import time
import sys
import os
from datetime import datetime
from dateutil.relativedelta import relativedelta
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText

AWSAccountID = boto3.client('sts').get_caller_identity()['Account']
AWSUser = boto3.client('sts').get_caller_identity()['UserId']
AWSAccount = os.environ["AWS_Account_Name"]
# create date variables
date_after_month = datetime.now() + relativedelta(days = 7)
# date_after_month.strftime('%d/%m/%Y')
today = datetime.now().strftime('%d/%m/%Y')

# AWS SES variables
EMAIL_HOST = os.environ["EMAIL_HOST"]
EMAIL_HOST_USER = os.environ["EMAIL_HOST_USER"] # Replace with your SMTP username
EMAIL_HOST_PASSWORD = os.environ["EMAIL_HOST_PASSWORD"] # Replace with your SMTP password
SENT_TO = os.environ["SENT_TO"]
EMAIL_PORT = 587

def lambda_handler(event, context):

    instance_ids = []
    launch_date = ""
    launched = ""
    launched1 = ""
    ec = boto3.client('ec2')
    ec2_regions = [region['RegionName'] for region in ec.describe_regions()['Regions']]
    for region in ec2_regions:
     ec = boto3.client('ec2', region_name = region)
     ec2 = boto3.resource('ec2',region_name = region)
     reservations = ec.describe_instances().get('Reservations', []) 

     for reservation in reservations:
      for instance in reservation['Instances']:
         tags = {}
         for tag in instance['Tags']:
             tags[tag['Key']] = tag['Value']
             if tag['Key'] == 'Name':
               name = tag['Value']
         if not 'Owner' in tags or tags['Owner'] == 'unknown' or tags['Owner'] == 'Unknown':
              instance_ids.append(instance['InstanceId'])  

                # Check if "TerminateOn" tag exists:

              if 'TerminateOn' in tags:
                  # compare TerminteOn value with current date
                    if tags["TerminateOn"] == today:

                    # Check if termination protection is enabled
                     terminate_protection = ec.describe_instance_attribute(InstanceId = instance['InstanceId'] ,Attribute = 'disableApiTermination')
                     protection_value = (terminate_protection['DisableApiTermination']['Value'])
                     #if enabled disable it
                     if protection_value == True:
                        ec.modify_instance_attribute(InstanceId = instance['InstanceId'],Attribute = "disableApiTermination",Value = "False" )

                     volumes_to_delete = ec.describe_instance_attribute(InstanceId=instance['InstanceId'],Attribute='blockDeviceMapping')

                     for v in volumes_to_delete['BlockDeviceMappings']:

                       if volumes_to_delete['InstanceId']:
                        launch_date = str(v['Ebs']['AttachTime'])
                        device_name = v['DeviceName'] 

                        ec.modify_instance_attribute(InstanceId = instance['InstanceId'],Attribute = 'blockDeviceMapping',BlockDeviceMappings = [{'DeviceName': device_name,'Ebs': {'DeleteOnTermination':True}}])

                     # send email that instance is terminated
                     body = "<b>AWS Account:</b>" + AWSAccount + "<b>AWSAccountNumber:</b>" + AWSAccountID + "<b>Instance Name:</b>" + name + "<b>Instance ID:</b>" + instance['InstanceId'] + "<b>Created At:</b>" + launch_date + "<b>To be terminated at</b>:Now <b>Note:</b>er tag is missing from this instance, hence,instance is removed."
                     msg = MIMEMultipart('alternative')
                     msg['Subject'] = "Notification of terminated instances in " + region + " AWS region"
                     msg['From'] = "ses@amazon.com"
                     msg['To'] = SENT_TO
                     html = body
                     mime_text = MIMEText(html, 'html')
                     msg.attach(mime_text)
                     s = smtplib.SMTP(EMAIL_HOST, EMAIL_PORT)
                     s.starttls()
                     s.login(EMAIL_HOST_USER, EMAIL_HOST_PASSWORD)
                     s.sendmail("ses@amazon.com", SENT_TO, msg.as_string())
                     s.quit()
                     # Terminate instance
                     ec.terminate_instances(InstanceIds = [instance['InstanceId']])

                    else:

                      now = datetime.now()
                      future = tags["TerminateOn"]
                      TerminateOn = datetime.strptime(future, "%d/%m/%Y")
                      days = (TerminateOn-now).days
                      volume1 = ec.describe_instance_attribute(InstanceId = instance['InstanceId'],Attribute = 'blockDeviceMapping')
                      for a in volume1['BlockDeviceMappings']:
                        if volume1['InstanceId']:
                         launched1  = str(a['Ebs']['AttachTime'])
                      body = "<b>AWS Account:</b>" + AWSAccount + "<b>AWSAccountNumber:</b>" + AWSAccountID + "<b>Instance Name:</b>" + name + "b&gt;Instance ID:" + instance['InstanceId'] + "<b>Created At:</b>" + launched1 + "Owner tag is mising from this instance, hence,instance will be removed."
                      msg = MIMEMultipart('alternative')
                      msg['Subject'] = "Notification of shutting down instances in " + region + " AWS region"
                      msg['From'] = "ses@amazon.com"
                      msg['To'] = SENT_TO
                      html = body
                      mime_text = MIMEText(html, 'html')
                      msg.attach(mime_text)
                      s = smtplib.SMTP(EMAIL_HOST, EMAIL_PORT)
                      s.starttls()
                      s.login(EMAIL_HOST_USER, EMAIL_HOST_PASSWORD)
                      s.sendmail("ses@amazon.com, SENT_TO, msg.as_string())
                      s.quit()
					  # stop instance
                      ec.stop_instances(InstanceIds=[instance['InstanceId']])

              else:
                 if not 'TerminateOn' in tags:#, create it
                  ec2.create_tags(Resources=[instance['InstanceId']],Tags=[{'Key':'TerminateOn','Value':date_after_month.strftime('%d/%m/%Y')}])
                  volume = ec.describe_instance_attribute(InstanceId = instance['InstanceId'],Attribute = 'blockDeviceMapping')

                  for s in volume['BlockDeviceMappings']:
                     if volume['InstanceId']:
                      launched  = str(s['Ebs']['AttachTime'])

                  body = "<b>AWS Account:</b>" + AWSAccount + "<b>AWS Account Number:</b>"AWSAccountID + "<b>Instance Name:</b>" + name + "<b>Instance ID:</b>" + instance['InstanceId'] + Created<b> At:</b>" + launched + "<b>To be terminated at</b>: (6 days from now) <b>Note:</b>Owner tag is missing from this instance.If you do not wish this instance to be removed, please update the Owner tag."
                  msg = MIMEMultipart('alternative')
                  msg['Subject'] = "Notification of shutting down instances in " + region + " AWS region"
                  msg['From'] = "ses@amazon.com"
                  msg['To'] = SENT_TO
                  html = body
                  mime_text = MIMEText(html, 'html')
                  msg.attach(mime_text)
                  s = smtplib.SMTP(EMAIL_HOST, EMAIL_PORT)
                  s.starttls()
                  s.login(EMAIL_HOST_USER, EMAIL_HOST_PASSWORD)
                  s.sendmail("ses@amazon.com", SENT_TO, msg.as_string())
                  s.quit()
				  # stop instance
                  ec.stop_instances(InstanceIds=[instance['InstanceId']])

In last post we configured site-to-site VPN between StrongSwan and AWS VPC Gateway using stating route. In this one we’ll use BGP.

1-0.PNG

I’ll be creating Site-to-Site VPN between 2 AWS regions, although we usually take adventage of VPC peering, for demonstration purposes i used EC2 instance (CentoOS 7), public IP:3.120.227.213, internal IP:172.31.36.231, AWS VPN gateway creates 2 tunnels, public IPs:34.246.169.212 and 54.49.220.63

Creating AWS VPN Gateway

From AWS VPC console click Customer gateway-New Customer Gateway

1.png

Specify IP adress of StrongSwan server (3.120.227.213) and BGP ASN (65600)

We’ll create BGP on StrongSwan server later on.

2.PNG

Create Virtual Private Gateway

3.PNG

4.PNG

Atach Virtual Private Gateway to VPC-select Virtual private gateway-Action-attach-select VPC and click yes, attach

5.png

Create VPN Connection-click Site-to-Site VPN Connection-create VPN connection

6.PNG

Select Virtual private gateway,Customer gateway, Routing option: Dynamic

7.PNG

Download VPN Gateway configuration

8.png

9.PNG

Install quagga (BGP router emulator) on StrongSwan server

sed -i 's/enforcing/disabled/g' /etc/selinux/config /etc/selinux/config
setenforce 0
yum install iptables-services
systemctl enable iptables
systemctl start iptables
yum install quagga
chmod -R 777 /etc/quagga/
systemctl enable zebra
systemctl start zebra
systemctl start bgpd
systemctl enable bgpd
cp /usr/share/doc/quagga-*/bgpd.conf.sample /etc/quagga/bgpd.conf

Creating BGP area

vtysh
config t
router bgp 65600
network 172.31.36.0/24
neighbor 169.254.20.181 remote-as 64512
neighbor 169.254.21.193 remote-as 64512
#if get BGP is already running; AS is 7675
no router bgp 7675
do write
exit
exit

AWS BGP ASN is 64512

10.PNG

Neighbor are defined in AWS Virtual gateway configuration file

For tunnel 1:
11.PNG

And tunnel 2:

0.PNG

Install StrongSwan

/etc/strongswan/ipsec.conf:

conn %default
# Authentication Method : Pre-Shared Key
#authby=psk
leftauth=psk
rightauth=psk
# Encryption Algorithm : aes-128-cbc
# Authentication Algorithm : sha1
# Perfect Forward Secrecy : Diffie-Hellman Group 2
ike=aes128-sha1-modp1024!
#ike=aes256-sha256-modp2048s256,aes128-sha1-modp1024!
# Lifetime : 28800 seconds
ikelifetime=28800s
# Phase 1 Negotiation Mode : main
aggressive=no
# Protocol : esp
# Encryption Algorithm : aes-128-cbc
# Authentication Algorithm : hmac-sha1-96
# Perfect Forward Secrecy : Diffie-Hellman Group 2
#esp=aes128-sha256-modp2048s256,aes128-sha1-modp1024!
esp=aes128-sha1-modp1024!
# Lifetime : 3600 seconds
lifetime=3600s
# Mode : tunnel
type=tunnel
# DPD Interval : 10
dpddelay=10s
# DPD Retries : 3
dpdtimeout=30s
# Tuning Parameters for AWS Virtual Private Gateway:
keyexchange=ikev1
#keyingtries=%forever
rekey=yes
reauth=no
dpdaction=restart
closeaction=restart
#left=%defaultroute
leftsubnet=0.0.0.0/0,::/0
rightsubnet=0.0.0.0/0,::/0
leftupdown=/etc/strongswan/ipsec-vti.sh
installpolicy=yes
compress=no
mobike=no
conn AWS-VPC-GW1
# Customer Gateway: :
left=172.31.36.231
leftid=3.120.227.213
# Virtual Private Gateway :
right=34.246.169.212
rightid=34.246.169.212
auto=start
mark=100
#reqid=1
conn AWS-VPC-GW2
# Customer Gateway: :
left=172.31.36.231
leftid=3.120.227.213
#leftsubnet=172.31.36.0/24
# Virtual Private Gateway :
right=52.49.220.63
rightid=52.49.220.63
#rightsubnet=172.31.16.0/24
auto=start
mark=200

Tunnel 1 Virtual cutomer/private gateway:

11.PNG

Tunnel 2 Virtual cutomer/private gateway:

12.PNG

Public IP of AWS VPN Gateway tunnel

11-a.PNG

/etc/strongswan/ipsec-vti.sh:

IP=$(which ip)
IPTABLES=$(which iptables)

PLUTO_MARK_OUT_ARR=(${PLUTO_MARK_OUT//// })
PLUTO_MARK_IN_ARR=(${PLUTO_MARK_IN//// })
case "$PLUTO_CONNECTION" in
AWS-VPC-GW1)
VTI_INTERFACE=vti1
VTI_LOCALADDR=169.254.20.182/30
VTI_REMOTEADDR=169.254.20.181/30
;;
AWS-VPC-GW2)
VTI_INTERFACE=vti2
VTI_LOCALADDR=169.254.21.194/30
VTI_REMOTEADDR=169.254.21.193/30
;;
esac

case "${PLUTO_VERB}" in
up-client)
#$IP tunnel add ${VTI_INTERFACE} mode vti local ${PLUTO_ME} remote ${PLUTO_PEER} okey ${PLUTO_MARK_OUT_ARR[0]} ikey ${PLUTO_MARK_IN_ARR[0]}
$IP link add ${VTI_INTERFACE} type vti local ${PLUTO_ME} remote ${PLUTO_PEER} okey ${PLUTO_MARK_OUT_ARR[0]} ikey ${PLUTO_MARK_IN_ARR[0]}
sysctl -w net.ipv4.conf.${VTI_INTERFACE}.disable_policy=1
sysctl -w net.ipv4.conf.${VTI_INTERFACE}.rp_filter=2 || sysctl -w net.ipv4.conf.${VTI_INTERFACE}.rp_filter=0
$IP addr add ${VTI_LOCALADDR} remote ${VTI_REMOTEADDR} dev ${VTI_INTERFACE}
$IP link set ${VTI_INTERFACE} up mtu 1436
$IPTABLES -t mangle -I FORWARD -o ${VTI_INTERFACE} -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
$IPTABLES -t mangle -I INPUT -p esp -s ${PLUTO_PEER} -d ${PLUTO_ME} -j MARK --set-xmark ${PLUTO_MARK_IN}
$IP route flush table 220
#/etc/init.d/bgpd reload || /etc/init.d/quagga force-reload bgpd
;;
down-client)
#$IP tunnel del ${VTI_INTERFACE}
$IP link del ${VTI_INTERFACE}
$IPTABLES -t mangle -D FORWARD -o ${VTI_INTERFACE} -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu
$IPTABLES -t mangle -D INPUT -p esp -s ${PLUTO_PEER} -d ${PLUTO_ME} -j MARK --set-xmark ${PLUTO_MARK_IN}
;;
esac

# Enable IPv4 forwarding
sysctl -w net.ipv4.ip_forward=1
sysctl -w net.ipv4.conf.eth0.disable_xfrm=1
sysctl -w net.ipv4.conf.eth0.disable_policy=1

/etc/strongswan/ipsec-vti.sh:

3.120.227.213 34.246.169.212 : PSK "yZ1oMi60GNgzgXmBSHo84w0M_uYMFL5R"
3.120.227.213 52.49.220.63 : PSK "RDihsBvmWrJ1PbI0HwJ7vMJW24qVJKbx"

If all is fine, both tunnels should be UP

15