Get AWS IAM reports – Python script

Posted: February 26, 2019 in Linux, Scripts

Script bellow will run under Docker container and will get IAM user, group membership and IAM policies assigned to user. Script will create HTML file from CSV, will check if there is any diffrencies between old and new files, if there is, then it will write changes in separate file and will send HTML files as email body. Finally, it will connect to CloudTrail to check if IAM policies has changed in last 24 hours, if yes, it will send separate email to specific mailbox. Jira monitors that specific mailbox and ticket will be created automatically from it.

start.py

#!/usr/bin/python3

import boto3
import json
import csv
import sys
import os
import shutil
import pandas as pd
import subprocess
import smtplib
import argparse
import time
from bson import json_util
from shutil import copyfile
from os.path import basename
from smtplib import SMTP
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.header import Header
from email.utils import parseaddr, formataddr
from base64 import encodebytes

###########################

#Input variables

##############################

parser = argparse.ArgumentParser()
parser.add_argument('-aws_acccess_key_id', '-aws_access_key_id', dest='aws_access_key_id', help='AWS Access Key ID.')
parser.add_argument('-aws_secret_access_key_id', '-aws_secret+access_key_id', dest='aws_secret_access_key', help='AWS Secret access key.')
parser.add_argument('-html_body_recipient', '-html_body_recipient', dest='html_body_recipient', help='Recipient of email with HTML files as body.')
parser.add_argument('-jira_email', '-jira_email', dest='jira_email', help='JIRA mailbox for automatic ticket creation .')
parser.add_argument('-support_email', '-support_email', dest='support_email', help='Support mailbox.')
args = parser.parse_args()

aws_access_key_id = args.aws_access_key_id
aws_secret_access_key = args.aws_secret_access_key

html_body_recipient = args.html_body_recipient

jira_email = args.jira_email

support_email = args.support_email

os.environ['AWS_ACCESS_KEY_ID'] = aws_access_key_id

os.environ['AWS_SECRET_ACCESS_KEY'] = aws_secret_access_key

os.environ['AWS_DEFAULT_REGION'] = "eu-west-1"

def read_files(file_list):
            data = ''
            for filename in file_list:
                with open(filename) as file:
                    data += file.read()
            return data

def send_email(recipients=[jira_email],
         subject="AWS IAM Role Changes",
         body="Dear colleagues,\nplease see attached files for recent changes in report",
         zipfiles=['/media/company/company_role_assignemnt_changes.txt', '/media/company/cloudtrail.csv'],
         server="localhost",
         sender="Rundeck ",
         replyto="=?ISO-8859-1?Q?M=F8=F8=F8?= "): #: bool
    """Sends an e-mail"""
    to = ",".join(recipients)
    charset = "utf-8"
    # Testing if body can be encoded with the charset
    try:
        body.encode(charset)
    except UnicodeEncodeError:
        print("Could not encode " + body + " as " + charset + ".")
        return False

    # Split real name (which is optional) and email address parts
    sender_name, sender_addr = parseaddr(sender)
    replyto_name, replyto_addr = parseaddr(replyto)

    sender_name = str(Header(sender_name, charset))
    replyto_name = str(Header(replyto_name, charset))

    # Create the message ('plain' stands for Content-Type: text/plain)
    try:
        msgtext = MIMEText(body.encode(charset), 'plain', charset)
    except TypeError:
        print("MIMEText fail")
        return False

    msg = MIMEMultipart()

    msg['From'] = formataddr((sender_name, sender_addr))
    msg['To'] = to #formataddr((recipient_name, recipient_addr))
    msg['Reply-to'] = formataddr((replyto_name, replyto_addr))
    msg['Subject'] = Header(subject, charset)
    msg['CC'] = support_email
    msg.attach(msgtext)

    for zipfile in zipfiles:
        part = MIMEBase('application', "zip")
        b = open(zipfile, "rb").read()
        # Convert from bytes to a base64-encoded ascii string
        bs = encodebytes(b).decode()
        # Add the ascii-string to the payload
        part.set_payload(bs)
        # Tell the e-mail client that we're using base 64
        part.add_header('Content-Transfer-Encoding', 'base64')
        part.add_header('Content-Disposition', 'attachment; filename="%s"' %
                        os.path.basename(zipfile))
        msg.attach(part)

    s = SMTP()
    try:
        s.connect(server)
    except:
        print("Could not connect to smtp server: " + server)
        return False

        print("Sending the e-mail")
    s.sendmail(sender, recipients, msg.as_string())
    s.quit()
    return True

#def main():
    #send_email()

#if __name__ == "__main__":
    #main()

def send_html_body(header, body):
           msg = MIMEText(body, 'html')  # second parameter is MIME type
           msg['Subject'] = header['Subject']
           msg['From'] = header['From']
           msg['To'] = header['To']
           s = smtplib.SMTP('localhost')
           s.send_message(msg)
           s.quit()

###########################

#start postfix service

###########################

os.system("service postfix start")

filename="/media/company/changes.zip"

if os.path.exists(filename):
    os.remove(filename)

################################################################################

#Make a backup of yesterday's reports

####################################################################################

shutil.copy2('/media/company/company_users.csv', '/media/company/company_users_old.csv')
shutil.copy2('/media/company/company_groups.csv', '/media/company/company_groups_old.csv')
shutil.copy2('/media/company/company_users_policies.csv', '/media/company/company_users_policies_old.csv')
shutil.copy2('/media/company/company_group_policies.csv',  '/media/company/company_group_policies_old.csv')
shutil.copy2('/media/company/company_role_policies.csv', '/media/company/company_role_policies_old.csv')
shutil.copy2('/media/company/company_role_assignment.csv', '/media/company/company_role_assignment_old.csv')
shutil.copy2('/media/company/roles_assign.html', '/media/company/roles_assign_old.html')
shutil.copy2('/media/company/output.json', '/media/company/output_old.json')

#-----------------------------------------------------------

#This section creates CSV reports

#-------------------------------------------------------

#GET Users-------------------------------------------

def subprocess_cmd(command):
    process = subprocess.Popen(command,stdout=subprocess.PIPE, shell=True)
    proc_stdout = process.communicate()[0].strip()
    print (proc_stdout)

subprocess_cmd("aws iam get-account-authorization-details > /media/company/output.json;aws cloudtrail lookup-events --start-time `date -d 'yesterday' '+%m-%d-%Y'` --end-time  `date -d 'today' '+%m-%d-%Y'`>/media/company/changes.json")

with open('/media/company/output.json') as file:
        data = json.load(file)

with open('/media/company/company_users.csv', 'wt') as file:
        file.write('Users\n')
        writer=csv.writer(file)
        for element in data['UserDetailList']:
         if 'UserName' in element.keys():
            s = element['UserName']
         file.write(s + '\n')

#Get Groups------------------------------------------------------

client = boto3.client('iam')

response = client.list_groups(

)

with open('/media/company/company_groups.csv', 'wt') as file:
        file.write('Groups\n')
        writer=csv.writer(file)
        for element in response['Groups']:
         file.write(user['GroupName']+ '\n')

#Get Users with associated policy--------------------------------------------------------

with open('/media/company/company_user_policies.csv', 'wt') as file:
    file.write('User,Policy\n')
    for element in data['UserDetailList']:
        if 'UserName' in element.keys():
            s = element['UserName']
        for policy in element['AttachedManagedPolicies']:
            c = s + ',' + policy['PolicyName']
            file.write(c + '\n')

#Get Groups with associated policies------------------------------------------------------------------

with open('/media/company/company_group_policies.csv', 'wt') as file:
       file.write('Group,Policies\n')
       for element in data['GroupDetailList']:
         s = element['GroupName']
         for policy in element['AttachedManagedPolicies']:

           file.write(s + "," + policy['PolicyName'] + '\n')

#Roles assigned to policies-----------------------------------------------------------------------------------------------------------------

with open('/media/company/company_role_policies.csv', 'wt') as file:
    file.write('Role,Policy\n')
    for element in data['RoleDetailList']:
     if 'RoleName' in element.keys():
         s= element['RoleName']
     for policy in element['AttachedManagedPolicies']:
         c = s + ',' + policy['PolicyName']
         file.write (c + '\n')

#Get IAM policies-----------------------------------------------------------------------------------------------------------

def get_user_group_service(element):
    s = ''
    for e in element['AssumeRolePolicyDocument']['Statement']:
        p = e['Principal']
        if 'Federated' in p:
            s += p['Federated']
        if 'Service' in p:
            obj = p['Service']
            if  type(obj) is str:
                s += obj  # element is string
            else:
                obj.sort()
                s += ''.join(obj) # element is array of strings

        if 'AWS' in p:
            s += p['AWS']
    return s

def get_policies(element):
    list = []
    if 'PolicyName' in element.keys():
        list.append(element['PolicyName'])
    for policy in element['AttachedManagedPolicies']:
        list.append(policy['PolicyName'])
    if len(element['RolePolicyList']) > 0:
        list.append(element['RolePolicyList'][0]['PolicyName'])
    return '--'.join(list)

def main():
        with open('/media/company/company_role_assignment.csv', 'wt') as file:
           file.write('Role,Policy,User/Group/Service\n')
           for element in data['RoleDetailList']:
              s = element['RoleName'] + ',' + get_policies(element) + ',' + get_user_group_service(element)
              file.write(s + '\n')

main()

##########################################################################################################################################

##Get cloudwatch events (if any)

with open('/media/company/changes.json') as file:
 data = json.load(file)

for element in data['Events']:
 for resource in element['Resources']:
  if 'Username' in element:
   with open('/media/company/cloudtrail.csv', 'wt') as file:
    file.write('ResourceType,ResourceName,EventName,UserName\n')
    file.write(resource['ResourceType'] + ',' + resource['ResourceName'] + ',' + element['EventName'] + ',' + element['Username'] + '\n')

###################################################################################################

#Check if there are diferences between old and new reports

######################################################################################################

def compare_files(file1, file2, result_file):
    fname1 = file1
    fname2 = file2
    result = result_file
    output_string = ""
    # Open file for reading in text mode (default mode)
    f1 = open(fname1)
    f2 = open(fname2)
    # Print confirmation
    #print("-----------------------------------")
    #print("Comparing files ", " > " + fname1, "  sign
        elif f1_line != '':
            print ("Line changed:Line-%d" % line_no + "-"+ f1_line)
            output_string += "Line changed:Line-%d" % line_no + "-" + f1_line +"\n"

        ########### If a line does not exist on file1 then mark the output with + sign
        if f1_line == '' and f2_line != '':
            print ("Line removed:Line-%d" % line_no + "-"+ f1_line)
            output_string += "Line removed:Line-%d" % line_no + "-" + f1_line +"\n"
          # otherwise output the line on file2 and mark it with < sign
         #elif f2_line != '':
            #print(&quot;<span id="mce_SELREST_start" style="overflow:hidden;line-height:0;"></span>&lt;&quot;, &quot;Line-%d&quot; %  line_no, f2_line)

         # Print a blank line
         #print()

    #Read the next line from the file
      f1_line = f1.readline()
      f2_line = f2.readline()
      #Increment line counter
      line_no += 1

    # Close the files
    f1.close()
    f2.close()
    #return output_string
    f = open(result, &quot;w&quot;)
    f.write(str(output_string))
    f.close

compare_files(&quot;/media/company/company_users.csv&quot;, &quot;/media/company/company_users_old.csv&quot;, &quot;/media/company/company_users_changes.txt&quot;)

compare_files(&quot;/media/company/company_groups.csv&quot;, &quot;/media/company/company_groups_old.csv&#039;, &#039;/media/company/company_groups_changes.txt&quot;)

compare_files(&quot;/media/company/company_users_policies.csv&quot;, &quot;/media/company/company_users_policies_old.csv&#039;, &#039;/media/company/company_users_policies_changes.txt&quot;)

compare_files(&quot;/media/company/company_group_policies.csv&quot;, &quot;/media/company/company_group_policies_old.csv&#039;, &#039;/media/company/company_group_policies_changes.txt&quot;)

compare_files(&quot;/media/company/company_role_policies.csv&quot;, &quot;/media/company/company_role_policies_old.csv&#039;, &#039;/media/company/company_role_policies_changes.txt&quot;)

compare_files(&quot;/media/company/company_role_assignment.csv&quot;, &quot;/media/company/company_role_assignment_old.csv&quot;, &quot;/media/company/company_role_assignemnt_changes.txt&quot;)

############################################################################################

#Look for changes and if any, collect it to changes.txt file

####################################################################################################

files = [&quot;/media/company/company_group_policies_changes.txt&quot;, &quot;/media/company/company_groups_changes.txt&quot;, &quot;/media/company/company_role_assignemnt_changes.txt&quot;, &quot;/media/company/company_role_policies_changes.txt&quot;, &quot;/media/company/company_users_changes.txt&quot;, &quot;/media/company/company_users_policies_changes.txt&quot;];

with open(&quot;/media/company/changes.txt&quot;,&quot;w&quot;) as file:

  for filename in files:
    with open(filename, &quot;r&quot;) as f:
        contents = f.read()
        output = &quot;&quot;
        if contents:
            output = &quot;FileName:&quot; + os.path.basename(filename) + &quot; &quot; + contents
            file.write(str(output + &#039;\n&#039; ))

#######################################################################################################

#Create HTML from CSV files

###################################################################################################

def html_from_csv(input_file, output_file):
    df = pd.read_csv(input_file)
    pd.set_option(&#039;display.max_colwidth&#039;, -1)
    pd.DataFrame({&#039;a&#039;: [1, 2]}).to_html()
    output = df.to_html(index=False)
    f = open(output_file, &#039;w&#039;)
    f.write(output)
    f.close


html_from_csv(&#039;/media//company/company_group_policies.csv&#039;, &#039;/media/company/group_policy.html&#039;)

html_from_csv(&#039;/media/company/company_groups.csv&#039;, &#039;/media/company/groups.html&#039;)

html_from_csv(&#039;/media/company/company_role_assignment.csv&#039;, &#039;/media/company/roles_assign.html&#039;)

html_from_csv(&#039;/media/company/company_role_policies.csv&#039;, &#039;/media/company/roles.html&#039;)

html_from_csv(&#039;/media/company/company_users_policies.csv&#039;, &#039;/media/company/user_policy.html&#039;)

html_from_csv(&#039;/media/company/company_users.csv&#039;, &#039;/media/company/users.html&#039;)

#########################################################

#Add caption  to HTML files

#################################################################

def html_caption(html_file, caption):
    append_copy = open(html_file, &quot;r&quot;)
    original_text = append_copy.read()
    append_copy.close()
    append_copy = open(html_file, &quot;w&quot;)
    append_copy.write(&quot;
<span id="mce_SELREST_end" style="overflow:hidden;line-height:0;"></span>\n<b>{caption}</b>

\n
\n".format(caption=caption))
    append_copy.write(original_text)
    append_copy.close()


html_caption('/media/company/groups.html', 'IAM Groups')

html_caption('/media/company/group_policy.html', 'IAM Group policy')

html_caption('/media/company/roles_assign.html', 'IAM Role Assignment')

html_caption('/media/company/roles.html', 'IAM Roles')

html_caption('/media/company/user_policy.html', 'IAM User policies')

################################################################################################

#if changes.txt file is not empty, convert it to HTML

###################################################################################################

with open('/media/company/changes.txt') as friendsfile:
    first = friendsfile.read(1)
    if not first:
        print('no changes')

    else:

        with open('/media/company/changes.txt') as fin, open('/media/company/change.txt', 'w') as fout:
           for line in fin:
               fout.write(line.replace(',', ''))
        contents = open("/media/company/changes.txt","r")
        with open("/media/company/changes.html", "w") as e:
             for lines in contents.readlines():
                 e.write("</pre>
<pre>" + lines + "</pre>
<pre>\n")
with open('/media/company/changes.txt') as friendsfile:
 first = friendsfile.read(1)
 if not first:
  #print ('file is empty)
  file_list = ['/media/company/no_changes.html','/media/company/users.html','/media/company/user_policy.html','/media/company/groups.html','/media/company/group_policy.html', '/media/company/roles.html', '/media/company/roles_assign.html']
  data = read_files(file_list)
  header = {'To': html_body_recipient, 'Subject': ' AWS IAM Reports' , 'From': 'svc@company.com'}
  send_html_body(header, data)
  time.sleep(10)
else:
  file_list = ['/media/company/changes.html', '/media/company/users.html',/media/company/user_policy.html','/media/company/groups.html','/media/company/group_policy.html', '/media/company/roles.html', '/media/company/roles_assign.html']
  data = read_files(file_list) header = {'To': html_body_recipient, 'Subject': ' AWS IAM Reports' , 'From': 'svc@company.com'} send_html_body(header, data)
  time.sleep(10)
#Create JIRA ticket if role assignment report has changes
with open('/media/company/company_role_assignemnt_changes.txt') as friendsfile: first = friendsfile.read(1)
if not first:
 print ('roleassignment is empty')
else:
 send_email()
 time.sleep(10)

requirements.txt:

requests
python-dateutil
json_tricks
boto3
pymongo
pandas
awscli

Dockerfile

FROM ubuntu:latest
WORKDIR /home
COPY . .

RUN echo "postfix postfix/mailname string rundeck.company.com" | debconf-set-selections && echo "postfix postfix/main_mailer_type string 'Internet Site'" | debconf-set-selections && apt-get update -y && apt-get install postfix sasl2-bin mailutils vim python3-pip -y && pip3 install --no-cache-dir -r requirements.txt && sed -i s/START=no/START=yes/g /etc/default/saslauthd && echo "[smtp.office365.com]:587 svc@company.com:pass" > /etc/postfix/sasl_passwd && echo "/.+/ svc@company.com" > /etc/postfix/sender_canonical && sed -i 's/inet_protocols = all/inet_protocols = ipv4/g' /etc/postfix/main.cf && sed -i 's/relayhost = /relayhost = [smtp.office365.com]:587/g' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_sasl_auth_enable = yes' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_sasl_security_options = noanonymous' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_tls_security_level = may' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a sender_canonical_maps = regexp:/etc/postfix/sender_canonical' /etc/postfix/main.cf && sed -i '/smtpd_use_tls=yes/a smtp_tls_CAfile = /etc/postfix/cacert.pem' /etc/postfix/main.cf &&  mv /home/cacert.pem /etc/postfix/ && postmap hash:/etc/postfix/sasl_passwd && postmap hash:/etc/postfix/sender_canonical

ENTRYPOINT ["./start.py"]

Finally, we’ll map host folder to docker container so we can keep track of yesterday’s reports

docker run -it -v /home/centos/docker:/media test /bin/bash

Advertisements

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out /  Change )

Google photo

You are commenting using your Google account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

Connecting to %s