Archive for the ‘AWS’ Category

Let’s say we have this AWS CloudWatch event

2.PNG

And we have tagged EC2 instance with AutoStopSchedule tag, values between 1-5

1.PNG

Following Lambda function will get cron expression from CloudWatch event and will dynamically filter which instances should be turned off

import boto3
import logging

# define rule name
rule_name = "stop_ec2"

#setup simple logging for INFO
logger = logging.getLogger()
logger.setLevel(logging.INFO)

#define the connection
ec2 = boto3.resource('ec2')

# connect to Clouwatch events
client = boto3.client('events')

def lambda_handler(event, context):
   # get cron expression for Specific CloudWatch rule
   response = client.describe_rule(Name=rule_name)
   expression = response['ScheduleExpression']

   # based on current expression create filter variable and populate it with value in range 1-5
   if "cron(20 * * * ? *)" in expression:
      filter = "1"
   elif "cron(0 */1 * * ? *)" in expression:
      filter = "2"
   elif "cron(0 */6 * * ? *)" in expression:
      filter = "3"
   elif "cron(0 */12 * * ? *)" in expression:
      filter = "4"
   elif "cron(0 10 * * ? *)" in expression:
      filter = "5"
   else:
      filter = "0"

   # Use the filter() method of the instances collection to retrieve
    # all running EC2 instances.
   filters = [

        {
            'Name': 'tag:AutoStopSchedule',
            'Values': [filter]
        },
        {
            'Name': 'instance-state-name',
            'Values': ['running']
        }
       ]
    #filter the instances
    #ec2 = boto3.client('ec2', region_name=region)
   instances = ec2.instances.filter(Filters=filters)

    #locate all running instances
   RunningInstances = [instance.id for instance in instances]

    #print the instances for logging purposes
   print (RunningInstances) 

    #make sure there are actually instances to shut down.
   if len(RunningInstances) > 0:
        #perform the shutdown
        shuttingDown = ec2.instances.filter(InstanceIds=RunningInstances).stop()
        #print shuttingDown
   else:
    print "Nothing to see here"

Make sure IAM policy has following

 
{"Action": [

"events:DescribeRule"
],
"Effect": "Allow",
"Resource": "*"
}
Advertisements
import boto3
ec2 = boto3.resource('ec2',region_name='eu-west-1')
def lambda_handler(event, context):
     for vol in ec2.volumes.all():
      if vol.state=='available':
        for tag in vol.tags:
         if tag in vol.tags or tag==None:
           print "VolumeID:"+ vol.id, "Volume Name:"+tag['Value'] + " AWS Region:Ireland"

 

 

This Python boto3 script will get notification if instances are stopped for more than 30 days

import json
import boto3
import re
import smtplib
import datetime
import time
from datetime import datetime
from datetime import date
from dateutil.relativedelta
import relativedeltainstance_ids = []
instance_names = []
stopped_reasons = []
output=[]
transition_timestamps=[]
ses = boto3.client('ses')
def send_mail(email_from, email_to, subject, body):
smtp_address = 'smtp.office365.com'
provider_username = 'sender@example.com'
provider_password = 'Pass'
smtpserver = smtplib.SMTP(smtp_address, 587)
smtpserver.ehlo()
smtpserver.starttls()
smtpserver.ehlo() # extra characters to permit edit
smtpserver.login(provider_username, provider_password)
header = 'To: ' + email_to + '\n' + 'From: ' + email_from + '\n' + 'Subject: ' + subject + '\n'
msg = header + '\n ' + body + ' \n\n'
smtpserver.sendmail(provider_username, email_to, msg)
smtpserver.quit()
def lambda_handler(event, context):
 client = boto3.client('ec2',region_name='eu-west-2')
 reservations = client.describe_instances().get('Reservations', [])
 for reservation in reservations:
    for instance in reservation['Instances']:
      tags = {}
      for tag in instance['Tags']:
            tags[tag['Key']] = tag['Value']
            if tag['Key'] == 'Name':
                name=tag['Value']
    if instance['State']['Name'] == 'stopped':
       instance_ids.append(instance['InstanceId'])
       instance_names.append(name)
       stopped_reason = instance['StateTransitionReason']
       stopped_reasons.append(stopped_reason)
       transition_timestamp = datetime.strptime(instance['StateTransitionReason'][16:39], '%Y-%m-%d %H:%M:%S %Z')
       transition_timestamps.append(str(transition_timestamp))
       days=(datetime.now() - transition_timestamp).days
       if days > 30:
          a = "InstanceID:" + instance['InstanceId'] + "," + ' Instance Name:' +name + "," + " Shutdown Time: "+str(transition_timestamp)
          output.append(a)
body= "\n\n".join(output)
if body:
emailbody = "The following instances are stopped for more than 30 days in London region\n\n\n\n" +body + "\n\n\n If instances are not needed anymore please terminate it or start it otherwise"
send_mail('sender@example.com', 'recipient@example.com', 'Notification of stopped instances', emailbody)

This code will list all EC2 instances for every region and if termination protection is enabled, it will be disabled

import json
import boto3
def lambda_handler(event, context):
  client = boto3.client('ec2')
  ec2_regions = [region['RegionName'] for region in client.describe_regions()['Regions']]
  for region in ec2_regions:
     client = boto3.client('ec2', region_name=region)
     conn = boto3.resource('ec2',region_name=region)
     instances = conn.instances.filter()
     for instance in instances:
       if instance.state["Name"] == "running":
       #print instance.id # , instance.instance_type, region)
        terminate_protection=client.describe_instance_attribute(InstanceId =instance.id,Attribute = 'disableApiTermination')
        protection_value=(terminate_protection['DisableApiTermination']['Value'])
        if protection_value == True:
          client.modify_instance_attribute(InstanceId=instance.id,Attribute="disableApiTermination",Value= "False" )

On Amazon side:

Create new elastic IP

Select Virtual Private Cloud-Elastic IPs-Allocate new address

1.PNG

Click allocate

2.PNG

I used default VPC if you need to create new VPC,take a look here

Create EC2 instance and assign VPC (default or custom one and subnet)

8.png

Allocate Elastic IP to instance-in EC2 select instance-Actions-Associate address

9.png

Resource Type-instance-select instance and  Private IP

10.PNG

Azure portal

Create Virtual Network Gateway (details here)

Create Local Network Gateway

3.png

IP Address:Amazon Elastic IP (created earlier)

Address Space (Amazon VPC subnet to which EC2 instance is assigned)

4.png

5.PNG

Once Local network gateway is created go to Connections-Add

6.PNG

Select Virtual Network gateway,local network gateway and shared key

7.PNG

Copy Virtual network gateway IP

11

find out Azure VM network

Click on Azure VM-Networking to find out subnet name

12.png

write down subnet, it will be needed for Powershell script

13.png

On AWS EC2 instance install RRAS and configure IPSec VPN.In this case 137.117.170.80 is Azure Virtual Network Gateway IP,10.0.1.0/24 Azure VM subnet and 123456 Secret Key

# Windows Azure Virtual Network

# This configuration template applies to Microsoft RRAS running on Windows Server 2012 R2.
# It configures an IPSec VPN tunnel connecting your on-premise VPN device with the Azure gateway.

# !!! Please notice that we have the following restrictions in our support for RRAS:
# !!! 1. Only IKEv2 is currently supported
# !!! 2. Only route-based VPN configuration is supported.
# !!! 3. Admin priveleges are required in order to run this script

Function Invoke-WindowsApi(
[string] $dllName,
[Type] $returnType,
[string] $methodName,
[Type[]] $parameterTypes,
[Object[]] $parameters
)
{
## Begin to build the dynamic assembly
$domain = [AppDomain]::CurrentDomain
$name = New-Object Reflection.AssemblyName 'PInvokeAssembly'
$assembly = $domain.DefineDynamicAssembly($name, 'Run')
$module = $assembly.DefineDynamicModule('PInvokeModule')
$type = $module.DefineType('PInvokeType', "Public,BeforeFieldInit")

$inputParameters = @()

for($counter = 1; $counter -le $parameterTypes.Length; $counter++)
{
$inputParameters += $parameters[$counter - 1]
}

$method = $type.DefineMethod($methodName, 'Public,HideBySig,Static,PinvokeImpl',$returnType, $parameterTypes)

## Apply the P/Invoke constructor
$ctor = [Runtime.InteropServices.DllImportAttribute].GetConstructor([string])
$attr = New-Object Reflection.Emit.CustomAttributeBuilder $ctor, $dllName
$method.SetCustomAttribute($attr)

## Create the temporary type, and invoke the method.
$realType = $type.CreateType()

$ret = $realType.InvokeMember($methodName, 'Public,Static,InvokeMethod', $null, $null, $inputParameters)

return $ret
}

Function Set-PrivateProfileString(
$file,
$category,
$key,
$value)
{
## Prepare the parameter types and parameter values for the Invoke-WindowsApi script
$parameterTypes = [string], [string], [string], [string]
$parameters = [string] $category, [string] $key, [string] $value, [string] $file

## Invoke the API
[void] (Invoke-WindowsApi "kernel32.dll" ([UInt32]) "WritePrivateProfileString" $parameterTypes $parameters)
}

# Install RRAS role
Import-Module ServerManager
Install-WindowsFeature RemoteAccess -IncludeManagementTools
Add-WindowsFeature -name Routing -IncludeManagementTools

# !!! NOTE: A reboot of the machine might be required here after which the script can be executed again.

# Install S2S VPN
Import-Module RemoteAccess
if ((Get-RemoteAccess).VpnS2SStatus -ne "Installed")
{
Install-RemoteAccess -VpnType VpnS2S
}

# Add and configure S2S VPN interface
Add-VpnS2SInterface -Protocol IKEv2 -AuthenticationMethod PSKOnly -NumberOfTries 3 -ResponderAuthenticationMethod PSKOnly -Name 137.117.170.80 -Destination 137.117.170.80 -IPv4Subnet @("10.0.1.0/24:100") -SharedSecret 123456

Set-VpnServerIPsecConfiguration -EncryptionType MaximumEncryption

Set-VpnS2Sinterface -Name 137.117.170.80 -InitiateConfigPayload $false -Force

# Set S2S VPN connection to be persistent by editing the router.pbk file (required admin priveleges)
Set-PrivateProfileString $env:windir\System32\ras\router.pbk "137.117.170.80 " "IdleDisconnectSeconds" "0"
Set-PrivateProfileString $env:windir\System32\ras\router.pbk "137.117.170.80 " "RedialOnLinkFailure" "1"

# Restart the RRAS service
Restart-Service RemoteAccess

# Dial-in to Azure gateway
Connect-VpnS2SInterface -Name 137.117.170.80

Test connection

get-VpnS2Sinterface

14.PNG

Connection from EC2 to Azure

 

15.PNG

 

 

 

Amazon EC2-Changing instance type

Posted: April 24, 2018 in AWS, Linux

If we face hardware limitations of our EC2 instance, we can’t just increase Memory/CPU cores as in VMWare, instead we must change instance type.It’s set of predefined images with different hardware specifications.(More info here)

First,stop EC2 instance:

 

1

 

Then, from Action menu,select Instance Settings-Change Instance Type

 

2

 

Select instance type and click Apply

 

3

Now start instance. (Note that new public IP is assigned.The instance retains its private IPv4 addresses

In previous article we created federation trust between Azure and AWS by creating Amazon user and used it’s credentials to create trust between Azure and AWS (automatic provisioning).This method has 2 main drawbacks: it takes a long time for Azure to retrive all IAM roles,and it’s not possible to provide more than 1 IAM credentials (situation when need to federate same Azure Enterprise application with 2 or more AWS accounts).Most of the steps are same as for manual provisioning but i’ll put it here again for the sake of transparency

Adding Amazon Application to Azure portal

On Azure portal Azure Active Directory-Enterprise Applications-All applications-New Application

1

In search box type Amazone-select Amazon Web Services (AWS)

2

On AWS app properties click on Single sign-on

3

Click Add attribute

4

Add attributes as in picture below

Attribute name Attribute value Namespace
RoleSessionName user.userprincipalname https://aws.amazon.com/SAML/Attributes
Role user.assignedroles https://aws.amazon.com/SAML/Attributes

 

 

5

In the SAML Signing Certificate section, select Metadata XML. Then save the metadata file on your computer.

5-1

5-2

Then click Save

5-3

AWS Console:Creating Provider and IAM role

In AWS console we need to add Provider, IAM role and policy

Select Identity and access management-IAM

6

Identity Providers-Create Provider

7

Choose SAML as Provider Type,set name and browse for metadata file downloaded from Azure portal

9

10

Still in IAM Click Roles-Create Role

11

Select Saml 2.0 Federation-SAML provider-provider we created earlier-Allow programatic and AWS Management Console Access (Attribute and Value fields populate automatically)

12

In Attach permission policies click Next:Review

13

In Create Role create as many roles as you need

14

Besides Azure_Role, i created another one and attached one IAM policy, we’ll map this role to another Azure AD Group

Untitled

Azure portal:Create User and Group-add user to group

In this section we’ll map Azure AD group to AWS role we just created (Azure_Role)

Creating new user:

Azure Active Directory-Users-All Users:

26

Create user

27

Creating AD group

Azure Active Directory-Groups

Untitled

Specify Group Type,name-Membership Type:Assigned-specify user(s) to add to group-Select-Create

Untitled

In the same way i created another Azure AD group (AWS_Second_Test_Group) to map it with another IAM role we created earlier (AWS_Second_Test_Role), i added Don.Hall user to this group too

Editing Azure Active directory manifest file

Manifest file is a JSON file that represents application’s identity configuration.We’ll edit this file to map Azure AD group with AWS IAM Role.Access scopes and roles are exposed through this file

In Azure portal, in search box type App Registrations-Select Amazon Web Services (AWS)

Untitled.png

Click on Manifest

Untitled

Now, we’ll map IAM AWS roles to Azure AD groups:

IAM Role name Azure AD Group Name
 Azure_Role Azure_AD_Group
AWS_Second_Test_Role AWS_Second_Test_Group

Ideally, names of IAM Roles and groups should be the same to avoid confusion

In order to edit manifest file we need to obtanin IAM Role ARN,AWS Identity provider ARN and Azure AD group ID (Azure AD Group ID must be unique-as a rule of thumb i just changed last 2 digits)

Capture

AWS IAM role ARN:

Untitled.png

Untitled.png

AWS Identity provider ARN:

Untitled

Azure Group’s ID:

Click on group-Properties:

Capture

Untitled

Remember, Azure AD group ID’s needs to be unique, so change last digit(s) values

These 2 sections are added to manifest file:

displayName:Name of Azure AD group

id:id of Azure AD group (changes last 2 digits-needs to be unique)

value:AWS IAM role ARN,AWS identity provider ARN

"appRoles": [
    {
      "allowedMemberTypes": [
        "User"
      ],
      "displayName": "AWS_Second_Test_Group",
      "id": "faa9acbc-49db-4a04-9a66-2050998f1c15",
      "isEnabled": true,
      "description": "Azure AD Second group",
      "value": "arn:aws:iam::233135199200:role/AWS_Second_Test_Role,arn:aws:iam::233135199200:saml-provider/WindowsAD"
    },
    {
      "allowedMemberTypes": [
        "User"
      ],
      "displayName": "Azure_AD_Group",
      "id": "b40569c7-ebf0-4c32-959c-b0b3b1cbfc12",
      "isEnabled": true,
      "description": "Azure AD First group",
      "value": "arn:aws:iam::233135199200:role/Azure_Role,arn:aws:iam::233135199200:saml-provider/WindowsAD"
    },

If we need to map more roles to groups we just need to add allowedMemberTypes sections (separate each one with comma)

Here is complete manifest file:

{
  "appId": "1def2fa6-5467-4565-b3f0-e598b3007b42",
  "appRoles": [
    {
      "allowedMemberTypes": [
        "User"
      ],
      "displayName": "AWS_Second_Test_Group",
      "id": "faa9acbc-49db-4a04-9a66-2050998f1c15",
      "isEnabled": true,
      "description": "Azure AD Second group",
      "value": "arn:aws:iam::233135199200:role/AWS_Second_Test_Role,arn:aws:iam::233135199200:saml-provider/WindowsAD"
    },
    {
      "allowedMemberTypes": [
        "User"
      ],
      "displayName": "Azure_AD_Group",
      "id": "b40569c7-ebf0-4c32-959c-b0b3b1cbfc12",
      "isEnabled": true,
      "description": "Azure AD First group",
      "value": "arn:aws:iam::233135199200:role/Azure_Role,arn:aws:iam::233135199200:saml-provider/WindowsAD"
    },
    {
      "allowedMemberTypes": [
        "User"
      ],
      "displayName": "msiam_access",
      "id": "7dfd756e-8c27-4472-b2b7-38c17fc5de5e",
      "isEnabled": true,
      "description": "msiam_access",
      "value": null
    }
  ],
  "availableToOtherTenants": false,
  "displayName": "Amazon Web Services (AWS)",
  "errorUrl": null,
  "groupMembershipClaims": null,
  "optionalClaims": null,
  "acceptMappedClaims": null,
  "homepage": "https://signin.aws.amazon.com/saml?metadata=aws|ISV9.1|primary|z",
  "informationalUrls": {
    "privacy": null,
    "termsOfService": null
  },
  "identifierUris": [
    "http://awsDC46DF5ECB354EEA858E81622348A0BE",
    "http://instanceid_8b1025e4-1dd2-430b-a150-2ef79cd700f5_EAAEA402D2364790A14A5099A13A3B7E",
    "http://aws/d38c1eb9-ca01-420f-a982-210c0583dc49"
  ],
  "keyCredentials": [],
  "knownClientApplications": [],
  "logoutUrl": null,
  "oauth2AllowImplicitFlow": false,
  "oauth2AllowUrlPathMatching": false,
  "oauth2Permissions": [
    {
      "adminConsentDescription": "Allow the application to access Amazon Web Services (AWS) on behalf of the signed-in user.",
      "adminConsentDisplayName": "Access Amazon Web Services (AWS)",
      "id": "e81ccfaa-9095-4cbc-87fe-10538a57f314",
      "isEnabled": true,
      "type": "User",
      "userConsentDescription": "Allow the application to access Amazon Web Services (AWS) on your behalf.",
      "userConsentDisplayName": "Access Amazon Web Services (AWS)",
      "value": "user_impersonation"
    }
  ],
  "oauth2RequirePostResponse": false,
  "objectId": "dd1dc07d-87dc-48bb-9fd3-1c0274c789a5",
  "parentalControlSettings": {
    "countriesBlockedForMinors": [],
    "legalAgeGroupRule": "Allow"
  },
  "passwordCredentials": [],
  "publicClient": false,
  "replyUrls": [
    "https://signin.aws.amazon.com/saml"
  ],
  "requiredResourceAccess": [],
  "samlMetadataUrl": null
}

Azure Active Directory-Enterprise Applications-Amazon Web Services (AWS)-Users and Groups-Add User

Capture.PNG

In Users section Assign user- in Roles section new roles should appear-select role

Capture

And assign it

Capture

Capture

Make sure manual provision method is selected (Amazon Web Services (AWS)-Provisioning

Capture

Testing access to AWS console

Don.Hall should be able now to acces to Amazon Web Console

Go to http://myapps.microsoft.com, log in as Don.Hall

click on Amazon Web Service, you should be able to sign in automatically to AWS console

31

 

Capture