+{+	"agent": {+		"metrics_collection_interval": 30,+		"run_as_user": "root"+	},+	"metrics": {+		"append_dimensions": {+			"AutoScalingGroupName": "${aws:AutoScalingGroupName}",+			"InstanceId": "${aws:InstanceId}",+			"InstanceType": "${aws:InstanceType}"

+{+	"agent": {+		"metrics_collection_interval": 30,+		"run_as_user": "root"

 for file in *; do 		chmod u+x "$file" 	fi done++wget https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm

+{

 aemm_helm_args=(   "$AEMM_DL_URL"   --namespace default   --set servicePort="$IMDS_PORT"+  --set 'rbac.pspEnabled=true'

 type combinedDrainData struct {  // Post makes a http post to send drain event data to webhook url func Post(additionalInfo ec2metadata.NodeMetadata, event *monitor.InterruptionEvent, nthconfig config.Config) {+	var webhookTemplateContent string++	if nthconfig.WebhookTemplateFile != "" {+		content, err := ioutil.ReadFile(nthconfig.WebhookTemplateFile)+		if err != nil {+			log.Log().Msgf("Webhook Error: Could not read template file %s - %s", nthconfig.WebhookTemplateFile, err)+			return+		}+		webhookTemplateContent = string(content)+		log.Log().Msgf("Template file content - %s", webhookTemplateContent)

+# AWS Node Termination Handler & Amazon EC2 Metadata Mock++We have open sourced a tool called the [amazon-ec2-metadata-mock](https://github.com/aws/amazon-ec2-metadata-mock) (AEMM)+that simulates spot interruption notices and more by starting a real webserver that serves data similar to EC2 Instance+Metadata Service. The tool is easily deployed to kubernetes with a Helm chart.++Below is a short guide on how to set AEMM up with your Node Termination Handler cluster in case you'd like to verify the+behavior yourself.++## Triggering AWS Node Termination Handler with Amazon EC2 Metadata Mock++Start by installing AEMM on your cluster. For full and up to date installation instructions reference the AEMM repository.+Here's just one way to do it.++Download the latest tar ball from the releases page, at the time of writing this that was v1.6.0. Then install it using+Helm:+```+helm install amazon-ec2-metadata-mock amazon-ec2-metadata-mock-1.6.0.tgz \+  --namespace default+```++Once AEMM is installed, you need to change the instance metadata url of Node Termination Handler to point+to the location AEMM is serving from. If you use the default values of AEMM, the installation will look similar to this:+```+helm upgrade --install aws-node-termination-handler \+  --namespace kube-system \+  --set instanceMetadataURL="http://amazon-ec2-metadata-mock-service.default.svc.cluster.local:1338" \+  eks/aws-node-termination-handler+```++That's it! Instead of polling the real IMDS service endpoint, AWS Node Termination Handler will instead poll AEMM.+If you open the logs of an AWS Node Termination Handler pod you should see that it receives (mock) interruption+events from AEMM and that the nodes are cordoned and drained. Keep in mind that these nodes won't actually get terminated,+so you might need to manually uncordon the nodes if you want to reset your test cluster.++### AEMM Advanced Configuration+If you run the example above you might notice that the logs are heavily populated. Here's an example output:+```+2020/09/15 21:13:41 Sending interruption event to the interruption channel+2020/09/15 21:13:41 Got interruption event from channel {InstanceID:i-1234567890abcdef0 InstanceType:m4.xlarge PublicHostname:ec2-192-0-2-54.compute-1.amazonaws.com PublicIP:192.0.2.54 LocalHostname:ip-172-16-34-43.ec2.internal LocalIP:172.16.34.43 AvailabilityZone:us-east-1a} {EventID:spot-itn-47ddfb5e39791606bec3e91fea4cdfa86f86a60ddaf014c8b4af8e008f134b19 Kind:SPOT_ITN Description:Spot ITN received. Instance will be interrupted at 2020-09-15T21:15:41Z+ State: NodeName:ip-192-168-123-456.us-east-1.compute.internal StartTime:2020-09-15 21:15:41 +0000 UTC EndTime:0001-01-01 00:00:00 +0000 UTC Drained:false PreDrainTask:0x113c8a0 PostDrainTask:<nil>}+WARNING: ignoring DaemonSet-managed Pods: default/amazon-ec2-metadata-mock-pszj2, kube-system/aws-node-bl2bj, kube-system/aws-node-termination-handler-2pvjr, kube-system/kube-proxy-fct9f+evicting pod "coredns-67bfd975c5-rgkh7"+evicting pod "coredns-67bfd975c5-6g88n"+2020/09/15 21:13:42 Node "ip-192-168-123-456.us-east-1.compute.internal" successfully cordoned and drained.+2020/09/15 21:13:43 Sending interruption event to the interruption channel+2020/09/15 21:13:43 Got interruption event from channel {InstanceID:i-1234567890abcdef0 InstanceType:m4.xlarge PublicHostname:ec2-192-0-2-54.compute-1.amazonaws.com PublicIP:192.0.2.54 LocalHostname:ip-172-16-34-43.ec2.internal LocalIP:172.16.34.43 AvailabilityZone:us-east-1a} {EventID:spot-itn-97be476b6246aba6401ba36e54437719bfdf987773e9c83fe30336eb7fea9704 Kind:SPOT_ITN Description:Spot ITN received. Instance will be interrupted at 2020-09-15T21:15:43Z+ State: NodeName:ip-192-168-123-456.us-east-1.compute.internal StartTime:2020-09-15 21:15:43 +0000 UTC EndTime:0001-01-01 00:00:00 +0000 UTC Drained:false PreDrainTask:0x113c8a0 PostDrainTask:<nil>}+WARNING: ignoring DaemonSet-managed Pods: default/amazon-ec2-metadata-mock-pszj2, kube-system/aws-node-bl2bj, kube-system/aws-node-termination-handler-2pvjr, kube-system/kube-proxy-fct9f+2020/09/15 21:13:44 Node "ip-192-168-123-456.us-east-1.compute.internal" successfully cordoned and drained.+2020/09/15 21:13:45 Sending interruption event to the interruption channel+2020/09/15 21:13:45 Got interruption event from channel...+```++This isn't a mistake, by default AEMM will respond to any request for metadata with a spot interruption occurring 2 minutes+later than the request time.\* AWS Node Termination Handler polls for events every 2 seconds by default, so the effect is+that new interruption events are found and processed every 2 seconds. ++In reality there will only be a single interruption event, and you can mock this by setting the `spot.time` parameter of+AEMM when installing it. +```+helm install amazon-ec2-metadata-mock amazon-ec2-metadata-mock-1.6.0.tgz \+  --set aemm.spot.time="2020-09-09T22:40:47Z" \+  --namespace default+```++Now when you check the logs you should only see a single event get processed. ++For more ways of configuring AEMM check out the [Helm configuration page](https://github.com/aws/amazon-ec2-metadata-mock/tree/master/helm/amazon-ec2-metadata-mock).++## Node Termination Handler E2E Tests++AEMM started out as a test server for aws-node-termination-handler's end-to-end tests in this repo. We use AEMM throughout+our end to end tests to create interruption notices.++The e2e tests install aws-node-termination-handler using Helm and set the metadata url [here](https://github.com/aws/aws-node-termination-handler/blob/master/test/e2e/spot-interruption-test#L36).+This becomes where aws-node-termination-handler looks for metadata; other applications on the node still look at the real+EC2 metadata service.++We set the metadata url environment variable [here](https://github.com/aws/aws-node-termination-handler/blob/master/test/k8s-local-cluster-test/run-test#L18)+for the local tests that use a kind cluster, and [here](https://github.com/aws/aws-node-termination-handler/blob/master/test/eks-cluster-test/run-test#L117)+for the eks-cluster e2e tests.++Check out the [ReadMe](https://github.com/aws/aws-node-termination-handler/tree/master/test) in our test folder for more+info on the e2e tests. ++---++\* Only the first two unique IPs to request data from AEMM receive spot itn information in the response. This was introduced+in AEMM v1.6.0. This can be overridden with a configuration paramter. For previous versions there is no unique IP restriction. 

 go 1.14  require ( 	github.com/aws/aws-sdk-go v1.31.5+	github.com/mitchellh/go-homedir v1.1.0

 Parameter | Description | Default in Helm | Default AEMM configuration `aemm.server.hostname` | hostname to run AEMM on | `""`, in order to listen on all available interfaces e.g. ClusterIP | `0.0.0.0` `aemm.mockDelaySec` | mock delay in seconds, relative to the start time of AEMM | `0` | `0` `aemm.mockTriggerTime` | mock trigger time in RFC3339 format | `""` | `""`+`aemm.mockIPCount` | number of IPs that can receive spot interrupts and/or scheduled events; subsequent requests will return 404 | `""` | `99`

 var ( 		gf.ConfigFileFlag:       cfg.GetDefaultCfgFileName(), 		gf.MockDelayInSecFlag:   0, 		gf.MockTriggerTimeFlag:  "",+		gf.MockIPCountFlag:      99,

+#! /usr/bin/env bash++set -euo pipefail++SCRIPTPATH="$(+  cd "$(dirname "$0")"+  pwd -P+)"+EXIT_CODE_TO_RETURN=0++function get_status() {+  pod=$1+  status=$(kubectl describe pod $pod | grep "Status:")+  status=${status//"Status:"/}+  status=$(echo $status | xargs)+  echo $status+}++function assert_value() {+  # assert actual == expected+  if [[ $1 == "$2" ]]; then+    echo "✅ Verified $3"+  else+    echo "❌ Failed $3 verification. Actual: $1 Expected: $2"+    EXIT_CODE_TO_RETURN=1+  fi+}++function clean_up() {+  kubectl delete pods/test-pod pods/test-pod-404+}++function test() {+  num_nodes=$1+  expected_test_pod_status=$2+  expected_test_pod_404_status=$3+  echo "executing term-nodes test with terminationNodes=$num_nodes"++  helm upgrade --install "$CLUSTER_NAME-aemm" \+    $AEMM_HELM_REPO \+    --wait \+    --namespace default \+    --values $AEMM_HELM_REPO/ci/local-image-values.yaml \+    --set aemm.spot.time="1994-05-15T00:00:00Z" \+    --set aemm.terminationNodes=$num_nodes++    # Deploy pods+    kubectl apply -f "$SCRIPTPATH/test-pod.yaml"+    sleep 1+    kubectl apply -f "$SCRIPTPATH/test-pod-404.yaml"++    # Proceed with copying only after pod is Running+    test_pod_404_status=$(get_status test-pod-404)+    while [ "$test_pod_404_status" != "Running" ]; do+      echo "test_pod_404_status: $test_pod_404_status"+      sleep 1+      test_pod_404_status=$(get_status test-pod-404)+    done++    echo "test_pod_404_status: $test_pod_404_status"+    echo "copying to 404-pod..."+    kubectl cp "$SCRIPTPATH/../../e2e/golden/404_response.golden" test-pod-404:/tmp/404_response.golden++    test_pod_status=$(get_status test-pod)+    test_pod_404_status=$(get_status test-pod-404)++    # Keep querying status until tests succeed or fail+    while [ "$test_pod_status" == "Running" ] || [ "$test_pod_404_status" == "Running" ]; do

 get_chart_test_config() {     echo "$config" } +test_termination_nodes() {+    if [[ $REUSE_ENV == false ]]; then+        mkdir -p $TMP_DIR+        install_kind+        create_kind_cluster+    fi++    build_and_load_image+    install_helm

 var ( 		gf.ConfigFileFlag:       cfg.GetDefaultCfgFileName(), 		gf.MockDelayInSecFlag:   0, 		gf.MockTriggerTimeFlag:  "",+		gf.TerminationNodesFlag: 99,

 Parameter | Description | Default in Helm | Default AEMM configuration `aemm.server.hostname` | hostname to run AEMM on | `""`, in order to listen on all available interfaces e.g. ClusterIP | `0.0.0.0` `aemm.mockDelaySec` | mock delay in seconds, relative to the start time of AEMM | `0` | `0` `aemm.mockTriggerTime` | mock trigger time in RFC3339 format | `""` | `""`+`aemm.terminationNodes` | number of nodes that can receive spot interrupts; subsequent requests will return 404 | `""` | `99`

 image:   tag: "v1.5.0"   pullPolicy: "IfNotPresent" +replicaCount: 1+ # nameOverride overrides the name of the helm chart nameOverride: "" # fullnameOverride overrides the name of the application fullnameOverride: "" -# Create node OS specific daemonset(s). (e.g. "linux", "windows", "linux windows")+# Create node OS specific deployment(s). (e.g. "linux", "windows", "linux windows") targetNodeOs: "linux"

 Flags:       --mock-trigger-time string   mock trigger time in RFC3339 format. This takes priority over mock-delay-sec (default: none)   -p, --port string                the HTTP port where the mock runs (default: 1338)   -s, --save-config-to-file        whether to save processed config from all input sources in .ec2-metadata-mock/.aemm-config-used.json in $HOME or working dir, if homedir is not found (default: false)+  -x, --termination-nodes int      number of nodes in a cluster that can receive Spot interrupt notice (default: 99, meaning unlimited nodes) (default 99)

 BUCKET_ROOT_DIR= TARGET_UTIL=0 

 ALL_INSTANCE_TYPES=$SUPPORTED_INSTANCE_TYPES,$UNSUPPORTED_INSTANCE_TYPE_AMI,$UNS  function init_test_resources() {   # create resources used for e2e tests-  AWS_DEFAULT_REGION=$DEFAULT_REGION

+#!/usr/bin/env bash++set -euo pipefail++SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"+BUILD_DIR="${SCRIPTPATH}/../../build"++KERNEL=$(uname -s | tr '[:upper:]' '[:lower:]')+SHELLCHECK_VERSION="0.7.1"++function exit_and_fail() {+   echo "❌ Test Failed! Found a shell script with errors."+   exit 1+}+trap exit_and_fail INT ERR TERM++curl -Lo ${BUILD_DIR}/shellcheck.tar.xz "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/shellcheck-v${SHELLCHECK_VERSION}.${KERNEL}.x86_64.tar.xz"+tar -C ${BUILD_DIR} -xvf "${BUILD_DIR}/shellcheck.tar.xz"+export PATH="${BUILD_DIR}/shellcheck-v${SHELLCHECK_VERSION}:$PATH"++# setup.go, .template and binaries have embedded user scripts; therefore, omit from grep+# shellcheck disable=SC2207+shell_files=($(grep -RInl --exclude=\*.{go,template} -e '#!.*/bin/bash' -e '#!.*/usr/bin/env bash' ${SCRIPTPATH}/../../))+shellcheck -S warning "${shell_files[@]}"

 BUCKET_ROOT_DIR=%s TARGET_UTIL=%d  adduser qualifier-cd /home/qualifier+cd /home/qualifier || :