deploy_aws_environment

#!/bin/bash

set -e

SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)

function usage() {
  >&2 cat <<EOF
  SYNOPSIS:
    Create a CloudFormation stack, deploy BOSH, and create initial CF and Diego stubs.

  USAGE:
    $0 <AWS_ACTION> <BOSH_ACTION> <BOSH_DEPLOYMENT_DIR> <DEPLOYMENT_DIR> <STACK_NAME>

  MANDATORY ARGUMENTS:
    AWS_ACTION:           Action to take on AWS CloudFormation stack: create-stack, update-stack, or skip-stack.
    BOSH_ACTION:          Action to take on BOSH director deployment: deploy-bosh or skip-bosh.
    BOSH_DEPLOYMENT_DIR:  Path to local copy of the bosh-deployment repository.
    DEPLOYMENT_DIR:       Directory to store files for this deployment.
    STACK_NAME:           Name of the CloudFormation stack to create.
EOF
}


indent() {
  sed -e 's/^/  /'
}

indent_contents_of() {
  indent < "$1"
}

block() {
  cat <<-EOF
$1: |
$(indent_contents_of "$2")
EOF
}

oneline() {
  cat <<-EOF
$1: '$(cat $2)'
EOF
}

cf_credentials() {
  cat <<-EOF
# GENERATED: NO TOUCHING
---
cf_credentials:
  ssh_host_key_fingerprint: "$(cat keypair/ssh-proxy-host-key-fingerprint)"
  cc:
$(block ca_cert      certs/cf-diego-certs/cf-diego-ca.crt      | indent | indent)
$(block public_cert  certs/cf-diego-certs/cloud-controller.crt | indent | indent)
$(block private_key  certs/cf-diego-certs/cloud-controller.key | indent | indent)
  consul:
$(block ca_cert     certs/consul-certs/server-ca.crt | indent | indent)
$(block agent_cert  certs/consul-certs/agent.crt     | indent | indent)
$(block agent_key   certs/consul-certs/agent.key     | indent | indent)
$(block server_cert certs/consul-certs/server.crt    | indent | indent)
$(block server_key  certs/consul-certs/server.key    | indent | indent)
  uaa:
$(block   signing_key       keypair/uaa                       | indent | indent)
$(block   verification_key  keypair/uaa.pub                   | indent | indent)
$(block   saml_cert         certs/uaa-certs/saml.crt          | indent | indent)
$(block   saml_key          certs/uaa-certs/saml.key          | indent | indent)
$(oneline saml_key_password certs/uaa-certs/saml.key.password | indent | indent)
$(block   ca_cert           certs/uaa-certs/server-ca.crt     | indent | indent)
$(block   server_key        certs/uaa-certs/server.key        | indent | indent)
$(block   server_cert       certs/uaa-certs/server.crt        | indent | indent)
  loggregator:
$(block ca_cert                certs/loggregator-certs/loggregator-ca.crt     | indent | indent)
$(block doppler_cert           certs/loggregator-certs/doppler.crt            | indent | indent)
$(block doppler_key            certs/loggregator-certs/doppler.key            | indent | indent)
$(block trafficcontroller_cert certs/loggregator-certs/trafficcontroller.crt  | indent | indent)
$(block trafficcontroller_key  certs/loggregator-certs/trafficcontroller.key  | indent | indent)
$(block metron_cert            certs/loggregator-certs/metron.crt             | indent | indent)
$(block metron_key             certs/loggregator-certs/metron.key             | indent | indent)
$(block statsd_injector_cert   certs/statsd-injector-certs/statsdinjector.crt | indent | indent)
$(block statsd_injector_key    certs/statsd-injector-certs/statsdinjector.key | indent | indent)
$(block syslogdrainbinder_cert certs/loggregator-certs/syslogdrainbinder.crt  | indent | indent)
$(block syslogdrainbinder_key  certs/loggregator-certs/syslogdrainbinder.key  | indent | indent)
$(block reverse_log_proxy_cert certs/loggregator-certs/reverselogproxy.crt  | indent | indent)
$(block reverse_log_proxy_key  certs/loggregator-certs/reverselogproxy.key  | indent | indent)
  scalablesyslog:
$(block adapter_ca_cert certs/loggregator-certs/loggregator-ca.crt | indent | indent)
$(block adapter_cert certs/loggregator-certs/ss-adapter.crt | indent | indent)
$(block adapter_key certs/loggregator-certs/ss-adapter.key | indent | indent)
$(block adapter_rlp_ca_cert certs/loggregator-certs/loggregator-ca.crt | indent | indent)
$(block adapter_rlp_cert certs/loggregator-certs/ss-adapter-rlp.crt | indent | indent)
$(block adapter_rlp_key certs/loggregator-certs/ss-adapter-rlp.key | indent | indent)
$(block scheduler_client_ca_cert certs/loggregator-certs/loggregator-ca.crt | indent | indent)
$(block scheduler_client_cert certs/loggregator-certs/ss-scheduler.crt | indent | indent)
$(block scheduler_client_key certs/loggregator-certs/ss-scheduler.key | indent | indent)
$(block scheduler_api_ca_cert certs/cf-diego-certs/cf-diego-ca.crt | indent | indent)
$(block scheduler_api_cert certs/loggregator-certs/ss-scheduler-api.crt | indent | indent)
$(block scheduler_api_key certs/loggregator-certs/ss-scheduler-api.key | indent | indent)
EOF
}

diego_credentials() {
  cat <<-EOF
# GENERATED: NO TOUCHING
---
diego_credentials:
$(block diego_ca     certs/cf-diego-certs/cf-diego-ca.crt | indent)
$(block ssh_proxy_host_key keypair/ssh-proxy-host-key.pem | indent)
  bbs:
$(block client_cert certs/bbs-certs/client.crt  | indent | indent)
$(block client_key  certs/bbs-certs/client.key  | indent | indent)
$(block server_cert certs/bbs-certs/server.crt  | indent | indent)
$(block server_key  certs/bbs-certs/server.key  | indent | indent)
  locket:
$(block server_cert certs/locket-certs/server.crt  | indent | indent)
$(block server_key  certs/locket-certs/server.key  | indent | indent)
  rep:
$(block client_cert certs/rep-certs/client.crt  | indent | indent)
$(block client_key  certs/rep-certs/client.key  | indent | indent)
$(block server_cert certs/rep-certs/server.crt  | indent | indent)
$(block server_key  certs/rep-certs/server.key  | indent | indent)
  auctioneer:
$(block client_cert certs/auctioneer-certs/client.crt  | indent | indent)
$(block client_key  certs/auctioneer-certs/client.key  | indent | indent)
$(block server_cert certs/auctioneer-certs/server.crt  | indent | indent)
$(block server_key  certs/auctioneer-certs/server.key  | indent | indent)
  tps:
$(block ca_cert     certs/cf-diego-certs/cf-diego-ca.crt  | indent | indent)
$(block client_cert certs/tps-certs/client.crt            | indent | indent)
$(block client_key  certs/tps-certs/client.key            | indent | indent)
  cc_uploader:
$(block ca_cert     certs/cf-diego-certs/cf-diego-ca.crt          | indent | indent)
$(block server_cert certs/cc-uploader-certs/server.crt            | indent | indent)
$(block server_key  certs/cc-uploader-certs/server.key            | indent | indent)
    cc:
$(block ca_cert     certs/cf-diego-certs/cf-diego-ca.crt            | indent | indent | indent)
$(block client_cert certs/cc-uploader-certs/cc/client.crt           | indent | indent | indent)
$(block client_key  certs/cc-uploader-certs/cc/client.key           | indent | indent | indent)
EOF
}

case "$1" in
  "create-stack")
    stack_action="create-stack"
    ;;

  "update-stack")
    stack_action="update-stack"
    ;;

  "skip-stack")
    stack_action=""
    ;;

  *)
    usage
    exit 1
    ;;
esac

shift

case "$1" in
  "deploy-bosh")
    bosh_action="deploy"
    ;;

  "skip-bosh")
    bosh_action="skip"
    ;;

  *)
    usage
    exit 1
    ;;
esac

shift
bosh_deployment_dir="$1"

shift
deployment_dir="$1"

shift
stack_name="$1"


if [ -z "${stack_name}" ]; then
  usage
  exit 1
fi

# name of BOSH CLI v2 executable
gobosh="${gobosh:-bosh}"

command -v aws >/dev/null || { echo "aws is required"; exit 1; }
command -v jq >/dev/null || { echo "jq is required"; exit 1; }
command -v spiff >/dev/null || { echo "spiff is required"; exit 1; }
command -v "${gobosh}" >/dev/null || { echo "BOSH CLI v2 is required"; exit 1; }
"${gobosh}" env --help >/dev/null 2>/dev/null || { echo "BOSH CLI v2 is required"; exit 1; }

set -x

pushd ${deployment_dir}

mkdir -p stubs/cf
mkdir -p stubs/infrastructure
mkdir -p deployments/bosh

source bootstrap_environment

# install certs for ELB
if ! aws iam get-server-certificate --server-certificate-name cfrouter; then
  aws iam upload-server-certificate                   \
    --server-certificate-name cfrouter                \
    --private-key file://certs/elb-cfrouter.key       \
    --certificate-body file://certs/elb-cfrouter.pem
fi

# generate stub to be fed into template for cloudformation
cat > stubs/infrastructure/certificates.yml <<EOF
# GENERATED: NO TOUCHING
EOF

aws iam get-server-certificate --server-certificate-name cfrouter \
  >> stubs/infrastructure/certificates.yml

function extra_subnets {
  cat <<EOF
---
EOF

  if [ -f stubs/infrastructure/extra_subnets.yml ]; then
    spiff merge \
        stubs/infrastructure/extra_subnets.yml \
        $SCRIPT_DIR/templates/infrastructure/meta.yml \
        stubs/infrastructure/availability_zones.yml
  fi
}

# generate cloudformation template
spiff merge \
  $SCRIPT_DIR/templates/infrastructure/boosh.yml \
  $SCRIPT_DIR/templates/infrastructure/boosh-internal.yml \
  stubs/bosh/vars.yml \
  stubs/domain.yml \
  stubs/infrastructure/certificates.yml \
  $SCRIPT_DIR/templates/infrastructure/meta.yml \
  stubs/infrastructure/availability_zones.yml \
  <(extra_subnets) \
  | boosh generate | jq -c -S . \
  > stubs/infrastructure/cloudformation.json

# deploy infrastructure
if [[ -n "$stack_action" ]]; then
  aws cloudformation "$stack_action" \
    --stack-name $stack_name \
    --template-body file://stubs/infrastructure/cloudformation.json

  # ensure that create or update was successful to aws
  boosh watch --name $stack_name
  # generate AWS resources stub for shared purposes
  cat > stubs/aws-resources.yml <<EOF
# GENERATED: NO TOUCHING
EOF

  boosh resources --name $stack_name >> stubs/aws-resources.yml
fi

cat > "${deployment_dir}/stubs/bosh/aws.yml" <<EOF
# GENERATED: NO TOUCHING
EOF

spiff merge \
  "${SCRIPT_DIR}/templates/bosh/aws.yml" \
  "${SCRIPT_DIR}/templates/bosh/aws-internal.yml" \
  "${deployment_dir}/stubs/aws-resources.yml" \
  >> "${deployment_dir}/stubs/bosh/aws.yml"

custom_ops_file_args=$(for i in $(ls ${deployment_dir}/ops-files/bosh/*.yml); do echo -n "--ops-file \"$i\" "; done)

bosh_datadog_args=""

if [[ -f "${deployment_dir}/stubs/bosh/datadog.yml" ]]; then
  bosh_datadog_args="
    --ops-file \"${SCRIPT_DIR}/ops-files/bosh/datadog.yml\" \
    --vars-file \"${deployment_dir}/stubs/bosh/datadog.yml\""
fi

bosh_domain_args=""

if [[ -f "${deployment_dir}/stubs/bosh/domain.yml" ]]; then
  bosh_domain_args="
    --ops-file \"${SCRIPT_DIR}/ops-files/bosh/domain.yml\" \
    --vars-file \"${deployment_dir}/stubs/bosh/domain.yml\""
fi

cat > "${deployment_dir}/deployments/bosh/bosh.yml" <<EOF
# GENERATED: NO TOUCHING
EOF

"${gobosh}" interpolate \
  --var-errs \
  --var-errs-unused \
  "${bosh_deployment_dir}/bosh.yml" \
  --vars-store "${deployment_dir}/deployments/bosh/creds.yml" \
  --ops-file "${bosh_deployment_dir}/aws/cpi.yml" \
  --ops-file "${bosh_deployment_dir}/external-ip-with-registry-not-recommended.yml" \
  $custom_ops_file_args \
  $bosh_datadog_args \
  $bosh_domain_args \
  --vars-file "${deployment_dir}/stubs/bosh/aws.yml" \
  --vars-file "${deployment_dir}/stubs/bosh/vars.yml" \
  --var "private_key=${deployment_dir}/keypair/id_rsa_bosh" \
  >> "${deployment_dir}/deployments/bosh/bosh.yml"

if [[ "$bosh_action" == "deploy" ]]; then
  "${gobosh}" create-env \
    --state "${deployment_dir}/deployments/bosh/state.json" \
    --vars-store "${deployment_dir}/deployments/bosh/creds.yml" \
    "${deployment_dir}/deployments/bosh/bosh.yml"
elif [[ "$bosh_action" == "skip" ]]; then
  >&2 echo "Skipping deployment of BOSH director"
fi

# generate director uuid stub for template to create deployment stub
bosh_ip=$(${gobosh} interpolate "${deployment_dir}/stubs/bosh/aws.yml" --path /external_ip)
director_uuid=$("${gobosh}" \
  -e ${bosh_ip} \
  --ca-cert "$("${gobosh}" interpolate "${deployment_dir}/deployments/bosh/creds.yml" --path /director_ssl/ca)" \
  env --json | jq -r '.Tables[0].Rows[0].uuid')

cat > stubs/director-uuid.yml <<EOF
# GENERATED: NO TOUCHING
---
director_uuid: ${director_uuid}
EOF

# generate stub with deployment base domain
cat > stubs/cf/domain.yml <<EOF
# GENERATED: NO TOUCHING
EOF

spiff merge $SCRIPT_DIR/templates/cf/domain.yml \
  $SCRIPT_DIR/templates/cf/domain-internal.yml \
  stubs/domain.yml \
  >> stubs/cf/domain.yml

# override cf and diego instance types
aws_instance_types=""
if [ -f stubs/aws-instance-types.yml ]; then
  aws_instance_types="stubs/aws-instance-types.yml"
fi

# generate deployment stub
cat > stubs/cf/stub.yml <<EOF
# GENERATED: NO TOUCHING
EOF

spiff merge \
  $SCRIPT_DIR/templates/cf/stub.yml \
  $SCRIPT_DIR/templates/cf/stub-internal.yml \
  $aws_instance_types \
  stubs/aws-resources.yml \
  stubs/cf/domain.yml \
  stubs/director-uuid.yml \
  <(cf_credentials) \
  >> stubs/cf/stub.yml

# copy CF property stub if not already present
if [ ! -f stubs/cf/properties.yml ]; then
  cp $SCRIPT_DIR/stubs/cf/properties.yml stubs/cf/properties.yml
fi

mkdir -p stubs/diego
mkdir -p stubs/diego-windows
mkdir -p stubs/routing

# generate Diego property-override stub with certs
if [ -f stubs/diego/property-overrides.yml ]; then
  # update BBS certs and keys in existing property-overrides stub
  temp_property_overrides=$(mktemp stubs/diego/property-overrides.yml.XXXXX)

  spiff merge \
    stubs/diego/property-overrides.yml \
    $SCRIPT_DIR/templates/diego/property-overrides-internal.yml \
    <(diego_credentials) \
    > "${temp_property_overrides}"

  mv "${temp_property_overrides}" stubs/diego/property-overrides.yml
else
  # create new property-overrides stub with default overrides
  spiff merge \
    $SCRIPT_DIR/templates/diego/property-overrides.yml \
    $SCRIPT_DIR/templates/diego/property-overrides-internal.yml \
    <(diego_credentials) \
    > stubs/diego/property-overrides.yml
fi

# generate Diego IaaS-settings stub
spiff merge \
  $SCRIPT_DIR/../../manifest-generation/misc-templates/aws-iaas-settings.yml \
  $SCRIPT_DIR/templates/diego/iaas-settings-internal.yml \
  $aws_instance_types stubs/aws-resources.yml \
  > stubs/diego/iaas-settings.yml

# generate Diego Windows IaaS-settings stub
spiff merge \
  $SCRIPT_DIR/../../manifest-generation/misc-templates/windows-iaas-settings.yml \
  $SCRIPT_DIR/templates/diego-windows/iaas-settings-internal.yml \
  $aws_instance_types stubs/aws-resources.yml \
  > stubs/diego-windows/iaas-settings.yml

# generate Routing IaaS-settings stub
spiff merge \
  $SCRIPT_DIR/templates/routing/iaas-settings.yml \
  stubs/aws-resources.yml \
  > stubs/routing/iaas-settings.yml

# generate mysql cloud config stub
spiff merge \
  $SCRIPT_DIR/templates/infrastructure/cloud-config.yml \
  $SCRIPT_DIR/templates/infrastructure/cloud-config-internal.yml \
  $aws_instance_types stubs/aws-resources.yml \
  > stubs/cloud-config.yml

mkdir -p stubs/cf-mysql

if [ ! -f stubs/cf-mysql/mysql-overrides-ops.yml ]; then
  cp $SCRIPT_DIR/templates/mysql/mysql-overrides-ops.yml stubs/cf-mysql/mysql-overrides-ops.yml
fi

spiff merge $SCRIPT_DIR/templates/mysql/consul-secrets.yml <(cf_credentials) > stubs/cf-mysql/consul-secrets.yml


# generate datadog firehose nozzle manifest if the user stubs exist
if [ -d stubs/datadog-firehose-nozzle ]; then
    spiff merge \
          $SCRIPT_DIR/templates/datadog-firehose-nozzle/datadog-firehose-nozzle.yml \
          stubs/datadog-firehose-nozzle/datadog.yml \
          stubs/datadog-firehose-nozzle/property-overrides.yml \
          stubs/aws-resources.yml \
          stubs/cf/properties.yml \
          stubs/cf/stub.yml \
          stubs/cf/domain.yml \
          stubs/director-uuid.yml \
          > deployments/datadog-firehose-nozzle.yml
fi
popd