Add a new AWS backend using Amazon Machine Images

The existing AWS backend uses system containers backed on to the AWS
hosted NFS service (EFS). This has some advantages, but also some
disadvantages.

Using the EFS service allows building a container on one instance,
with the state being held on the EFS, and then launching a new
instance to run the container. Using EFS also provides persistence, at
least beyond individual EC2 instances.

However, build performance when using EFS is poor compared with a
local store without the overhead of the network latency. Additionally,
the startup speed of the container running off EFS is slow compared to
local storage.

This backend doesn't use EFS, instead the Guix store sits on instance
storage. Rather than using a system container for a Mini Environment,
an Amazon Machine Image (AMI) is built instead. The fast local storage
makes builds faster, and using EBS storage for the Mini Envirnoments
as well as not having the overhead of starting Ubuntu, then starting
the container also makes startup faster.

3 files changed, 568 insertions, 0 deletions

diff --git a/terraform/aws_using_ami/backend/guix-daemon.service.tpl b/terraform/aws_using_ami/backend/guix-daemon.service.tpl
new file mode 100644
index 0000000..184e3d5
--- /dev/null
+++ b/terraform/aws_using_ami/backend/guix-daemon.service.tpl
@@ -0,0 +1,17 @@
+[Unit]
+Description=Build daemon for GNU Guix
+
+[Service]
+ExecStart=/var/guix/profiles/per-user/root/current-guix/bin/guix-daemon --build-users-group=guixbuild --substitute-urls="${substitute_servers}" --listen=0.0.0.0 --listen=/var/guix/daemon-socket/socket --max-jobs=4
+Environment=GUIX_LOCPATH='/var/guix/profiles/per-user/root/guix-profile/lib/locale' LC_ALL=en_US.utf8 TMPDIR=/gnu/tmp
+RemainAfterExit=yes
+StandardOutput=syslog
+StandardError=syslog
+
+# See <https://lists.gnu.org/archive/html/guix-devel/2016-04/msg00608.html>.
+# Some package builds (for example, go@1.8.1) may require even more than
+# 1024 tasks.
+TasksMax=8192
+
+[Install]
+WantedBy=multi-user.target
diff --git a/terraform/aws_using_ami/backend/main.tf b/terraform/aws_using_ami/backend/main.tf
new file mode 100644
index 0000000..fead443
--- /dev/null
+++ b/terraform/aws_using_ami/backend/main.tf
@@ -0,0 +1,437 @@
+terraform {
+  backend "http" {}
+}
+
+variable "aws_access_key" {
+  type = "string"
+}
+
+variable "aws_secret_key" {
+  type = "string"
+}
+
+variable "aws_region" {
+  type = "string"
+}
+
+variable "aws_vpc_id" {
+  type = "string"
+}
+
+variable "aws_route_53_zone_id" {
+  type = "string"
+}
+
+variable "ssh_public_key" {
+  type = "string"
+}
+
+variable "ssh_private_key" {
+  type = "string"
+}
+
+variable "guix_substitute_servers" {
+  type = "map"
+  default = {
+    "https://ci.guix.gnu.org"    = <<EOF
+(entry
+ (public-key
+  (ecc
+   (curve Ed25519)
+   (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#)
+  )
+ )
+ (tag
+  (guix import)
+ )
+)
+EOF
+  }
+}
+
+variable "mini_environment_admin_guix_public_key" {
+  type = "string"
+}
+
+variable "mini_environment_admin_egress_cidr_blocks" {
+  type = "list"
+}
+
+variable "backend_slug" {
+  type = "string"
+}
+
+locals {
+  guix_daemon_substitute_servers = "${join(" ", keys(var.guix_substitute_servers))}"
+}
+
+provider "aws" {
+  access_key = "${var.aws_access_key}"
+  secret_key = "${var.aws_secret_key}"
+  region  = "${var.aws_region}"
+}
+
+data "aws_route53_zone" "main" {
+  zone_id = "${var.aws_route_53_zone_id}"
+}
+
+data "template_file" "guix_daemon_service" {
+  template = "${file("${path.module}/guix-daemon.service.tpl")}"
+
+  vars {
+    substitute_servers = "${local.guix_daemon_substitute_servers}"
+  }
+}
+
+data "aws_availability_zones" "available" {}
+
+resource "aws_key_pair" "deployer" {
+  key_name = "govuk_mini_environment_admin/${var.backend_slug}/deployer"
+  public_key = "${var.ssh_public_key}"
+}
+
+resource "aws_security_group" "public_webserver" {
+  name        = "govuk_mini_environment_admin/${var.backend_slug}/public_webserver"
+  description = "For instances running public facing web servers"
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port       = 0
+    to_port         = 80
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port       = 0
+    to_port         = 443
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port       = 0
+    to_port         = 8080
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port       = 0
+    to_port         = 8443
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "ssh_access_from_mini_environment_admin" {
+  name        = "govuk_mini_environment_admin/${var.backend_slug}/ssh_access_from_mini_environment_admin"
+  description = "For instances that need SSH access for Terraform and Guix builds"
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port = 0
+    to_port = 22
+    protocol = "tcp"
+    cidr_blocks = [
+      "${var.mini_environment_admin_egress_cidr_blocks}"
+    ]
+  }
+}
+
+resource "aws_security_group" "guix_client" {
+  name        = "govuk_mini_environment_admin/${var.backend_slug}/guix_client"
+  description = "For instances with access to the guix_daemon instance"
+  vpc_id      = "${var.aws_vpc_id}"
+
+  egress {
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "guix_publish" {
+  name        = "govuk_mini_environment_admin/${var.backend_slug}/guix_publish"
+  description = "For guix publish on the builder instance."
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port       = 80
+    to_port         = 80
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "guix_daemon" {
+  name        = "govuk_mini_environment_admin/${var.backend_slug}/guix_daemon"
+  description = "For the guix_daemon instance."
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port       = 44146
+    to_port         = 44146
+    protocol        = "tcp"
+    security_groups = ["${aws_security_group.guix_client.id}"]
+  }
+
+  egress {
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_iam_policy" "packer" {
+  name        = "packer_policy"
+  path        = "/govuk/mini_environment_admin/${var.backend_slug}/"
+  description = "Policy to enable using packer to build AMIs"
+
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [{
+      "Effect": "Allow",
+      "Action" : [
+        "ec2:AttachVolume",
+        "ec2:AuthorizeSecurityGroupIngress",
+        "ec2:CopyImage",
+        "ec2:CreateImage",
+        "ec2:CreateKeypair",
+        "ec2:CreateSecurityGroup",
+        "ec2:CreateSnapshot",
+        "ec2:CreateTags",
+        "ec2:CreateVolume",
+        "ec2:DeleteKeyPair",
+        "ec2:DeleteSecurityGroup",
+        "ec2:DeleteSnapshot",
+        "ec2:DeleteVolume",
+        "ec2:DeregisterImage",
+        "ec2:DescribeImageAttribute",
+        "ec2:DescribeImages",
+        "ec2:DescribeInstances",
+        "ec2:DescribeInstanceStatus",
+        "ec2:DescribeRegions",
+        "ec2:DescribeSecurityGroups",
+        "ec2:DescribeSnapshots",
+        "ec2:DescribeSubnets",
+        "ec2:DescribeTags",
+        "ec2:DescribeVolumes",
+        "ec2:DetachVolume",
+        "ec2:GetPasswordData",
+        "ec2:ModifyImageAttribute",
+        "ec2:ModifyInstanceAttribute",
+        "ec2:ModifySnapshotAttribute",
+        "ec2:RegisterImage",
+        "ec2:RunInstances",
+        "ec2:StopInstances",
+        "ec2:TerminateInstances"
+      ],
+      "Resource" : "*"
+  }]
+}
+EOF
+}
+
+resource "aws_iam_role" "builder" {
+  name = "builder_role"
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "builder_role_packer_policy" {
+  role       = "${aws_iam_role.builder.name}"
+  policy_arn = "${aws_iam_policy.packer.arn}"
+}
+
+resource "aws_iam_instance_profile" "builder" {
+  name = "builder_profile"
+  role = "${aws_iam_role.builder.name}"
+}
+
+resource "aws_spot_instance_request" "main" {
+  ami             = "ami-0b2a4d260c54e8d3d"
+  instance_type   = "i3.xlarge"
+  key_name        = "${aws_key_pair.deployer.key_name}"
+  security_groups = [
+    "${aws_security_group.guix_daemon.name}",
+    "${aws_security_group.guix_publish.name}",
+    "${aws_security_group.ssh_access_from_mini_environment_admin.name}"
+  ]
+  iam_instance_profile = "${aws_iam_instance_profile.builder.name}"
+
+  wait_for_fulfillment = true
+  spot_price = "0.172"
+  spot_type = "one-time"
+  availability_zone = "${data.aws_availability_zones.available.names[1]}"
+
+  root_block_device {
+    volume_size = "50"
+  }
+
+  provisioner "file" {
+    content = "${data.template_file.guix_daemon_service.rendered}"
+    destination = "/home/ubuntu/guix-daemon.service"
+
+    connection {
+      type        = "ssh"
+      user        = "ubuntu"
+      private_key = "${var.ssh_private_key}"
+    }
+  }
+
+  provisioner "file" {
+    content = "(acl\n ${join("\n", values(var.guix_substitute_servers))}\n${var.mini_environment_admin_guix_public_key}\n)\n"
+    destination = "/home/ubuntu/acl"
+
+    connection {
+      type        = "ssh"
+      user        = "ubuntu"
+      private_key = "${var.ssh_private_key}"
+    }
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      <<EOF
+sudo sgdisk --largest-new=0 /dev/nvme0n1
+
+sudo mkfs.ext4 -m 0 /dev/nvme0n1p1
+
+echo "/dev/nvme0n1p1	 /gnu	 ext4	defaults,discard	0 0" | sudo tee -a /etc/fstab
+
+sudo cat /etc/fstab
+
+sudo mkdir -p /gnu
+sudo mount -a
+sudo mkdir -p /gnu/tmp
+EOF
+      ,
+      <<EOF
+set -e
+
+FILENAME="guix-binary-1.0.0.x86_64-linux.tar.xz"
+
+wget https://ftp.gnu.org/gnu/guix/$FILENAME
+
+cd /
+sudo tar --warning=no-timestamp -xf /home/ubuntu/$FILENAME
+cd -
+
+rm "$FILENAME"
+
+sudo mkdir -p ~root/.config/guix
+sudo ln -sf /var/guix/profiles/per-user/root/current-guix ~root/.config/guix/current
+
+sudo mkdir -p /usr/local/bin
+sudo ln --symbolic --no-target-directory /var/guix/profiles/per-user/root/current-guix/bin/guix /usr/local/bin/guix
+EOF
+      ,
+      "sudo mv /home/ubuntu/guix-daemon.service /etc/systemd/system/guix-daemon.service",
+      "sudo mkdir /etc/guix",
+      "sudo mv /home/ubuntu/acl /etc/guix/acl",
+      "sudo groupadd --system guixbuild",
+      <<EOF
+for i in `seq -w 1 32`;
+do
+  sudo useradd -g guixbuild -G guixbuild           \
+          -d /var/empty -s `which nologin`    \
+          -c "Guix build user $i" --system    \
+          guix$i;
+done
+EOF
+      ,
+      # Disable apparmor, as it seems to cause guile to segfault
+      # sometimes
+      "sudo systemctl stop apparmor",
+      "sudo service apparmor teardown",
+      "sudo systemctl disable apparmor",
+      "sudo systemctl daemon-reload",
+      "sudo systemctl enable guix-daemon.service",
+      "sudo systemctl start guix-daemon.service",
+      "sudo mkdir -p /var/guix/gcroots/govuk-development-data /var/guix/gcroots/govuk-mini-environment-admin",
+      "sudo chown ubuntu:ubuntu /var/guix/gcroots/govuk-development-data /var/guix/gcroots/govuk-mini-environment-admin",
+      "mkdir -p ~/.config", # Make the ~/.config directory, to ensure it's owned by ubuntu
+      # Setup the profile for the ubuntu user
+      <<EOF
+if [ ! -d "/var/guix/profiles/per-user/ubuntu" ]; then
+  # Install openssl and nss-certs so that downloads work for custom
+  # package versions, and guile-sqlite3
+  /var/guix/profiles/per-user/root/current-guix/bin/guix package --fallback -i glibc-locales guile bash openssl nss-certs guile-sqlite3
+fi
+EOF
+      ,
+      # This is needed for things like guix copy to work
+      <<EOF
+echo 'export GUIX_LOCPATH="$HOME/.guix-profile/lib/locale"
+
+GUIX_PROFILE=/home/ubuntu/.guix-profile; source /home/ubuntu/.guix-profile/etc/profile
+GUIX_PROFILE=/home/ubuntu/.config/guix/current; source /home/ubuntu/.config/guix/current/etc/profile
+' | cat - .bashrc > temp && mv temp .bashrc
+EOF
+    ]
+
+    connection {
+      type        = "ssh"
+      user        = "ubuntu"
+      private_key = "${var.ssh_private_key}"
+    }
+  }
+}
+
+resource "aws_route53_record" "main" {
+  zone_id = "${data.aws_route53_zone.main.zone_id}"
+  name    = "builder.${var.backend_slug}"
+  type    = "A"
+  ttl     = "60"
+  records = ["${aws_spot_instance_request.main.public_ip}"]
+
+  depends_on = ["aws_spot_instance_request.main"]
+}
+
+# Outputs
+
+output "backend_up" {
+  value = "${length(aws_spot_instance_request.main.public_ip) != 0}"
+}
+
+output "deployer_key_pair_name" {
+  value = "${aws_key_pair.deployer.key_name}"
+}
+
+output "guix_client_security_group_name" {
+  value = "${aws_security_group.guix_client.name}"
+}
+
+output "public_webserver_security_group_name" {
+  value = "${aws_security_group.public_webserver.name}"
+}
+
+output "ssh_access_from_mini_environment_admin_security_group_name" {
+  value = "${aws_security_group.ssh_access_from_mini_environment_admin.name}"
+}
+
+output "guix_daemon_private_dns" {
+  value = "${aws_spot_instance_request.main.private_dns}"
+}
+
+output "guix_daemon_public_dns" {
+  value = "${aws_spot_instance_request.main.public_dns}"
+}
diff --git a/terraform/aws_using_ami/mini_environment/main.tf b/terraform/aws_using_ami/mini_environment/main.tf
new file mode 100644
index 0000000..08445af
--- /dev/null
+++ b/terraform/aws_using_ami/mini_environment/main.tf
@@ -0,0 +1,114 @@
+terraform {
+  backend "http" {}
+}
+
+variable "slug" {
+  type = "string"
+}
+
+variable "aws_access_key" {
+  type = "string"
+}
+
+variable "aws_secret_key" {
+  type = "string"
+}
+
+variable "aws_region" {
+  type = "string"
+}
+
+variable "aws_route_53_zone_id" {
+  type = "string"
+}
+
+variable "ami_id" {
+  type = "string"
+}
+
+variable "backend_remote_state_address" {
+  type = "string"
+}
+
+variable "backend_remote_state_username" {
+  type = "string"
+}
+
+variable "backend_remote_state_password" {
+  type = "string"
+}
+
+variable "ssh_private_key" {
+  type = "string"
+}
+
+provider "aws" {
+  access_key = "${var.aws_access_key}"
+  secret_key = "${var.aws_secret_key}"
+  region  = "${var.aws_region}"
+}
+
+
+data "terraform_remote_state" "backend" {
+  backend = "http"
+  config {
+    address = "${var.backend_remote_state_address}"
+    username = "${var.backend_remote_state_username}"
+    password = "${var.backend_remote_state_password}"
+  }
+}
+
+data "aws_route53_zone" "main" {
+  zone_id = "${var.aws_route_53_zone_id}"
+}
+
+
+resource "aws_spot_instance_request" "main" {
+  ami           = "${var.ami_id}"
+  instance_type = "t2.xlarge"
+  key_name      = "${data.terraform_remote_state.backend.deployer_key_pair_name}"
+  security_groups = [
+    "${data.terraform_remote_state.backend.guix_client_security_group_name}",
+    "${data.terraform_remote_state.backend.public_webserver_security_group_name}",
+    "${data.terraform_remote_state.backend.ssh_access_from_mini_environment_admin_security_group_name}"
+  ]
+
+  wait_for_fulfillment = true
+  spot_price = "0.21"
+  spot_type = "one-time"
+
+  root_block_device {
+    volume_size = "200"
+  }
+}
+
+resource "aws_route53_record" "main" {
+  zone_id = "${data.aws_route53_zone.main.zone_id}"
+  name    = "${var.slug}"
+  type    = "A"
+  ttl     = "60"
+  records = ["${aws_spot_instance_request.main.public_ip}"]
+}
+
+resource "aws_route53_record" "wildcard" {
+  zone_id = "${data.aws_route53_zone.main.zone_id}"
+  name    = "*.${var.slug}"
+  type    = "A"
+  ttl     = "60"
+  records = ["${aws_spot_instance_request.main.public_ip}"]
+}
+
+# Outputs
+
+output "spot_bid_status" {
+  value = "${aws_spot_instance_request.main.spot_bid_status}"
+}
+
+output "spot_request_status" {
+  value = "${aws_spot_instance_request.main.spot_request_state}"
+}
+
+output "mini_environment_up" {
+  value = "${aws_spot_instance_request.main.spot_bid_status == "fulfilled"}"
+}
+