Improve the AWS backend

This gets the AWS backend to the point where you can deploy the
backend, and then a mini environment without any actions outside of
the govuk-mini-environment-admin.

3 files changed, 352 insertions, 26 deletions

diff --git a/terraform/aws/backend/guix-daemon.service.tpl b/terraform/aws/backend/guix-daemon.service.tpl
new file mode 100644
index 0000000..adf07a1
--- /dev/null
+++ b/terraform/aws/backend/guix-daemon.service.tpl
@@ -0,0 +1,17 @@
+[Unit]
+Description=Build daemon for GNU Guix
+
+[Service]
+ExecStart=/var/guix/profiles/per-user/root/guix-profile/bin/guix-daemon --build-users-group=guixbuild --disable-deduplication --substitute-urls="${substitute_servers}" --listen=0.0.0.0 --listen=/var/guix/daemon-socket/socket --max-jobs=16
+Environment=GUIX_LOCPATH=/root/.guix-profile/lib/locale
+RemainAfterExit=yes
+StandardOutput=syslog
+StandardError=syslog
+
+# See <https://lists.gnu.org/archive/html/guix-devel/2016-04/msg00608.html>.
+# Some package builds (for example, go@1.8.1) may require even more than
+# 1024 tasks.
+TasksMax=8192
+
+[Install]
+WantedBy=multi-user.target
diff --git a/terraform/aws/backend/main.tf b/terraform/aws/backend/main.tf
new file mode 100644
index 0000000..56b91b9
--- /dev/null
+++ b/terraform/aws/backend/main.tf
@@ -0,0 +1,302 @@
+terraform {
+  backend "http" {}
+}
+
+variable "aws_access_key" {
+  type = "string"
+}
+
+variable "aws_secret_key" {
+  type = "string"
+}
+
+variable "aws_region" {
+  type = "string"
+}
+
+variable "aws_vpc_id" {
+  type = "string"
+}
+
+variable "aws_route_53_zone_id" {
+  type = "string"
+}
+
+variable "aws_efs_file_system_id" {
+  type = "string"
+}
+
+variable "ssh_public_key" {
+  type = "string"
+}
+
+variable "guix_substitute_servers" {
+  type = "map"
+  default = {
+    "https://berlin.guixsd.org"    = <<EOF
+(entry
+ (public-key
+  (ecc
+   (curve Ed25519)
+   (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#)
+  )
+ )
+ (tag
+  (guix import)
+ )
+)
+EOF
+    "http://beid.cbaines.net"      = <<EOF
+(entry
+ (public-key
+  (ecc
+   (curve Ed25519)
+   (q #50349E83123851A65A569304A12E512B698223A81E10BEEC5A3E56EDAEE5DAC1#)
+  )
+ )
+ (tag
+  (guix import)
+ )
+)
+EOF
+  }
+}
+
+variable "mini_environment_admin_guix_public_key" {
+  type = "string"
+}
+
+variable "mini_environment_admin_public_ip_address" {
+  type = "string"
+}
+
+locals {
+  guix_daemon_substitute_servers = "${join(" ", keys(var.guix_substitute_servers))}"
+}
+
+provider "aws" {
+  access_key = "${var.aws_access_key}"
+  secret_key = "${var.aws_secret_key}"
+  region  = "${var.aws_region}"
+}
+
+data "aws_route53_zone" "main" {
+  zone_id = "${var.aws_route_53_zone_id}"
+}
+
+data "aws_efs_file_system" "main" {
+  file_system_id = "${var.aws_efs_file_system_id}"
+}
+
+data "template_file" "guix_daemon_service" {
+  template = "${file("${path.module}/guix-daemon.service.tpl")}"
+
+  vars {
+    substitute_servers = "${local.guix_daemon_substitute_servers}"
+  }
+}
+
+
+resource "aws_key_pair" "deployer" {
+  public_key = "${var.ssh_public_key}"
+}
+
+resource "aws_security_group" "public_webserver" {
+  name        = "govuk_mini_environment_admin_public_webserver"
+  description = "For instances running public facing web servers"
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port       = 0
+    to_port         = 80
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port       = 0
+    to_port         = 443
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port       = 0
+    to_port         = 8080
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port       = 0
+    to_port         = 8443
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "ssh_access_from_mini_environment_admin" {
+  name        = "govuk_mini_environment_admin_ssh_access_from_mini_environment_admin"
+  description = "For instances that need SSH access for Terraform and Guix builds"
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port = 0
+    to_port = 22
+    protocol = "tcp"
+    cidr_blocks = ["${var.mini_environment_admin_public_ip_address}/32"]
+  }
+}
+
+resource "aws_security_group" "guix_client" {
+  name        = "govuk_mini_environment_admin_guix_client"
+  description = "For instances with access to the guix_daemon instance"
+  vpc_id      = "${var.aws_vpc_id}"
+
+  egress {
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "guix_daemon" {
+  name        = "govuk_mini_environment_admin_guix_daemon"
+  description = "For the guix_daemon instance."
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port       = 44146
+    to_port         = 44146
+    protocol        = "tcp"
+    security_groups = ["${aws_security_group.guix_client.id}"]
+  }
+
+  egress {
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "efs_mount_target" {
+  name        = "govuk_mini_environment_admin_efs_mount_target"
+  description = "For the EFS File System mount targets"
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port       = 0
+    to_port         = 0
+    protocol        = "-1"
+    security_groups = [
+      "${aws_security_group.guix_client.id}",
+      "${aws_security_group.guix_daemon.id}",
+    ]
+  }
+}
+
+resource "aws_spot_instance_request" "main" {
+  ami             = "ami-8fd760f6"
+  instance_type   = "t2.medium"
+  key_name        = "${aws_key_pair.deployer.key_name}"
+  security_groups = [
+    "${aws_security_group.guix_daemon.name}",
+    "${aws_security_group.ssh_access_from_mini_environment_admin.name}"
+
+  ]
+
+  wait_for_fulfillment = true
+  spot_price = "0.05"
+
+  provisioner "file" {
+    content = "${data.template_file.guix_daemon_service.rendered}"
+    destination = "/home/ubuntu/guix-daemon.service"
+
+    connection {
+      type     = "ssh"
+      user     = "ubuntu"
+    }
+  }
+
+  provisioner "file" {
+    content = "(acl\n ${join("\n", values(var.guix_substitute_servers))}\n${var.mini_environment_admin_guix_public_key}\n)\n"
+    destination = "/home/ubuntu/acl"
+
+    connection {
+      type     = "ssh"
+      user     = "ubuntu"
+    }
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo apt-get update",
+      "sudo apt-get update",
+      "sudo apt-get -y install nfs-common cachefilesd nscd",
+      "sudo tune2fs -o user_xattr /dev/xvda1",
+      "sudo sed 's/#RUN/RUN/' -i /etc/default/cachefilesd",
+      "echo \"${data.aws_efs_file_system.main.dns_name}:/var/guix /var/guix nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab",
+      "echo \"${data.aws_efs_file_system.main.dns_name}:/gnu/store /gnu/store nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,fsc,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab",
+      "echo \"${data.aws_efs_file_system.main.dns_name}:/ /mnt/efs nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,fsc,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab",
+      "sudo mkdir -p /var/guix /gnu/store /mnt/efs",
+      "sudo mount -a",
+      "sudo mv /home/ubuntu/guix-daemon.service /etc/systemd/system/guix-daemon.service",
+      "sudo mkdir /etc/guix",
+      "sudo mv /home/ubuntu/acl /etc/guix/acl",
+      "sudo groupadd --system guixbuild",
+      <<EOF
+for i in `seq -w 1 17`;
+do
+  sudo useradd -g guixbuild -G guixbuild           \
+          -d /var/empty -s `which nologin`    \
+          -c "Guix build user $i" --system    \
+          guixbuilder$i;
+done
+EOF
+      ,
+      "sudo systemctl daemon-reload",
+      "sudo systemctl enable guix-daemon.service",
+      "sudo systemctl start guix-daemon.service",
+      "ln -s /var/guix/profiles/per-user/ubuntu/guix-profile ~/.guix-profile",
+      # This is needed for things like guix copy to work
+      "echo 'GUIX_PROFILE=/home/ubuntu/.guix-profile; source /home/ubuntu/.guix-profile/etc/profile' | cat - .bashrc > temp && mv temp .bashrc"
+    ]
+
+    connection {
+      type     = "ssh"
+      user     = "ubuntu"
+    }
+  }
+}
+
+resource "aws_route53_record" "main" {
+  zone_id = "${data.aws_route53_zone.main.zone_id}"
+  name    = "guix-daemon"
+  type    = "A"
+  ttl     = "60"
+  records = ["${aws_spot_instance_request.main.public_ip}"]
+}
+
+# Outputs
+
+output "deployer_key_pair_name" {
+  value = "${aws_key_pair.deployer.key_name}"
+}
+
+output "guix_client_security_group_name" {
+  value = "${aws_security_group.guix_client.name}"
+}
+
+output "public_webserver_security_group_name" {
+  value = "${aws_security_group.public_webserver.name}"
+}
+
+output "ssh_access_from_mini_environment_admin_security_group_name" {
+  value = "${aws_security_group.ssh_access_from_mini_environment_admin.name}"
+}
+
+output "guix_daemon_private_dns" {
+  value = "${aws_spot_instance_request.main.private_dns}"
+}
diff --git a/terraform/aws/mini_environment.tf b/terraform/aws/mini_environment/main.tf
index 06bc0f3..4f9b9c3 100644
--- a/terraform/aws/mini_environment.tf
+++ b/terraform/aws/mini_environment/main.tf
@@ -18,7 +18,11 @@ variable "aws_region" {
   type = "string"
 }
 
-variable "ssh_public_key" {
+variable "aws_route_53_zone_id" {
+  type = "string"
+}
+
+variable "aws_efs_file_system_id" {
   type = "string"
 }
 
@@ -26,48 +30,49 @@ variable "start_command" {
   type = "string"
 }
 
+variable "backend_remote_state_address" {
+  type = "string"
+}
+
 provider "aws" {
   access_key = "${var.aws_access_key}"
   secret_key = "${var.aws_secret_key}"
   region  = "${var.aws_region}"
 }
 
-resource "aws_key_pair" "deployer" {
-  public_key = "${var.ssh_public_key}"
-}
-
-data "aws_security_group" "guix-client" {
-  id = "sg-d8003ba3"
-}
 
-data "aws_instance" "guix-daemon" {
-  instance_id = "i-010e25f85dfa73e72"
+data "terraform_remote_state" "backend" {
+  backend = "http"
+  config {
+    address = "${var.backend_remote_state_address}"
+  }
 }
 
 data "aws_route53_zone" "main" {
-  zone_id = "ZD004G8DN6AQZ"
+  zone_id = "${var.aws_route_53_zone_id}"
 }
 
 data "aws_efs_file_system" "main" {
-  file_system_id = "fs-81e05e48"
+  file_system_id = "${var.aws_efs_file_system_id}"
 }
 
 data "template_file" "govuk_service" {
-  template = "${file("${path.module}/mini_environment/govuk.service.tpl")}"
+  template = "${file("${path.module}/govuk.service.tpl")}"
 
   vars {
     start_command = "${var.start_command}"
   }
 }
 
-resource "aws_spot_instance_request" "example" {
+
+resource "aws_spot_instance_request" "main" {
   ami           = "ami-8fd760f6"
   instance_type = "t2.large"
-  key_name      = "${aws_key_pair.deployer.key_name}"
+  key_name      = "${data.terraform_remote_state.backend.deployer_key_pair_name}"
   security_groups = [
-    "${data.aws_security_group.guix-client.name}",
-    "default",
-    "public-webserver"
+    "${data.terraform_remote_state.backend.guix_client_security_group_name}",
+    "${data.terraform_remote_state.backend.public_webserver_security_group_name}",
+    "${data.terraform_remote_state.backend.ssh_access_from_mini_environment_admin_security_group_name}"
   ]
 
   wait_for_fulfillment = true
@@ -76,7 +81,7 @@ resource "aws_spot_instance_request" "example" {
   provisioner "file" {
     content = "${data.template_file.govuk_service.rendered}"
     destination = "/home/ubuntu/govuk.service"
-    
+
     connection {
       type     = "ssh"
       user     = "ubuntu"
@@ -94,10 +99,12 @@ resource "aws_spot_instance_request" "example" {
       "sudo mount -t nfs4 -o ro,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,actimeo=600,fsc,nocto,retrans=2 ${data.aws_efs_file_system.main.dns_name}:gnu/store /gnu/store",
       "sudo mkdir -p /var/guix",
       "sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 ${data.aws_efs_file_system.main.dns_name}:var/guix /var/guix",
-      "echo \"export GUIX_DAEMON_SOCKET=guix://${data.aws_instance.guix-daemon.private_dns}\" | sudo tee /etc/profile.d/guix-daemon-socket.sh",
+      "echo \"export GUIX_DAEMON_SOCKET=guix://${data.terraform_remote_state.backend.guix_daemon_private_dns}\" | sudo tee /etc/profile.d/guix-daemon-socket.sh",
       #"sudo systemctl restart cachefilesd",
       "sudo iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080",
       "sudo iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8443",
+      "sudo iptables -A OUTPUT -t nat -o lo -p tcp --dport 80 -j REDIRECT --to-port 8080",
+      "sudo iptables -A OUTPUT -t nat -o lo -p tcp --dport 443 -j REDIRECT --to-port 8443",
       "sudo mv /home/ubuntu/govuk.service /etc/systemd/system/govuk.service",
       "sudo systemctl daemon-reload",
       "sudo systemctl enable govuk.service",
@@ -111,28 +118,28 @@ resource "aws_spot_instance_request" "example" {
   }
 }
 
-resource "aws_route53_record" "example" {
+resource "aws_route53_record" "main" {
   zone_id = "${data.aws_route53_zone.main.zone_id}"
   name    = "${var.slug}"
   type    = "A"
   ttl     = "60"
-  records = ["${aws_spot_instance_request.example.public_ip}"]
+  records = ["${aws_spot_instance_request.main.public_ip}"]
 }
 
-resource "aws_route53_record" "example_wildcard" {
+resource "aws_route53_record" "wildcard" {
   zone_id = "${data.aws_route53_zone.main.zone_id}"
   name    = "*.${var.slug}"
   type    = "A"
   ttl     = "60"
-  records = ["${aws_spot_instance_request.example.public_ip}"]
+  records = ["${aws_spot_instance_request.main.public_ip}"]
 }
 
 # Outputs
 
 output "spot_bid_status" {
-  value = "${aws_spot_instance_request.example.spot_bid_status}"
+  value = "${aws_spot_instance_request.main.spot_bid_status}"
 }
 
 output "spot_request_status" {
-  value = "${aws_spot_instance_request.example.spot_request_state}"
+  value = "${aws_spot_instance_request.main.spot_request_state}"
 }