Improve the AWS backend

This gets the AWS backend to the point where you can deploy the
backend, and then a mini environment without any actions outside of
the govuk-mini-environment-admin.

7 files changed, 455 insertions, 53 deletions

diff --git a/app/jobs/govuk_guix/build_job.rb b/app/jobs/govuk_guix/build_job.rb
index a74b928..fb5e256 100644
--- a/app/jobs/govuk_guix/build_job.rb
+++ b/app/jobs/govuk_guix/build_job.rb
@@ -27,22 +27,40 @@ class GovukGuix::BuildJob < GovukGuix::Job
 
   @retry_interval = 30
 
-  def run(mini_environment_id, services, arguments)
+  def run(mini_environment_id, options)
     logger.info(self.class) do
       "Building mini environment #{mini_environment_id}"
     end
 
     mini_environment = MiniEnvironment.find(mini_environment_id)
 
+    remote_host = options[:run_remotely_on_host]
+
+    if remote_host
+      ssh_command = ['ssh', remote_host]
+
+      # Copy the revision to the remote host, to ensure it's available
+      # there
+      run_command(
+        'guix',
+        'copy',
+        "--to=#{remote_host}",
+        mini_environment.govuk_guix_revision.store_path
+      )
+    else
+      ssh_command = []
+    end
+
     output = run_command(
+      *ssh_command,
       "#{mini_environment.govuk_guix_revision.store_path}/bin/govuk",
       'system',
       'build',
       *hash_to_arguments(
-        DEFAULT_ARGUMENTS.merge(arguments)
+        DEFAULT_ARGUMENTS.merge(options[:arguments])
       ),
       *signon_user_arguments(mini_environment.signon_users),
-      *services
+      *options[:services]
     )
 
     build_output = output.last.strip
@@ -69,7 +87,7 @@ class GovukGuix::BuildJob < GovukGuix::Job
 
   def signon_user_arguments(signon_users)
     signon_users.map do |signon_user|
-      "--signon-user=#{signon_user_to_sexp(signon_user)}"
+      "--signon-user='#{signon_user_to_sexp(signon_user)}'"
     end
   end
 
diff --git a/app/jobs/govuk_guix/job.rb b/app/jobs/govuk_guix/job.rb
index c20b9de..b79d380 100644
--- a/app/jobs/govuk_guix/job.rb
+++ b/app/jobs/govuk_guix/job.rb
@@ -46,12 +46,14 @@ class GovukGuix::Job < Que::Job
 
   def hash_to_arguments(hash)
     hash.map do |(key, value)|
-      transfomed_key = key.tr('_', '-')
+      transformed_key = key.tr('_', '-')
 
       if value == true
-        "--#{transfomed_key}"
+        "--#{transformed_key}"
+      elsif value.include? ' '
+        "--#{transformed_key}='#{value}'"
       else
-        "--#{transfomed_key}=#{value}"
+        "--#{transformed_key}=#{value}"
       end
     end
   end
diff --git a/app/models/backends/terraform_aws.rb b/app/models/backends/terraform_aws.rb
index d45dfc7..c50e2de 100644
--- a/app/models/backends/terraform_aws.rb
+++ b/app/models/backends/terraform_aws.rb
@@ -55,11 +55,19 @@ class Backends::TerraformAws < ApplicationRecord
 
     GovukGuix::BuildJob.enqueue(
       mini_environment.id,
-      %w(whitehall government-frontend),
-      type: 'container-start-script',
-      app_domain: "#{slug}.#{domain}",
-      web_domain: "www.#{slug}.#{domain}",
-      use_https: 'certbot'
+      services: %w(whitehall government-frontend),
+      arguments: {
+        type: 'container-start-script',
+        app_domain: "#{slug}.#{domain}",
+        web_domain: "www.#{slug}.#{domain}",
+        use_https: 'certbot',
+        http_ports_mode: 'alternative',
+        read_bundle_install_input_as_tar_archive: true,
+        signon_instance_name: slug,
+        admin_environment_label: mini_environment.name,
+        read_bundle_install_input_as_tar_archive: 'true'
+      },
+      run_remotely_on_host: "ubuntu@guix-daemon.#{domain}"
     )
   end
 
@@ -83,10 +91,33 @@ class Backends::TerraformAws < ApplicationRecord
     end
   end
 
+  def deploy_backend
+    public_ip_address = ENV[
+      'GOVUK_MINI_ENVIRONMENT_ADMIN_PUBLIC_IP_ADDRESS'
+    ]
+
+    raise 'missing public ip address' if public_ip_address.nil?
+
+    TerraformWorkingDirectory.new(
+      terraform_state_id,
+      'terraform/aws/backend'
+    ).within_working_directory do
+      RubyTerraform.apply(
+        vars: common_terraform_variables.merge(
+          aws_vpc_id: vpc_id,
+          ssh_public_key: ssh_public_key,
+          mini_environment_admin_guix_public_key: guix_public_key,
+          mini_environment_admin_public_ip_address: public_ip_address
+        ),
+        auto_approve: true
+      )
+    end
+  end
+
   def within_terraform_working_directory(mini_environment, &block)
     TerraformWorkingDirectory.new(
       "mini_environment/#{mini_environment.id}",
-      'terraform/aws'
+      'terraform/aws/mini_environment'
     ).within_working_directory(&block)
   end
 
@@ -94,18 +125,41 @@ class Backends::TerraformAws < ApplicationRecord
     "https://signon.#{mini_environment.name.parameterize}.#{domain}"
   end
 
-  def terraform_variables(mini_environment)
+  def common_terraform_variables
     {
       aws_access_key: aws_access_key_id,
       aws_secret_key: aws_secret_access_key,
       aws_region: aws_region,
-      slug: mini_environment.name.parameterize,
       ssh_public_key: ssh_public_key,
-      start_command: mini_environment.backend_data['build_output']
+      aws_route_53_zone_id: route_53_zone_id,
+      aws_efs_file_system_id: efs_file_system_id
     }
   end
 
+  def terraform_variables(mini_environment)
+    common_terraform_variables.merge(
+      slug: mini_environment.name.parameterize,
+      start_command: mini_environment.backend_data['build_output'],
+      backend_remote_state_address: (
+        'http://localhost:3000' +
+        Rails
+          .application
+          .routes
+          .url_helpers
+          .terraform_http_backend_path(terraform_state_id)
+      )
+    )
+  end
+
+  def terraform_state_id
+    "backend/terraform_aws/#{id}"
+  end
+
   def ssh_public_key
     File.open("#{ENV['HOME']}/.ssh/id_rsa.pub", &:readline)
   end
+
+  def guix_public_key
+    "(entry #{File.read("/etc/guix/signing-key.pub")} (tag (guix import)))"
+  end
 end
diff --git a/app/models/backends/terraform_libvirt.rb b/app/models/backends/terraform_libvirt.rb
index b069b70..de29bed 100644
--- a/app/models/backends/terraform_libvirt.rb
+++ b/app/models/backends/terraform_libvirt.rb
@@ -50,17 +50,19 @@ class Backends::TerraformLibvirt < ApplicationRecord
 
     GovukGuix::BuildJob.enqueue(
       mini_environment.id,
-      %w(whitehall government-frontend),
-      type: 'vm-image-and-system',
-      app_domain: "#{slug}.#{domain}",
-      web_domain: "www.#{slug}.#{domain}",
-      # Assume that this is a local environment, and not externally
-      # accessible
-      use_https: 'development',
-      http_ports_mode: 'standard',
-      host_name: "#{slug}.#{domain}",
-      admin_environment_label: mini_environment.name,
-      signon_instance_name: slug
+      services: %w(whitehall government-frontend),
+      arguments: {
+        type: 'vm-image-and-system',
+        app_domain: "#{slug}.#{domain}",
+        web_domain: "www.#{slug}.#{domain}",
+        # Assume that this is a local environment, and not externally
+        # accessible
+        use_https: 'development',
+        http_ports_mode: 'standard',
+        host_name: "#{slug}.#{domain}",
+        admin_environment_label: mini_environment.name,
+        signon_instance_name: slug
+      }
     )
   end
 
diff --git a/terraform/aws/backend/guix-daemon.service.tpl b/terraform/aws/backend/guix-daemon.service.tpl
new file mode 100644
index 0000000..adf07a1
--- /dev/null
+++ b/terraform/aws/backend/guix-daemon.service.tpl
@@ -0,0 +1,17 @@
+[Unit]
+Description=Build daemon for GNU Guix
+
+[Service]
+ExecStart=/var/guix/profiles/per-user/root/guix-profile/bin/guix-daemon --build-users-group=guixbuild --disable-deduplication --substitute-urls="${substitute_servers}" --listen=0.0.0.0 --listen=/var/guix/daemon-socket/socket --max-jobs=16
+Environment=GUIX_LOCPATH=/root/.guix-profile/lib/locale
+RemainAfterExit=yes
+StandardOutput=syslog
+StandardError=syslog
+
+# See <https://lists.gnu.org/archive/html/guix-devel/2016-04/msg00608.html>.
+# Some package builds (for example, go@1.8.1) may require even more than
+# 1024 tasks.
+TasksMax=8192
+
+[Install]
+WantedBy=multi-user.target
diff --git a/terraform/aws/backend/main.tf b/terraform/aws/backend/main.tf
new file mode 100644
index 0000000..56b91b9
--- /dev/null
+++ b/terraform/aws/backend/main.tf
@@ -0,0 +1,302 @@
+terraform {
+  backend "http" {}
+}
+
+variable "aws_access_key" {
+  type = "string"
+}
+
+variable "aws_secret_key" {
+  type = "string"
+}
+
+variable "aws_region" {
+  type = "string"
+}
+
+variable "aws_vpc_id" {
+  type = "string"
+}
+
+variable "aws_route_53_zone_id" {
+  type = "string"
+}
+
+variable "aws_efs_file_system_id" {
+  type = "string"
+}
+
+variable "ssh_public_key" {
+  type = "string"
+}
+
+variable "guix_substitute_servers" {
+  type = "map"
+  default = {
+    "https://berlin.guixsd.org"    = <<EOF
+(entry
+ (public-key
+  (ecc
+   (curve Ed25519)
+   (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#)
+  )
+ )
+ (tag
+  (guix import)
+ )
+)
+EOF
+    "http://beid.cbaines.net"      = <<EOF
+(entry
+ (public-key
+  (ecc
+   (curve Ed25519)
+   (q #50349E83123851A65A569304A12E512B698223A81E10BEEC5A3E56EDAEE5DAC1#)
+  )
+ )
+ (tag
+  (guix import)
+ )
+)
+EOF
+  }
+}
+
+variable "mini_environment_admin_guix_public_key" {
+  type = "string"
+}
+
+variable "mini_environment_admin_public_ip_address" {
+  type = "string"
+}
+
+locals {
+  guix_daemon_substitute_servers = "${join(" ", keys(var.guix_substitute_servers))}"
+}
+
+provider "aws" {
+  access_key = "${var.aws_access_key}"
+  secret_key = "${var.aws_secret_key}"
+  region  = "${var.aws_region}"
+}
+
+data "aws_route53_zone" "main" {
+  zone_id = "${var.aws_route_53_zone_id}"
+}
+
+data "aws_efs_file_system" "main" {
+  file_system_id = "${var.aws_efs_file_system_id}"
+}
+
+data "template_file" "guix_daemon_service" {
+  template = "${file("${path.module}/guix-daemon.service.tpl")}"
+
+  vars {
+    substitute_servers = "${local.guix_daemon_substitute_servers}"
+  }
+}
+
+
+resource "aws_key_pair" "deployer" {
+  public_key = "${var.ssh_public_key}"
+}
+
+resource "aws_security_group" "public_webserver" {
+  name        = "govuk_mini_environment_admin_public_webserver"
+  description = "For instances running public facing web servers"
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port       = 0
+    to_port         = 80
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port       = 0
+    to_port         = 443
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port       = 0
+    to_port         = 8080
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port       = 0
+    to_port         = 8443
+    protocol        = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "ssh_access_from_mini_environment_admin" {
+  name        = "govuk_mini_environment_admin_ssh_access_from_mini_environment_admin"
+  description = "For instances that need SSH access for Terraform and Guix builds"
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port = 0
+    to_port = 22
+    protocol = "tcp"
+    cidr_blocks = ["${var.mini_environment_admin_public_ip_address}/32"]
+  }
+}
+
+resource "aws_security_group" "guix_client" {
+  name        = "govuk_mini_environment_admin_guix_client"
+  description = "For instances with access to the guix_daemon instance"
+  vpc_id      = "${var.aws_vpc_id}"
+
+  egress {
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "guix_daemon" {
+  name        = "govuk_mini_environment_admin_guix_daemon"
+  description = "For the guix_daemon instance."
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port       = 44146
+    to_port         = 44146
+    protocol        = "tcp"
+    security_groups = ["${aws_security_group.guix_client.id}"]
+  }
+
+  egress {
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_security_group" "efs_mount_target" {
+  name        = "govuk_mini_environment_admin_efs_mount_target"
+  description = "For the EFS File System mount targets"
+  vpc_id      = "${var.aws_vpc_id}"
+
+  ingress {
+    from_port       = 0
+    to_port         = 0
+    protocol        = "-1"
+    security_groups = [
+      "${aws_security_group.guix_client.id}",
+      "${aws_security_group.guix_daemon.id}",
+    ]
+  }
+}
+
+resource "aws_spot_instance_request" "main" {
+  ami             = "ami-8fd760f6"
+  instance_type   = "t2.medium"
+  key_name        = "${aws_key_pair.deployer.key_name}"
+  security_groups = [
+    "${aws_security_group.guix_daemon.name}",
+    "${aws_security_group.ssh_access_from_mini_environment_admin.name}"
+
+  ]
+
+  wait_for_fulfillment = true
+  spot_price = "0.05"
+
+  provisioner "file" {
+    content = "${data.template_file.guix_daemon_service.rendered}"
+    destination = "/home/ubuntu/guix-daemon.service"
+
+    connection {
+      type     = "ssh"
+      user     = "ubuntu"
+    }
+  }
+
+  provisioner "file" {
+    content = "(acl\n ${join("\n", values(var.guix_substitute_servers))}\n${var.mini_environment_admin_guix_public_key}\n)\n"
+    destination = "/home/ubuntu/acl"
+
+    connection {
+      type     = "ssh"
+      user     = "ubuntu"
+    }
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "sudo apt-get update",
+      "sudo apt-get update",
+      "sudo apt-get -y install nfs-common cachefilesd nscd",
+      "sudo tune2fs -o user_xattr /dev/xvda1",
+      "sudo sed 's/#RUN/RUN/' -i /etc/default/cachefilesd",
+      "echo \"${data.aws_efs_file_system.main.dns_name}:/var/guix /var/guix nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab",
+      "echo \"${data.aws_efs_file_system.main.dns_name}:/gnu/store /gnu/store nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,fsc,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab",
+      "echo \"${data.aws_efs_file_system.main.dns_name}:/ /mnt/efs nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,fsc,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab",
+      "sudo mkdir -p /var/guix /gnu/store /mnt/efs",
+      "sudo mount -a",
+      "sudo mv /home/ubuntu/guix-daemon.service /etc/systemd/system/guix-daemon.service",
+      "sudo mkdir /etc/guix",
+      "sudo mv /home/ubuntu/acl /etc/guix/acl",
+      "sudo groupadd --system guixbuild",
+      <<EOF
+for i in `seq -w 1 17`;
+do
+  sudo useradd -g guixbuild -G guixbuild           \
+          -d /var/empty -s `which nologin`    \
+          -c "Guix build user $i" --system    \
+          guixbuilder$i;
+done
+EOF
+      ,
+      "sudo systemctl daemon-reload",
+      "sudo systemctl enable guix-daemon.service",
+      "sudo systemctl start guix-daemon.service",
+      "ln -s /var/guix/profiles/per-user/ubuntu/guix-profile ~/.guix-profile",
+      # This is needed for things like guix copy to work
+      "echo 'GUIX_PROFILE=/home/ubuntu/.guix-profile; source /home/ubuntu/.guix-profile/etc/profile' | cat - .bashrc > temp && mv temp .bashrc"
+    ]
+
+    connection {
+      type     = "ssh"
+      user     = "ubuntu"
+    }
+  }
+}
+
+resource "aws_route53_record" "main" {
+  zone_id = "${data.aws_route53_zone.main.zone_id}"
+  name    = "guix-daemon"
+  type    = "A"
+  ttl     = "60"
+  records = ["${aws_spot_instance_request.main.public_ip}"]
+}
+
+# Outputs
+
+output "deployer_key_pair_name" {
+  value = "${aws_key_pair.deployer.key_name}"
+}
+
+output "guix_client_security_group_name" {
+  value = "${aws_security_group.guix_client.name}"
+}
+
+output "public_webserver_security_group_name" {
+  value = "${aws_security_group.public_webserver.name}"
+}
+
+output "ssh_access_from_mini_environment_admin_security_group_name" {
+  value = "${aws_security_group.ssh_access_from_mini_environment_admin.name}"
+}
+
+output "guix_daemon_private_dns" {
+  value = "${aws_spot_instance_request.main.private_dns}"
+}
diff --git a/terraform/aws/mini_environment.tf b/terraform/aws/mini_environment/main.tf
index 06bc0f3..4f9b9c3 100644
--- a/terraform/aws/mini_environment.tf
+++ b/terraform/aws/mini_environment/main.tf
@@ -18,7 +18,11 @@ variable "aws_region" {
   type = "string"
 }
 
-variable "ssh_public_key" {
+variable "aws_route_53_zone_id" {
+  type = "string"
+}
+
+variable "aws_efs_file_system_id" {
   type = "string"
 }
 
@@ -26,48 +30,49 @@ variable "start_command" {
   type = "string"
 }
 
+variable "backend_remote_state_address" {
+  type = "string"
+}
+
 provider "aws" {
   access_key = "${var.aws_access_key}"
   secret_key = "${var.aws_secret_key}"
   region  = "${var.aws_region}"
 }
 
-resource "aws_key_pair" "deployer" {
-  public_key = "${var.ssh_public_key}"
-}
-
-data "aws_security_group" "guix-client" {
-  id = "sg-d8003ba3"
-}
 
-data "aws_instance" "guix-daemon" {
-  instance_id = "i-010e25f85dfa73e72"
+data "terraform_remote_state" "backend" {
+  backend = "http"
+  config {
+    address = "${var.backend_remote_state_address}"
+  }
 }
 
 data "aws_route53_zone" "main" {
-  zone_id = "ZD004G8DN6AQZ"
+  zone_id = "${var.aws_route_53_zone_id}"
 }
 
 data "aws_efs_file_system" "main" {
-  file_system_id = "fs-81e05e48"
+  file_system_id = "${var.aws_efs_file_system_id}"
 }
 
 data "template_file" "govuk_service" {
-  template = "${file("${path.module}/mini_environment/govuk.service.tpl")}"
+  template = "${file("${path.module}/govuk.service.tpl")}"
 
   vars {
     start_command = "${var.start_command}"
   }
 }
 
-resource "aws_spot_instance_request" "example" {
+
+resource "aws_spot_instance_request" "main" {
   ami           = "ami-8fd760f6"
   instance_type = "t2.large"
-  key_name      = "${aws_key_pair.deployer.key_name}"
+  key_name      = "${data.terraform_remote_state.backend.deployer_key_pair_name}"
   security_groups = [
-    "${data.aws_security_group.guix-client.name}",
-    "default",
-    "public-webserver"
+    "${data.terraform_remote_state.backend.guix_client_security_group_name}",
+    "${data.terraform_remote_state.backend.public_webserver_security_group_name}",
+    "${data.terraform_remote_state.backend.ssh_access_from_mini_environment_admin_security_group_name}"
   ]
 
   wait_for_fulfillment = true
@@ -76,7 +81,7 @@ resource "aws_spot_instance_request" "example" {
   provisioner "file" {
     content = "${data.template_file.govuk_service.rendered}"
     destination = "/home/ubuntu/govuk.service"
-    
+
     connection {
       type     = "ssh"
       user     = "ubuntu"
@@ -94,10 +99,12 @@ resource "aws_spot_instance_request" "example" {
       "sudo mount -t nfs4 -o ro,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,actimeo=600,fsc,nocto,retrans=2 ${data.aws_efs_file_system.main.dns_name}:gnu/store /gnu/store",
       "sudo mkdir -p /var/guix",
       "sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 ${data.aws_efs_file_system.main.dns_name}:var/guix /var/guix",
-      "echo \"export GUIX_DAEMON_SOCKET=guix://${data.aws_instance.guix-daemon.private_dns}\" | sudo tee /etc/profile.d/guix-daemon-socket.sh",
+      "echo \"export GUIX_DAEMON_SOCKET=guix://${data.terraform_remote_state.backend.guix_daemon_private_dns}\" | sudo tee /etc/profile.d/guix-daemon-socket.sh",
       #"sudo systemctl restart cachefilesd",
       "sudo iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080",
       "sudo iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8443",
+      "sudo iptables -A OUTPUT -t nat -o lo -p tcp --dport 80 -j REDIRECT --to-port 8080",
+      "sudo iptables -A OUTPUT -t nat -o lo -p tcp --dport 443 -j REDIRECT --to-port 8443",
       "sudo mv /home/ubuntu/govuk.service /etc/systemd/system/govuk.service",
       "sudo systemctl daemon-reload",
       "sudo systemctl enable govuk.service",
@@ -111,28 +118,28 @@ resource "aws_spot_instance_request" "example" {
   }
 }
 
-resource "aws_route53_record" "example" {
+resource "aws_route53_record" "main" {
   zone_id = "${data.aws_route53_zone.main.zone_id}"
   name    = "${var.slug}"
   type    = "A"
   ttl     = "60"
-  records = ["${aws_spot_instance_request.example.public_ip}"]
+  records = ["${aws_spot_instance_request.main.public_ip}"]
 }
 
-resource "aws_route53_record" "example_wildcard" {
+resource "aws_route53_record" "wildcard" {
   zone_id = "${data.aws_route53_zone.main.zone_id}"
   name    = "*.${var.slug}"
   type    = "A"
   ttl     = "60"
-  records = ["${aws_spot_instance_request.example.public_ip}"]
+  records = ["${aws_spot_instance_request.main.public_ip}"]
 }
 
 # Outputs
 
 output "spot_bid_status" {
-  value = "${aws_spot_instance_request.example.spot_bid_status}"
+  value = "${aws_spot_instance_request.main.spot_bid_status}"
 }
 
 output "spot_request_status" {
-  value = "${aws_spot_instance_request.example.spot_request_state}"
+  value = "${aws_spot_instance_request.main.spot_request_state}"
 }