aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--app/jobs/govuk_guix/build_job.rb26
-rw-r--r--app/jobs/govuk_guix/job.rb8
-rw-r--r--app/models/backends/terraform_aws.rb72
-rw-r--r--app/models/backends/terraform_libvirt.rb24
-rw-r--r--terraform/aws/backend/guix-daemon.service.tpl17
-rw-r--r--terraform/aws/backend/main.tf302
-rw-r--r--terraform/aws/mini_environment/main.tf (renamed from terraform/aws/mini_environment.tf)59
7 files changed, 455 insertions, 53 deletions
diff --git a/app/jobs/govuk_guix/build_job.rb b/app/jobs/govuk_guix/build_job.rb
index a74b928..fb5e256 100644
--- a/app/jobs/govuk_guix/build_job.rb
+++ b/app/jobs/govuk_guix/build_job.rb
@@ -27,22 +27,40 @@ class GovukGuix::BuildJob < GovukGuix::Job
@retry_interval = 30
- def run(mini_environment_id, services, arguments)
+ def run(mini_environment_id, options)
logger.info(self.class) do
"Building mini environment #{mini_environment_id}"
end
mini_environment = MiniEnvironment.find(mini_environment_id)
+ remote_host = options[:run_remotely_on_host]
+
+ if remote_host
+ ssh_command = ['ssh', remote_host]
+
+ # Copy the revision to the remote host, to ensure it's available
+ # there
+ run_command(
+ 'guix',
+ 'copy',
+ "--to=#{remote_host}",
+ mini_environment.govuk_guix_revision.store_path
+ )
+ else
+ ssh_command = []
+ end
+
output = run_command(
+ *ssh_command,
"#{mini_environment.govuk_guix_revision.store_path}/bin/govuk",
'system',
'build',
*hash_to_arguments(
- DEFAULT_ARGUMENTS.merge(arguments)
+ DEFAULT_ARGUMENTS.merge(options[:arguments])
),
*signon_user_arguments(mini_environment.signon_users),
- *services
+ *options[:services]
)
build_output = output.last.strip
@@ -69,7 +87,7 @@ class GovukGuix::BuildJob < GovukGuix::Job
def signon_user_arguments(signon_users)
signon_users.map do |signon_user|
- "--signon-user=#{signon_user_to_sexp(signon_user)}"
+ "--signon-user='#{signon_user_to_sexp(signon_user)}'"
end
end
diff --git a/app/jobs/govuk_guix/job.rb b/app/jobs/govuk_guix/job.rb
index c20b9de..b79d380 100644
--- a/app/jobs/govuk_guix/job.rb
+++ b/app/jobs/govuk_guix/job.rb
@@ -46,12 +46,14 @@ class GovukGuix::Job < Que::Job
def hash_to_arguments(hash)
hash.map do |(key, value)|
- transfomed_key = key.tr('_', '-')
+ transformed_key = key.tr('_', '-')
if value == true
- "--#{transfomed_key}"
+ "--#{transformed_key}"
+ elsif value.include? ' '
+ "--#{transformed_key}='#{value}'"
else
- "--#{transfomed_key}=#{value}"
+ "--#{transformed_key}=#{value}"
end
end
end
diff --git a/app/models/backends/terraform_aws.rb b/app/models/backends/terraform_aws.rb
index d45dfc7..c50e2de 100644
--- a/app/models/backends/terraform_aws.rb
+++ b/app/models/backends/terraform_aws.rb
@@ -55,11 +55,19 @@ class Backends::TerraformAws < ApplicationRecord
GovukGuix::BuildJob.enqueue(
mini_environment.id,
- %w(whitehall government-frontend),
- type: 'container-start-script',
- app_domain: "#{slug}.#{domain}",
- web_domain: "www.#{slug}.#{domain}",
- use_https: 'certbot'
+ services: %w(whitehall government-frontend),
+ arguments: {
+ type: 'container-start-script',
+ app_domain: "#{slug}.#{domain}",
+ web_domain: "www.#{slug}.#{domain}",
+ use_https: 'certbot',
+ http_ports_mode: 'alternative',
+ read_bundle_install_input_as_tar_archive: true,
+ signon_instance_name: slug,
+ admin_environment_label: mini_environment.name,
+ read_bundle_install_input_as_tar_archive: 'true'
+ },
+ run_remotely_on_host: "ubuntu@guix-daemon.#{domain}"
)
end
@@ -83,10 +91,33 @@ class Backends::TerraformAws < ApplicationRecord
end
end
+ def deploy_backend
+ public_ip_address = ENV[
+ 'GOVUK_MINI_ENVIRONMENT_ADMIN_PUBLIC_IP_ADDRESS'
+ ]
+
+ raise 'missing public ip address' if public_ip_address.nil?
+
+ TerraformWorkingDirectory.new(
+ terraform_state_id,
+ 'terraform/aws/backend'
+ ).within_working_directory do
+ RubyTerraform.apply(
+ vars: common_terraform_variables.merge(
+ aws_vpc_id: vpc_id,
+ ssh_public_key: ssh_public_key,
+ mini_environment_admin_guix_public_key: guix_public_key,
+ mini_environment_admin_public_ip_address: public_ip_address
+ ),
+ auto_approve: true
+ )
+ end
+ end
+
def within_terraform_working_directory(mini_environment, &block)
TerraformWorkingDirectory.new(
"mini_environment/#{mini_environment.id}",
- 'terraform/aws'
+ 'terraform/aws/mini_environment'
).within_working_directory(&block)
end
@@ -94,18 +125,41 @@ class Backends::TerraformAws < ApplicationRecord
"https://signon.#{mini_environment.name.parameterize}.#{domain}"
end
- def terraform_variables(mini_environment)
+ def common_terraform_variables
{
aws_access_key: aws_access_key_id,
aws_secret_key: aws_secret_access_key,
aws_region: aws_region,
- slug: mini_environment.name.parameterize,
ssh_public_key: ssh_public_key,
- start_command: mini_environment.backend_data['build_output']
+ aws_route_53_zone_id: route_53_zone_id,
+ aws_efs_file_system_id: efs_file_system_id
}
end
+ def terraform_variables(mini_environment)
+ common_terraform_variables.merge(
+ slug: mini_environment.name.parameterize,
+ start_command: mini_environment.backend_data['build_output'],
+ backend_remote_state_address: (
+ 'http://localhost:3000' +
+ Rails
+ .application
+ .routes
+ .url_helpers
+ .terraform_http_backend_path(terraform_state_id)
+ )
+ )
+ end
+
+ def terraform_state_id
+ "backend/terraform_aws/#{id}"
+ end
+
def ssh_public_key
File.open("#{ENV['HOME']}/.ssh/id_rsa.pub", &:readline)
end
+
+ def guix_public_key
+ "(entry #{File.read("/etc/guix/signing-key.pub")} (tag (guix import)))"
+ end
end
diff --git a/app/models/backends/terraform_libvirt.rb b/app/models/backends/terraform_libvirt.rb
index b069b70..de29bed 100644
--- a/app/models/backends/terraform_libvirt.rb
+++ b/app/models/backends/terraform_libvirt.rb
@@ -50,17 +50,19 @@ class Backends::TerraformLibvirt < ApplicationRecord
GovukGuix::BuildJob.enqueue(
mini_environment.id,
- %w(whitehall government-frontend),
- type: 'vm-image-and-system',
- app_domain: "#{slug}.#{domain}",
- web_domain: "www.#{slug}.#{domain}",
- # Assume that this is a local environment, and not externally
- # accessible
- use_https: 'development',
- http_ports_mode: 'standard',
- host_name: "#{slug}.#{domain}",
- admin_environment_label: mini_environment.name,
- signon_instance_name: slug
+ services: %w(whitehall government-frontend),
+ arguments: {
+ type: 'vm-image-and-system',
+ app_domain: "#{slug}.#{domain}",
+ web_domain: "www.#{slug}.#{domain}",
+ # Assume that this is a local environment, and not externally
+ # accessible
+ use_https: 'development',
+ http_ports_mode: 'standard',
+ host_name: "#{slug}.#{domain}",
+ admin_environment_label: mini_environment.name,
+ signon_instance_name: slug
+ }
)
end
diff --git a/terraform/aws/backend/guix-daemon.service.tpl b/terraform/aws/backend/guix-daemon.service.tpl
new file mode 100644
index 0000000..adf07a1
--- /dev/null
+++ b/terraform/aws/backend/guix-daemon.service.tpl
@@ -0,0 +1,17 @@
+[Unit]
+Description=Build daemon for GNU Guix
+
+[Service]
+ExecStart=/var/guix/profiles/per-user/root/guix-profile/bin/guix-daemon --build-users-group=guixbuild --disable-deduplication --substitute-urls="${substitute_servers}" --listen=0.0.0.0 --listen=/var/guix/daemon-socket/socket --max-jobs=16
+Environment=GUIX_LOCPATH=/root/.guix-profile/lib/locale
+RemainAfterExit=yes
+StandardOutput=syslog
+StandardError=syslog
+
+# See <https://lists.gnu.org/archive/html/guix-devel/2016-04/msg00608.html>.
+# Some package builds (for example, go@1.8.1) may require even more than
+# 1024 tasks.
+TasksMax=8192
+
+[Install]
+WantedBy=multi-user.target
diff --git a/terraform/aws/backend/main.tf b/terraform/aws/backend/main.tf
new file mode 100644
index 0000000..56b91b9
--- /dev/null
+++ b/terraform/aws/backend/main.tf
@@ -0,0 +1,302 @@
+terraform {
+ backend "http" {}
+}
+
+variable "aws_access_key" {
+ type = "string"
+}
+
+variable "aws_secret_key" {
+ type = "string"
+}
+
+variable "aws_region" {
+ type = "string"
+}
+
+variable "aws_vpc_id" {
+ type = "string"
+}
+
+variable "aws_route_53_zone_id" {
+ type = "string"
+}
+
+variable "aws_efs_file_system_id" {
+ type = "string"
+}
+
+variable "ssh_public_key" {
+ type = "string"
+}
+
+variable "guix_substitute_servers" {
+ type = "map"
+ default = {
+ "https://berlin.guixsd.org" = <<EOF
+(entry
+ (public-key
+ (ecc
+ (curve Ed25519)
+ (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#)
+ )
+ )
+ (tag
+ (guix import)
+ )
+)
+EOF
+ "http://beid.cbaines.net" = <<EOF
+(entry
+ (public-key
+ (ecc
+ (curve Ed25519)
+ (q #50349E83123851A65A569304A12E512B698223A81E10BEEC5A3E56EDAEE5DAC1#)
+ )
+ )
+ (tag
+ (guix import)
+ )
+)
+EOF
+ }
+}
+
+variable "mini_environment_admin_guix_public_key" {
+ type = "string"
+}
+
+variable "mini_environment_admin_public_ip_address" {
+ type = "string"
+}
+
+locals {
+ guix_daemon_substitute_servers = "${join(" ", keys(var.guix_substitute_servers))}"
+}
+
+provider "aws" {
+ access_key = "${var.aws_access_key}"
+ secret_key = "${var.aws_secret_key}"
+ region = "${var.aws_region}"
+}
+
+data "aws_route53_zone" "main" {
+ zone_id = "${var.aws_route_53_zone_id}"
+}
+
+data "aws_efs_file_system" "main" {
+ file_system_id = "${var.aws_efs_file_system_id}"
+}
+
+data "template_file" "guix_daemon_service" {
+ template = "${file("${path.module}/guix-daemon.service.tpl")}"
+
+ vars {
+ substitute_servers = "${local.guix_daemon_substitute_servers}"
+ }
+}
+
+
+resource "aws_key_pair" "deployer" {
+ public_key = "${var.ssh_public_key}"
+}
+
+resource "aws_security_group" "public_webserver" {
+ name = "govuk_mini_environment_admin_public_webserver"
+ description = "For instances running public facing web servers"
+ vpc_id = "${var.aws_vpc_id}"
+
+ ingress {
+ from_port = 0
+ to_port = 80
+ protocol = "tcp"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+
+ ingress {
+ from_port = 0
+ to_port = 443
+ protocol = "tcp"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+
+ ingress {
+ from_port = 0
+ to_port = 8080
+ protocol = "tcp"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+
+ ingress {
+ from_port = 0
+ to_port = 8443
+ protocol = "tcp"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+}
+
+resource "aws_security_group" "ssh_access_from_mini_environment_admin" {
+ name = "govuk_mini_environment_admin_ssh_access_from_mini_environment_admin"
+ description = "For instances that need SSH access for Terraform and Guix builds"
+ vpc_id = "${var.aws_vpc_id}"
+
+ ingress {
+ from_port = 0
+ to_port = 22
+ protocol = "tcp"
+ cidr_blocks = ["${var.mini_environment_admin_public_ip_address}/32"]
+ }
+}
+
+resource "aws_security_group" "guix_client" {
+ name = "govuk_mini_environment_admin_guix_client"
+ description = "For instances with access to the guix_daemon instance"
+ vpc_id = "${var.aws_vpc_id}"
+
+ egress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+}
+
+resource "aws_security_group" "guix_daemon" {
+ name = "govuk_mini_environment_admin_guix_daemon"
+ description = "For the guix_daemon instance."
+ vpc_id = "${var.aws_vpc_id}"
+
+ ingress {
+ from_port = 44146
+ to_port = 44146
+ protocol = "tcp"
+ security_groups = ["${aws_security_group.guix_client.id}"]
+ }
+
+ egress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["0.0.0.0/0"]
+ }
+}
+
+resource "aws_security_group" "efs_mount_target" {
+ name = "govuk_mini_environment_admin_efs_mount_target"
+ description = "For the EFS File System mount targets"
+ vpc_id = "${var.aws_vpc_id}"
+
+ ingress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ security_groups = [
+ "${aws_security_group.guix_client.id}",
+ "${aws_security_group.guix_daemon.id}",
+ ]
+ }
+}
+
+resource "aws_spot_instance_request" "main" {
+ ami = "ami-8fd760f6"
+ instance_type = "t2.medium"
+ key_name = "${aws_key_pair.deployer.key_name}"
+ security_groups = [
+ "${aws_security_group.guix_daemon.name}",
+ "${aws_security_group.ssh_access_from_mini_environment_admin.name}"
+
+ ]
+
+ wait_for_fulfillment = true
+ spot_price = "0.05"
+
+ provisioner "file" {
+ content = "${data.template_file.guix_daemon_service.rendered}"
+ destination = "/home/ubuntu/guix-daemon.service"
+
+ connection {
+ type = "ssh"
+ user = "ubuntu"
+ }
+ }
+
+ provisioner "file" {
+ content = "(acl\n ${join("\n", values(var.guix_substitute_servers))}\n${var.mini_environment_admin_guix_public_key}\n)\n"
+ destination = "/home/ubuntu/acl"
+
+ connection {
+ type = "ssh"
+ user = "ubuntu"
+ }
+ }
+
+ provisioner "remote-exec" {
+ inline = [
+ "sudo apt-get update",
+ "sudo apt-get update",
+ "sudo apt-get -y install nfs-common cachefilesd nscd",
+ "sudo tune2fs -o user_xattr /dev/xvda1",
+ "sudo sed 's/#RUN/RUN/' -i /etc/default/cachefilesd",
+ "echo \"${data.aws_efs_file_system.main.dns_name}:/var/guix /var/guix nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab",
+ "echo \"${data.aws_efs_file_system.main.dns_name}:/gnu/store /gnu/store nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,fsc,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab",
+ "echo \"${data.aws_efs_file_system.main.dns_name}:/ /mnt/efs nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,fsc,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab",
+ "sudo mkdir -p /var/guix /gnu/store /mnt/efs",
+ "sudo mount -a",
+ "sudo mv /home/ubuntu/guix-daemon.service /etc/systemd/system/guix-daemon.service",
+ "sudo mkdir /etc/guix",
+ "sudo mv /home/ubuntu/acl /etc/guix/acl",
+ "sudo groupadd --system guixbuild",
+ <<EOF
+for i in `seq -w 1 17`;
+do
+ sudo useradd -g guixbuild -G guixbuild \
+ -d /var/empty -s `which nologin` \
+ -c "Guix build user $i" --system \
+ guixbuilder$i;
+done
+EOF
+ ,
+ "sudo systemctl daemon-reload",
+ "sudo systemctl enable guix-daemon.service",
+ "sudo systemctl start guix-daemon.service",
+ "ln -s /var/guix/profiles/per-user/ubuntu/guix-profile ~/.guix-profile",
+ # This is needed for things like guix copy to work
+ "echo 'GUIX_PROFILE=/home/ubuntu/.guix-profile; source /home/ubuntu/.guix-profile/etc/profile' | cat - .bashrc > temp && mv temp .bashrc"
+ ]
+
+ connection {
+ type = "ssh"
+ user = "ubuntu"
+ }
+ }
+}
+
+resource "aws_route53_record" "main" {
+ zone_id = "${data.aws_route53_zone.main.zone_id}"
+ name = "guix-daemon"
+ type = "A"
+ ttl = "60"
+ records = ["${aws_spot_instance_request.main.public_ip}"]
+}
+
+# Outputs
+
+output "deployer_key_pair_name" {
+ value = "${aws_key_pair.deployer.key_name}"
+}
+
+output "guix_client_security_group_name" {
+ value = "${aws_security_group.guix_client.name}"
+}
+
+output "public_webserver_security_group_name" {
+ value = "${aws_security_group.public_webserver.name}"
+}
+
+output "ssh_access_from_mini_environment_admin_security_group_name" {
+ value = "${aws_security_group.ssh_access_from_mini_environment_admin.name}"
+}
+
+output "guix_daemon_private_dns" {
+ value = "${aws_spot_instance_request.main.private_dns}"
+}
diff --git a/terraform/aws/mini_environment.tf b/terraform/aws/mini_environment/main.tf
index 06bc0f3..4f9b9c3 100644
--- a/terraform/aws/mini_environment.tf
+++ b/terraform/aws/mini_environment/main.tf
@@ -18,7 +18,11 @@ variable "aws_region" {
type = "string"
}
-variable "ssh_public_key" {
+variable "aws_route_53_zone_id" {
+ type = "string"
+}
+
+variable "aws_efs_file_system_id" {
type = "string"
}
@@ -26,48 +30,49 @@ variable "start_command" {
type = "string"
}
+variable "backend_remote_state_address" {
+ type = "string"
+}
+
provider "aws" {
access_key = "${var.aws_access_key}"
secret_key = "${var.aws_secret_key}"
region = "${var.aws_region}"
}
-resource "aws_key_pair" "deployer" {
- public_key = "${var.ssh_public_key}"
-}
-
-data "aws_security_group" "guix-client" {
- id = "sg-d8003ba3"
-}
-data "aws_instance" "guix-daemon" {
- instance_id = "i-010e25f85dfa73e72"
+data "terraform_remote_state" "backend" {
+ backend = "http"
+ config {
+ address = "${var.backend_remote_state_address}"
+ }
}
data "aws_route53_zone" "main" {
- zone_id = "ZD004G8DN6AQZ"
+ zone_id = "${var.aws_route_53_zone_id}"
}
data "aws_efs_file_system" "main" {
- file_system_id = "fs-81e05e48"
+ file_system_id = "${var.aws_efs_file_system_id}"
}
data "template_file" "govuk_service" {
- template = "${file("${path.module}/mini_environment/govuk.service.tpl")}"
+ template = "${file("${path.module}/govuk.service.tpl")}"
vars {
start_command = "${var.start_command}"
}
}
-resource "aws_spot_instance_request" "example" {
+
+resource "aws_spot_instance_request" "main" {
ami = "ami-8fd760f6"
instance_type = "t2.large"
- key_name = "${aws_key_pair.deployer.key_name}"
+ key_name = "${data.terraform_remote_state.backend.deployer_key_pair_name}"
security_groups = [
- "${data.aws_security_group.guix-client.name}",
- "default",
- "public-webserver"
+ "${data.terraform_remote_state.backend.guix_client_security_group_name}",
+ "${data.terraform_remote_state.backend.public_webserver_security_group_name}",
+ "${data.terraform_remote_state.backend.ssh_access_from_mini_environment_admin_security_group_name}"
]
wait_for_fulfillment = true
@@ -76,7 +81,7 @@ resource "aws_spot_instance_request" "example" {
provisioner "file" {
content = "${data.template_file.govuk_service.rendered}"
destination = "/home/ubuntu/govuk.service"
-
+
connection {
type = "ssh"
user = "ubuntu"
@@ -94,10 +99,12 @@ resource "aws_spot_instance_request" "example" {
"sudo mount -t nfs4 -o ro,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,actimeo=600,fsc,nocto,retrans=2 ${data.aws_efs_file_system.main.dns_name}:gnu/store /gnu/store",
"sudo mkdir -p /var/guix",
"sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 ${data.aws_efs_file_system.main.dns_name}:var/guix /var/guix",
- "echo \"export GUIX_DAEMON_SOCKET=guix://${data.aws_instance.guix-daemon.private_dns}\" | sudo tee /etc/profile.d/guix-daemon-socket.sh",
+ "echo \"export GUIX_DAEMON_SOCKET=guix://${data.terraform_remote_state.backend.guix_daemon_private_dns}\" | sudo tee /etc/profile.d/guix-daemon-socket.sh",
#"sudo systemctl restart cachefilesd",
"sudo iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080",
"sudo iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8443",
+ "sudo iptables -A OUTPUT -t nat -o lo -p tcp --dport 80 -j REDIRECT --to-port 8080",
+ "sudo iptables -A OUTPUT -t nat -o lo -p tcp --dport 443 -j REDIRECT --to-port 8443",
"sudo mv /home/ubuntu/govuk.service /etc/systemd/system/govuk.service",
"sudo systemctl daemon-reload",
"sudo systemctl enable govuk.service",
@@ -111,28 +118,28 @@ resource "aws_spot_instance_request" "example" {
}
}
-resource "aws_route53_record" "example" {
+resource "aws_route53_record" "main" {
zone_id = "${data.aws_route53_zone.main.zone_id}"
name = "${var.slug}"
type = "A"
ttl = "60"
- records = ["${aws_spot_instance_request.example.public_ip}"]
+ records = ["${aws_spot_instance_request.main.public_ip}"]
}
-resource "aws_route53_record" "example_wildcard" {
+resource "aws_route53_record" "wildcard" {
zone_id = "${data.aws_route53_zone.main.zone_id}"
name = "*.${var.slug}"
type = "A"
ttl = "60"
- records = ["${aws_spot_instance_request.example.public_ip}"]
+ records = ["${aws_spot_instance_request.main.public_ip}"]
}
# Outputs
output "spot_bid_status" {
- value = "${aws_spot_instance_request.example.spot_bid_status}"
+ value = "${aws_spot_instance_request.main.spot_bid_status}"
}
output "spot_request_status" {
- value = "${aws_spot_instance_request.example.spot_request_state}"
+ value = "${aws_spot_instance_request.main.spot_request_state}"
}