diff options
| -rw-r--r-- | app/jobs/govuk_guix/build_job.rb | 26 | ||||
| -rw-r--r-- | app/jobs/govuk_guix/job.rb | 8 | ||||
| -rw-r--r-- | app/models/backends/terraform_aws.rb | 72 | ||||
| -rw-r--r-- | app/models/backends/terraform_libvirt.rb | 24 | ||||
| -rw-r--r-- | terraform/aws/backend/guix-daemon.service.tpl | 17 | ||||
| -rw-r--r-- | terraform/aws/backend/main.tf | 302 | ||||
| -rw-r--r-- | terraform/aws/mini_environment/main.tf (renamed from terraform/aws/mini_environment.tf) | 59 |
7 files changed, 455 insertions, 53 deletions
diff --git a/app/jobs/govuk_guix/build_job.rb b/app/jobs/govuk_guix/build_job.rb index a74b928..fb5e256 100644 --- a/app/jobs/govuk_guix/build_job.rb +++ b/app/jobs/govuk_guix/build_job.rb @@ -27,22 +27,40 @@ class GovukGuix::BuildJob < GovukGuix::Job @retry_interval = 30 - def run(mini_environment_id, services, arguments) + def run(mini_environment_id, options) logger.info(self.class) do "Building mini environment #{mini_environment_id}" end mini_environment = MiniEnvironment.find(mini_environment_id) + remote_host = options[:run_remotely_on_host] + + if remote_host + ssh_command = ['ssh', remote_host] + + # Copy the revision to the remote host, to ensure it's available + # there + run_command( + 'guix', + 'copy', + "--to=#{remote_host}", + mini_environment.govuk_guix_revision.store_path + ) + else + ssh_command = [] + end + output = run_command( + *ssh_command, "#{mini_environment.govuk_guix_revision.store_path}/bin/govuk", 'system', 'build', *hash_to_arguments( - DEFAULT_ARGUMENTS.merge(arguments) + DEFAULT_ARGUMENTS.merge(options[:arguments]) ), *signon_user_arguments(mini_environment.signon_users), - *services + *options[:services] ) build_output = output.last.strip @@ -69,7 +87,7 @@ class GovukGuix::BuildJob < GovukGuix::Job def signon_user_arguments(signon_users) signon_users.map do |signon_user| - "--signon-user=#{signon_user_to_sexp(signon_user)}" + "--signon-user='#{signon_user_to_sexp(signon_user)}'" end end diff --git a/app/jobs/govuk_guix/job.rb b/app/jobs/govuk_guix/job.rb index c20b9de..b79d380 100644 --- a/app/jobs/govuk_guix/job.rb +++ b/app/jobs/govuk_guix/job.rb @@ -46,12 +46,14 @@ class GovukGuix::Job < Que::Job def hash_to_arguments(hash) hash.map do |(key, value)| - transfomed_key = key.tr('_', '-') + transformed_key = key.tr('_', '-') if value == true - "--#{transfomed_key}" + "--#{transformed_key}" + elsif value.include? ' ' + "--#{transformed_key}='#{value}'" else - "--#{transfomed_key}=#{value}" + "--#{transformed_key}=#{value}" end end end diff --git a/app/models/backends/terraform_aws.rb b/app/models/backends/terraform_aws.rb index d45dfc7..c50e2de 100644 --- a/app/models/backends/terraform_aws.rb +++ b/app/models/backends/terraform_aws.rb @@ -55,11 +55,19 @@ class Backends::TerraformAws < ApplicationRecord GovukGuix::BuildJob.enqueue( mini_environment.id, - %w(whitehall government-frontend), - type: 'container-start-script', - app_domain: "#{slug}.#{domain}", - web_domain: "www.#{slug}.#{domain}", - use_https: 'certbot' + services: %w(whitehall government-frontend), + arguments: { + type: 'container-start-script', + app_domain: "#{slug}.#{domain}", + web_domain: "www.#{slug}.#{domain}", + use_https: 'certbot', + http_ports_mode: 'alternative', + read_bundle_install_input_as_tar_archive: true, + signon_instance_name: slug, + admin_environment_label: mini_environment.name, + read_bundle_install_input_as_tar_archive: 'true' + }, + run_remotely_on_host: "ubuntu@guix-daemon.#{domain}" ) end @@ -83,10 +91,33 @@ class Backends::TerraformAws < ApplicationRecord end end + def deploy_backend + public_ip_address = ENV[ + 'GOVUK_MINI_ENVIRONMENT_ADMIN_PUBLIC_IP_ADDRESS' + ] + + raise 'missing public ip address' if public_ip_address.nil? + + TerraformWorkingDirectory.new( + terraform_state_id, + 'terraform/aws/backend' + ).within_working_directory do + RubyTerraform.apply( + vars: common_terraform_variables.merge( + aws_vpc_id: vpc_id, + ssh_public_key: ssh_public_key, + mini_environment_admin_guix_public_key: guix_public_key, + mini_environment_admin_public_ip_address: public_ip_address + ), + auto_approve: true + ) + end + end + def within_terraform_working_directory(mini_environment, &block) TerraformWorkingDirectory.new( "mini_environment/#{mini_environment.id}", - 'terraform/aws' + 'terraform/aws/mini_environment' ).within_working_directory(&block) end @@ -94,18 +125,41 @@ class Backends::TerraformAws < ApplicationRecord "https://signon.#{mini_environment.name.parameterize}.#{domain}" end - def terraform_variables(mini_environment) + def common_terraform_variables { aws_access_key: aws_access_key_id, aws_secret_key: aws_secret_access_key, aws_region: aws_region, - slug: mini_environment.name.parameterize, ssh_public_key: ssh_public_key, - start_command: mini_environment.backend_data['build_output'] + aws_route_53_zone_id: route_53_zone_id, + aws_efs_file_system_id: efs_file_system_id } end + def terraform_variables(mini_environment) + common_terraform_variables.merge( + slug: mini_environment.name.parameterize, + start_command: mini_environment.backend_data['build_output'], + backend_remote_state_address: ( + 'http://localhost:3000' + + Rails + .application + .routes + .url_helpers + .terraform_http_backend_path(terraform_state_id) + ) + ) + end + + def terraform_state_id + "backend/terraform_aws/#{id}" + end + def ssh_public_key File.open("#{ENV['HOME']}/.ssh/id_rsa.pub", &:readline) end + + def guix_public_key + "(entry #{File.read("/etc/guix/signing-key.pub")} (tag (guix import)))" + end end diff --git a/app/models/backends/terraform_libvirt.rb b/app/models/backends/terraform_libvirt.rb index b069b70..de29bed 100644 --- a/app/models/backends/terraform_libvirt.rb +++ b/app/models/backends/terraform_libvirt.rb @@ -50,17 +50,19 @@ class Backends::TerraformLibvirt < ApplicationRecord GovukGuix::BuildJob.enqueue( mini_environment.id, - %w(whitehall government-frontend), - type: 'vm-image-and-system', - app_domain: "#{slug}.#{domain}", - web_domain: "www.#{slug}.#{domain}", - # Assume that this is a local environment, and not externally - # accessible - use_https: 'development', - http_ports_mode: 'standard', - host_name: "#{slug}.#{domain}", - admin_environment_label: mini_environment.name, - signon_instance_name: slug + services: %w(whitehall government-frontend), + arguments: { + type: 'vm-image-and-system', + app_domain: "#{slug}.#{domain}", + web_domain: "www.#{slug}.#{domain}", + # Assume that this is a local environment, and not externally + # accessible + use_https: 'development', + http_ports_mode: 'standard', + host_name: "#{slug}.#{domain}", + admin_environment_label: mini_environment.name, + signon_instance_name: slug + } ) end diff --git a/terraform/aws/backend/guix-daemon.service.tpl b/terraform/aws/backend/guix-daemon.service.tpl new file mode 100644 index 0000000..adf07a1 --- /dev/null +++ b/terraform/aws/backend/guix-daemon.service.tpl @@ -0,0 +1,17 @@ +[Unit] +Description=Build daemon for GNU Guix + +[Service] +ExecStart=/var/guix/profiles/per-user/root/guix-profile/bin/guix-daemon --build-users-group=guixbuild --disable-deduplication --substitute-urls="${substitute_servers}" --listen=0.0.0.0 --listen=/var/guix/daemon-socket/socket --max-jobs=16 +Environment=GUIX_LOCPATH=/root/.guix-profile/lib/locale +RemainAfterExit=yes +StandardOutput=syslog +StandardError=syslog + +# See <https://lists.gnu.org/archive/html/guix-devel/2016-04/msg00608.html>. +# Some package builds (for example, go@1.8.1) may require even more than +# 1024 tasks. +TasksMax=8192 + +[Install] +WantedBy=multi-user.target diff --git a/terraform/aws/backend/main.tf b/terraform/aws/backend/main.tf new file mode 100644 index 0000000..56b91b9 --- /dev/null +++ b/terraform/aws/backend/main.tf @@ -0,0 +1,302 @@ +terraform { + backend "http" {} +} + +variable "aws_access_key" { + type = "string" +} + +variable "aws_secret_key" { + type = "string" +} + +variable "aws_region" { + type = "string" +} + +variable "aws_vpc_id" { + type = "string" +} + +variable "aws_route_53_zone_id" { + type = "string" +} + +variable "aws_efs_file_system_id" { + type = "string" +} + +variable "ssh_public_key" { + type = "string" +} + +variable "guix_substitute_servers" { + type = "map" + default = { + "https://berlin.guixsd.org" = <<EOF +(entry + (public-key + (ecc + (curve Ed25519) + (q #8D156F295D24B0D9A86FA5741A840FF2D24F60F7B6C4134814AD55625971B394#) + ) + ) + (tag + (guix import) + ) +) +EOF + "http://beid.cbaines.net" = <<EOF +(entry + (public-key + (ecc + (curve Ed25519) + (q #50349E83123851A65A569304A12E512B698223A81E10BEEC5A3E56EDAEE5DAC1#) + ) + ) + (tag + (guix import) + ) +) +EOF + } +} + +variable "mini_environment_admin_guix_public_key" { + type = "string" +} + +variable "mini_environment_admin_public_ip_address" { + type = "string" +} + +locals { + guix_daemon_substitute_servers = "${join(" ", keys(var.guix_substitute_servers))}" +} + +provider "aws" { + access_key = "${var.aws_access_key}" + secret_key = "${var.aws_secret_key}" + region = "${var.aws_region}" +} + +data "aws_route53_zone" "main" { + zone_id = "${var.aws_route_53_zone_id}" +} + +data "aws_efs_file_system" "main" { + file_system_id = "${var.aws_efs_file_system_id}" +} + +data "template_file" "guix_daemon_service" { + template = "${file("${path.module}/guix-daemon.service.tpl")}" + + vars { + substitute_servers = "${local.guix_daemon_substitute_servers}" + } +} + + +resource "aws_key_pair" "deployer" { + public_key = "${var.ssh_public_key}" +} + +resource "aws_security_group" "public_webserver" { + name = "govuk_mini_environment_admin_public_webserver" + description = "For instances running public facing web servers" + vpc_id = "${var.aws_vpc_id}" + + ingress { + from_port = 0 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + from_port = 0 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + from_port = 0 + to_port = 8080 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + from_port = 0 + to_port = 8443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_security_group" "ssh_access_from_mini_environment_admin" { + name = "govuk_mini_environment_admin_ssh_access_from_mini_environment_admin" + description = "For instances that need SSH access for Terraform and Guix builds" + vpc_id = "${var.aws_vpc_id}" + + ingress { + from_port = 0 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["${var.mini_environment_admin_public_ip_address}/32"] + } +} + +resource "aws_security_group" "guix_client" { + name = "govuk_mini_environment_admin_guix_client" + description = "For instances with access to the guix_daemon instance" + vpc_id = "${var.aws_vpc_id}" + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_security_group" "guix_daemon" { + name = "govuk_mini_environment_admin_guix_daemon" + description = "For the guix_daemon instance." + vpc_id = "${var.aws_vpc_id}" + + ingress { + from_port = 44146 + to_port = 44146 + protocol = "tcp" + security_groups = ["${aws_security_group.guix_client.id}"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_security_group" "efs_mount_target" { + name = "govuk_mini_environment_admin_efs_mount_target" + description = "For the EFS File System mount targets" + vpc_id = "${var.aws_vpc_id}" + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + security_groups = [ + "${aws_security_group.guix_client.id}", + "${aws_security_group.guix_daemon.id}", + ] + } +} + +resource "aws_spot_instance_request" "main" { + ami = "ami-8fd760f6" + instance_type = "t2.medium" + key_name = "${aws_key_pair.deployer.key_name}" + security_groups = [ + "${aws_security_group.guix_daemon.name}", + "${aws_security_group.ssh_access_from_mini_environment_admin.name}" + + ] + + wait_for_fulfillment = true + spot_price = "0.05" + + provisioner "file" { + content = "${data.template_file.guix_daemon_service.rendered}" + destination = "/home/ubuntu/guix-daemon.service" + + connection { + type = "ssh" + user = "ubuntu" + } + } + + provisioner "file" { + content = "(acl\n ${join("\n", values(var.guix_substitute_servers))}\n${var.mini_environment_admin_guix_public_key}\n)\n" + destination = "/home/ubuntu/acl" + + connection { + type = "ssh" + user = "ubuntu" + } + } + + provisioner "remote-exec" { + inline = [ + "sudo apt-get update", + "sudo apt-get update", + "sudo apt-get -y install nfs-common cachefilesd nscd", + "sudo tune2fs -o user_xattr /dev/xvda1", + "sudo sed 's/#RUN/RUN/' -i /etc/default/cachefilesd", + "echo \"${data.aws_efs_file_system.main.dns_name}:/var/guix /var/guix nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab", + "echo \"${data.aws_efs_file_system.main.dns_name}:/gnu/store /gnu/store nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,fsc,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab", + "echo \"${data.aws_efs_file_system.main.dns_name}:/ /mnt/efs nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,fsc,hard,timeo=600,retrans=2 0 0\" | sudo tee -a /etc/fstab", + "sudo mkdir -p /var/guix /gnu/store /mnt/efs", + "sudo mount -a", + "sudo mv /home/ubuntu/guix-daemon.service /etc/systemd/system/guix-daemon.service", + "sudo mkdir /etc/guix", + "sudo mv /home/ubuntu/acl /etc/guix/acl", + "sudo groupadd --system guixbuild", + <<EOF +for i in `seq -w 1 17`; +do + sudo useradd -g guixbuild -G guixbuild \ + -d /var/empty -s `which nologin` \ + -c "Guix build user $i" --system \ + guixbuilder$i; +done +EOF + , + "sudo systemctl daemon-reload", + "sudo systemctl enable guix-daemon.service", + "sudo systemctl start guix-daemon.service", + "ln -s /var/guix/profiles/per-user/ubuntu/guix-profile ~/.guix-profile", + # This is needed for things like guix copy to work + "echo 'GUIX_PROFILE=/home/ubuntu/.guix-profile; source /home/ubuntu/.guix-profile/etc/profile' | cat - .bashrc > temp && mv temp .bashrc" + ] + + connection { + type = "ssh" + user = "ubuntu" + } + } +} + +resource "aws_route53_record" "main" { + zone_id = "${data.aws_route53_zone.main.zone_id}" + name = "guix-daemon" + type = "A" + ttl = "60" + records = ["${aws_spot_instance_request.main.public_ip}"] +} + +# Outputs + +output "deployer_key_pair_name" { + value = "${aws_key_pair.deployer.key_name}" +} + +output "guix_client_security_group_name" { + value = "${aws_security_group.guix_client.name}" +} + +output "public_webserver_security_group_name" { + value = "${aws_security_group.public_webserver.name}" +} + +output "ssh_access_from_mini_environment_admin_security_group_name" { + value = "${aws_security_group.ssh_access_from_mini_environment_admin.name}" +} + +output "guix_daemon_private_dns" { + value = "${aws_spot_instance_request.main.private_dns}" +} diff --git a/terraform/aws/mini_environment.tf b/terraform/aws/mini_environment/main.tf index 06bc0f3..4f9b9c3 100644 --- a/terraform/aws/mini_environment.tf +++ b/terraform/aws/mini_environment/main.tf @@ -18,7 +18,11 @@ variable "aws_region" { type = "string" } -variable "ssh_public_key" { +variable "aws_route_53_zone_id" { + type = "string" +} + +variable "aws_efs_file_system_id" { type = "string" } @@ -26,48 +30,49 @@ variable "start_command" { type = "string" } +variable "backend_remote_state_address" { + type = "string" +} + provider "aws" { access_key = "${var.aws_access_key}" secret_key = "${var.aws_secret_key}" region = "${var.aws_region}" } -resource "aws_key_pair" "deployer" { - public_key = "${var.ssh_public_key}" -} - -data "aws_security_group" "guix-client" { - id = "sg-d8003ba3" -} -data "aws_instance" "guix-daemon" { - instance_id = "i-010e25f85dfa73e72" +data "terraform_remote_state" "backend" { + backend = "http" + config { + address = "${var.backend_remote_state_address}" + } } data "aws_route53_zone" "main" { - zone_id = "ZD004G8DN6AQZ" + zone_id = "${var.aws_route_53_zone_id}" } data "aws_efs_file_system" "main" { - file_system_id = "fs-81e05e48" + file_system_id = "${var.aws_efs_file_system_id}" } data "template_file" "govuk_service" { - template = "${file("${path.module}/mini_environment/govuk.service.tpl")}" + template = "${file("${path.module}/govuk.service.tpl")}" vars { start_command = "${var.start_command}" } } -resource "aws_spot_instance_request" "example" { + +resource "aws_spot_instance_request" "main" { ami = "ami-8fd760f6" instance_type = "t2.large" - key_name = "${aws_key_pair.deployer.key_name}" + key_name = "${data.terraform_remote_state.backend.deployer_key_pair_name}" security_groups = [ - "${data.aws_security_group.guix-client.name}", - "default", - "public-webserver" + "${data.terraform_remote_state.backend.guix_client_security_group_name}", + "${data.terraform_remote_state.backend.public_webserver_security_group_name}", + "${data.terraform_remote_state.backend.ssh_access_from_mini_environment_admin_security_group_name}" ] wait_for_fulfillment = true @@ -76,7 +81,7 @@ resource "aws_spot_instance_request" "example" { provisioner "file" { content = "${data.template_file.govuk_service.rendered}" destination = "/home/ubuntu/govuk.service" - + connection { type = "ssh" user = "ubuntu" @@ -94,10 +99,12 @@ resource "aws_spot_instance_request" "example" { "sudo mount -t nfs4 -o ro,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,actimeo=600,fsc,nocto,retrans=2 ${data.aws_efs_file_system.main.dns_name}:gnu/store /gnu/store", "sudo mkdir -p /var/guix", "sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 ${data.aws_efs_file_system.main.dns_name}:var/guix /var/guix", - "echo \"export GUIX_DAEMON_SOCKET=guix://${data.aws_instance.guix-daemon.private_dns}\" | sudo tee /etc/profile.d/guix-daemon-socket.sh", + "echo \"export GUIX_DAEMON_SOCKET=guix://${data.terraform_remote_state.backend.guix_daemon_private_dns}\" | sudo tee /etc/profile.d/guix-daemon-socket.sh", #"sudo systemctl restart cachefilesd", "sudo iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 8080", "sudo iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8443", + "sudo iptables -A OUTPUT -t nat -o lo -p tcp --dport 80 -j REDIRECT --to-port 8080", + "sudo iptables -A OUTPUT -t nat -o lo -p tcp --dport 443 -j REDIRECT --to-port 8443", "sudo mv /home/ubuntu/govuk.service /etc/systemd/system/govuk.service", "sudo systemctl daemon-reload", "sudo systemctl enable govuk.service", @@ -111,28 +118,28 @@ resource "aws_spot_instance_request" "example" { } } -resource "aws_route53_record" "example" { +resource "aws_route53_record" "main" { zone_id = "${data.aws_route53_zone.main.zone_id}" name = "${var.slug}" type = "A" ttl = "60" - records = ["${aws_spot_instance_request.example.public_ip}"] + records = ["${aws_spot_instance_request.main.public_ip}"] } -resource "aws_route53_record" "example_wildcard" { +resource "aws_route53_record" "wildcard" { zone_id = "${data.aws_route53_zone.main.zone_id}" name = "*.${var.slug}" type = "A" ttl = "60" - records = ["${aws_spot_instance_request.example.public_ip}"] + records = ["${aws_spot_instance_request.main.public_ip}"] } # Outputs output "spot_bid_status" { - value = "${aws_spot_instance_request.example.spot_bid_status}" + value = "${aws_spot_instance_request.main.spot_bid_status}" } output "spot_request_status" { - value = "${aws_spot_instance_request.example.spot_request_state}" + value = "${aws_spot_instance_request.main.spot_request_state}" } |