From 97f6d7e326a1fd7ddcbe7cf32dc0e4c0a508913d Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Sun, 28 Jan 2018 14:46:03 +0000 Subject: Add terraform backend and initial guix configuration --- .gitignore | 2 + .../terraform_http_backend_controller.rb | 25 +++++ app/jobs/setup_job.rb | 19 ++++ app/models/mini_environment.rb | 4 + app/models/terraform_state.rb | 2 + app/views/mini_environments/show.html.erb | 12 +++ config/initializers/que.rb | 1 + config/routes.rb | 6 +- .../20180128125638_create_terraform_states.rb | 9 ++ db/schema.rb | 8 +- guix.scm | 38 ++++++++ terraform/aws/mini_environment.tf | 104 +++++++++++++++++++++ terraform/aws/mini_environment/govuk.service.tpl | 13 +++ terraform/example.tf | 95 ------------------- terraform/example/govuk.service.tpl | 13 --- 15 files changed, 241 insertions(+), 110 deletions(-) create mode 100644 app/controllers/terraform_http_backend_controller.rb create mode 100644 app/models/terraform_state.rb create mode 100644 config/initializers/que.rb create mode 100644 db/migrate/20180128125638_create_terraform_states.rb create mode 100644 guix.scm create mode 100644 terraform/aws/mini_environment.tf create mode 100644 terraform/aws/mini_environment/govuk.service.tpl delete mode 100644 terraform/example.tf delete mode 100644 terraform/example/govuk.service.tpl diff --git a/.gitignore b/.gitignore index 68bdf90..4c26e9e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ /yarn-error.log .byebug_history + +.terraform diff --git a/app/controllers/terraform_http_backend_controller.rb b/app/controllers/terraform_http_backend_controller.rb new file mode 100644 index 0000000..6b94d8d --- /dev/null +++ b/app/controllers/terraform_http_backend_controller.rb @@ -0,0 +1,25 @@ +class TerraformHttpBackendController < ApplicationController + skip_before_action :verify_authenticity_token + + def create + TerraformState.create(data: params) + + render json: '{ "success": true }', status: 200 + end + + def show + state = TerraformState.last + + if state.nil? + render json: '{}', status: 404 + else + render json: state.data, status: 200 + end + end + + def destroy + TerraformState.delete_all + + render json: '{ "success": true }', status: 200 + end +end diff --git a/app/jobs/setup_job.rb b/app/jobs/setup_job.rb index c7b1b80..6b4f9fe 100644 --- a/app/jobs/setup_job.rb +++ b/app/jobs/setup_job.rb @@ -1,5 +1,24 @@ +require 'ruby_terraform' + class SetupJob < TerraformJob def run_terraform puts "Setting up #{@mini_environment.name}" + + RubyTerraform.init( + source: 'terraform/aws', + path: 'terraform/aws', + backend: true, + backend_config: { + address: 'http://localhost:3000' + Rails.application.routes.url_helpers.terraform_http_backend_path + } + ) + + RubyTerraform.apply( + directory: 'terraform/aws', + vars: { + slug: @mini_environment.name.parameterize + }, + auto_approve: true + ) end end diff --git a/app/models/mini_environment.rb b/app/models/mini_environment.rb index d53a8b1..2d6d8ae 100644 --- a/app/models/mini_environment.rb +++ b/app/models/mini_environment.rb @@ -1,3 +1,7 @@ class MiniEnvironment < ApplicationRecord has_many :finished_terraform_jobs, dependent: :destroy + + def enqueued_terraform_jobs + Que.execute("SELECT * FROM que_jobs WHERE args->>0 = '#{id}'") + end end diff --git a/app/models/terraform_state.rb b/app/models/terraform_state.rb new file mode 100644 index 0000000..fe1d364 --- /dev/null +++ b/app/models/terraform_state.rb @@ -0,0 +1,2 @@ +class TerraformState < ApplicationRecord +end diff --git a/app/views/mini_environments/show.html.erb b/app/views/mini_environments/show.html.erb index 2d8988c..5b2199e 100644 --- a/app/views/mini_environments/show.html.erb +++ b/app/views/mini_environments/show.html.erb @@ -1,6 +1,18 @@

Name: <%= @mini_environment.name %>

+

Finished jobs

+ <% @mini_environment.finished_terraform_jobs.each do |job| %> <%= job.job_class %> <%= job.created_at %> <% end %> + +

Enqueued jobs

+ +<% @mini_environment.enqueued_terraform_jobs.each do |job| %> + <%= job['job_class'] %> + + 
+    <%= job['last_error'] %>
+
+<% end %> diff --git a/config/initializers/que.rb b/config/initializers/que.rb new file mode 100644 index 0000000..b9e5a68 --- /dev/null +++ b/config/initializers/que.rb @@ -0,0 +1 @@ +Que.mode = :off diff --git a/config/routes.rb b/config/routes.rb index 2296683..0dcd800 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -1,5 +1,9 @@ Rails.application.routes.draw do root :to => 'mini_environments#index' - resources :mini_environments, :path => '/' + resource :terraform_http_backend, + controller: :terraform_http_backend, + except: %i[new edit update] + + resources :mini_environments, path: '/' end diff --git a/db/migrate/20180128125638_create_terraform_states.rb b/db/migrate/20180128125638_create_terraform_states.rb new file mode 100644 index 0000000..1882af9 --- /dev/null +++ b/db/migrate/20180128125638_create_terraform_states.rb @@ -0,0 +1,9 @@ +class CreateTerraformStates < ActiveRecord::Migration[5.1] + def change + create_table :terraform_states do |t| + t.json :data + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 70325bb..a20083e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180127222948) do +ActiveRecord::Schema.define(version: 20180128125638) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -45,6 +45,12 @@ ActiveRecord::Schema.define(version: 20180127222948) do t.text "queue", default: "", null: false end + create_table "terraform_states", force: :cascade do |t| + t.json "data" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + end + create_table "users", force: :cascade do |t| t.string "name" t.string "email" diff --git a/guix.scm b/guix.scm new file mode 100644 index 0000000..332ed1e --- /dev/null +++ b/guix.scm @@ -0,0 +1,38 @@ +(use-modules (guix packages) + (guix build-system ruby) + (gnu packages ruby) + (gnu packages databases) + (gnu packages rails) + (gnu packages rails-wip) + (gds packages govuk ruby)) + +(define govuk-mini-environment-admin + (package + (name "govuk-mini-environment-admin") + (version "0") + (source #f) + (build-system ruby-build-system) + (inputs + `(("ruby-rails" ,ruby-rails) + ("ruby-pg" ,ruby-pg) + ("ruby-gds-sso" ,ruby-gds-sso) + ("ruby-govuk-admin-template" ,ruby-govuk-admin-template) + ("ruby-plek" ,ruby-plek) + ("ruby-terraform" ,ruby-terraform) + ("ruby-que" ,ruby-que))) + (synopsis "") + (description "") + (home-page "") + (license ""))) + +(define govuk-mini-environment-admin-development-environment + (package + (inherit govuk-mini-environment-admin) + (inputs + `(,@(package-inputs govuk-mini-environment-admin) + ("ruby" ,ruby) + ("postgresql" ,postgresql))))) + +govuk-mini-environment-admin-development-environment + + diff --git a/terraform/aws/mini_environment.tf b/terraform/aws/mini_environment.tf new file mode 100644 index 0000000..56c8a3c --- /dev/null +++ b/terraform/aws/mini_environment.tf @@ -0,0 +1,104 @@ +terraform { + backend "http" {} +} + +variable "slug" { + type = "string" +} + + +provider "aws" { + region = "eu-west-1" +} + +resource "aws_key_pair" "deployer" { + key_name = "deployer" + public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCwICJ0eU/M+373AwzuvFtr+xCQdIQfK8CgbroCVMR2nezt/M+I8TIHtY9eb7M7J9Wnpgo+ObVbEmLXZeyzjme4BVBEBCUfKnkxmysqQFCb3NM96rLz509HFsKx/evo8Y+oazuW2L3vnLEKkqXq8jhL2YhlRWZwdNoEBa5N6Lsk/C4zwElAJKRkUDURLZcaNQOiTBtXh4lviX6Fj8rXjRgw/rZZ/fkWkLhP0RuS9V6Pw+f58sgFPkw19ZXj0LZNGHxeeCnyU7Ll0WlZa3WkDwbhNDvHJy6ZcIYZYHJicRKfbeCgBS7KRJvAlnW88au2wbU9t02H9INJyI5Mwua23X9v7tPvFLIFUOHIL5oCJEFYO4iM3lHTwrAov3UQ4/hsV/EVL1rQ1htDMt0QoXxQnJH0u7ThssGi1shJb34/F828lj1qPE4vjvoEyOLZs/pUwXbKHnKevQyiU159J/41shp7HNYh0d6eAeyFSnyvdfhvzx2TdKII0LaXdIgA3BYhN+j4ljNuN1BLEllJNb3u2L8FyyV/PA53k9XE8RdVU3JpE2m1u/49sgYiboruQzzQqelyuvBnajf/4q5wMQrJ8lf4PXp/oTwcvolJ/qiQ5qkfCx7sZojgLZlz1ReqsGAubEpZydme1Ujm5SGVkSSHC/Kx4sDADetJ3k6b1s4Y6w6cfw== chris@giedi" +} + +data "aws_security_group" "guix-client" { + id = "sg-d8003ba3" +} + +data "aws_instance" "guix-daemon" { + instance_id = "i-010e25f85dfa73e72" +} + +data "aws_route53_zone" "main" { + zone_id = "ZD004G8DN6AQZ" +} + +data "template_file" "govuk_service" { + template = "file(example/govuk.service.tpl)" + + vars { + guix_daemon_socket = "guix://${data.aws_instance.guix-daemon.private_dns}", + app_domain = "${var.slug}.aws.cbaines.net", + web_domain = "www.${var.slug}.aws.cbaines.net" + } +} + +resource "aws_spot_instance_request" "example" { + ami = "ami-8fd760f6" + instance_type = "t2.large" + key_name = "${aws_key_pair.deployer.key_name}" + security_groups = [ + "${data.aws_security_group.guix-client.name}", + "default", + "public-webserver" + ] + + wait_for_fulfillment = true + spot_price = "0.05" + + provisioner "file" { + content = "${data.template_file.govuk_service.rendered}" + destination = "/home/ubuntu/govuk.service" + + connection { + type = "ssh" + user = "ubuntu" + } + } + + provisioner "remote-exec" { + inline = [ + "sudo apt-get update", + "sudo apt-get update", + "sudo apt-get -y install nfs-common cachefilesd", + "sudo tune2fs -o user_xattr /dev/xvda1", + "sudo sed 's/#RUN/RUN/' -i /etc/default/cachefilesd", + "sudo mkdir -p /gnu/store", + "sudo mount -t nfs4 -o ro,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,actimeo=600,fsc,nocto,retrans=2 fs-81e05e48.efs.eu-west-1.amazonaws.com:gnu/store /gnu/store", + "sudo mkdir -p /var/guix", + "sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 fs-81e05e48.efs.eu-west-1.amazonaws.com:var/guix /var/guix", + "echo \"export GUIX_DAEMON_SOCKET=guix://${data.aws_instance.guix-daemon.private_dns}\" | sudo tee /etc/profile.d/guix-daemon-socket.sh", + #"sudo systemctl restart cachefilesd", + "sudo mv /home/ubuntu/govuk.service /etc/systemd/system/govuk.service", + "sudo systemctl daemon-reload", + "sudo systemctl enable govuk.service", + "sudo systemctl start govuk.service" + ] + + connection { + type = "ssh" + user = "ubuntu" + } + } +} + +resource "aws_route53_record" "example" { + zone_id = "${data.aws_route53_zone.main.zone_id}" + name = "${var.slug}" + type = "A" + ttl = "60" + records = ["${aws_spot_instance_request.example.public_ip}"] +} + +resource "aws_route53_record" "example_wildcard" { + zone_id = "${data.aws_route53_zone.main.zone_id}" + name = "*.${var.slug}" + type = "A" + ttl = "60" + records = ["${aws_spot_instance_request.example.public_ip}"] +} diff --git a/terraform/aws/mini_environment/govuk.service.tpl b/terraform/aws/mini_environment/govuk.service.tpl new file mode 100644 index 0000000..52b4b56 --- /dev/null +++ b/terraform/aws/mini_environment/govuk.service.tpl @@ -0,0 +1,13 @@ +[Unit] +Description=GOV.UK +After=network.target + +[Service] +Type=simple +User=ubuntu +WorkingDirectory=/home/ubuntu +Environment="GUIX_DAEMON_SOCKET=guix://${guix_daemon_socket}" +ExecStart=/var/guix/profiles/per-user/ubuntu/guix-profile/bin/govuk system start --rails-environment=production --app-domain=${app_domain} --web-domain=${web_domain} --use-high-ports=false --use-https=certbot --fallback + +[Install] +WantedBy=multi-user.target diff --git a/terraform/example.tf b/terraform/example.tf deleted file mode 100644 index 0efeed8..0000000 --- a/terraform/example.tf +++ /dev/null @@ -1,95 +0,0 @@ -provider "aws" { - region = "eu-west-1" -} - -resource "aws_key_pair" "deployer" { - key_name = "deployer" - public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCwICJ0eU/M+373AwzuvFtr+xCQdIQfK8CgbroCVMR2nezt/M+I8TIHtY9eb7M7J9Wnpgo+ObVbEmLXZeyzjme4BVBEBCUfKnkxmysqQFCb3NM96rLz509HFsKx/evo8Y+oazuW2L3vnLEKkqXq8jhL2YhlRWZwdNoEBa5N6Lsk/C4zwElAJKRkUDURLZcaNQOiTBtXh4lviX6Fj8rXjRgw/rZZ/fkWkLhP0RuS9V6Pw+f58sgFPkw19ZXj0LZNGHxeeCnyU7Ll0WlZa3WkDwbhNDvHJy6ZcIYZYHJicRKfbeCgBS7KRJvAlnW88au2wbU9t02H9INJyI5Mwua23X9v7tPvFLIFUOHIL5oCJEFYO4iM3lHTwrAov3UQ4/hsV/EVL1rQ1htDMt0QoXxQnJH0u7ThssGi1shJb34/F828lj1qPE4vjvoEyOLZs/pUwXbKHnKevQyiU159J/41shp7HNYh0d6eAeyFSnyvdfhvzx2TdKII0LaXdIgA3BYhN+j4ljNuN1BLEllJNb3u2L8FyyV/PA53k9XE8RdVU3JpE2m1u/49sgYiboruQzzQqelyuvBnajf/4q5wMQrJ8lf4PXp/oTwcvolJ/qiQ5qkfCx7sZojgLZlz1ReqsGAubEpZydme1Ujm5SGVkSSHC/Kx4sDADetJ3k6b1s4Y6w6cfw== chris@giedi" -} - -data "aws_security_group" "guix-client" { - id = "sg-d8003ba3" -} - -data "aws_instance" "guix-daemon" { - instance_id = "i-010e25f85dfa73e72" -} - -data "aws_route53_zone" "main" { - zone_id = "ZD004G8DN6AQZ" -} - -data "template_file" "govuk_service" { - template = "file(example/govuk.service.tpl)" - - vars { - guix_daemon_socket = "guix://${data.aws_instance.guix-daemon.private_dns}", - app_domain = "banana.aws.cbaines.net", - web_domain = "www.banana.aws.cbaines.net" - } -} - -resource "aws_spot_instance_request" "example" { - ami = "ami-8fd760f6" - instance_type = "t2.large" - key_name = "${aws_key_pair.deployer.key_name}" - security_groups = [ - "${data.aws_security_group.guix-client.name}", - "default", - "public-webserver" - ] - - wait_for_fulfillment = true - spot_price = "0.05" - - provisioner "file" { - content = "${data.template_file.govuk_service.rendered}" - destination = "/home/ubuntu/govuk.service" - - connection { - type = "ssh" - user = "ubuntu" - } - } - - provisioner "remote-exec" { - inline = [ - "sudo apt-get update", - "sudo apt-get update", - "sudo apt-get -y install nfs-common cachefilesd", - "sudo tune2fs -o user_xattr /dev/xvda1", - "sudo sed 's/#RUN/RUN/' -i /etc/default/cachefilesd", - "sudo mkdir -p /gnu/store", - "sudo mount -t nfs4 -o ro,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,actimeo=600,fsc,nocto,retrans=2 fs-81e05e48.efs.eu-west-1.amazonaws.com:gnu/store /gnu/store", - "sudo mkdir -p /var/guix", - "sudo mount -t nfs4 -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 fs-81e05e48.efs.eu-west-1.amazonaws.com:var/guix /var/guix", - "echo \"export GUIX_DAEMON_SOCKET=guix://${data.aws_instance.guix-daemon.private_dns}\" | sudo tee /etc/profile.d/guix-daemon-socket.sh", - #"sudo systemctl restart cachefilesd", - "sudo mv /home/ubuntu/govuk.service /etc/systemd/system/govuk.service", - "sudo systemctl daemon-reload", - "sudo systemctl enable govuk.service", - "sudo systemctl start govuk.service" - ] - - connection { - type = "ssh" - user = "ubuntu" - } - } -} - -resource "aws_route53_record" "example" { - zone_id = "${data.aws_route53_zone.main.zone_id}" - name = "banana" - type = "A" - ttl = "60" - records = ["${aws_spot_instance_request.example.public_ip}"] -} - -resource "aws_route53_record" "example_wildcard" { - zone_id = "${data.aws_route53_zone.main.zone_id}" - name = "*.banana" - type = "A" - ttl = "60" - records = ["${aws_spot_instance_request.example.public_ip}"] -} diff --git a/terraform/example/govuk.service.tpl b/terraform/example/govuk.service.tpl deleted file mode 100644 index 52b4b56..0000000 --- a/terraform/example/govuk.service.tpl +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=GOV.UK -After=network.target - -[Service] -Type=simple -User=ubuntu -WorkingDirectory=/home/ubuntu -Environment="GUIX_DAEMON_SOCKET=guix://${guix_daemon_socket}" -ExecStart=/var/guix/profiles/per-user/ubuntu/guix-profile/bin/govuk system start --rails-environment=production --app-domain=${app_domain} --web-domain=${web_domain} --use-high-ports=false --use-https=certbot --fallback - -[Install] -WantedBy=multi-user.target -- cgit v1.2.3