Start tracking which store paths are in use

This will enable garbage collection of the Guix store, without
removing things that are still in use.

17 files changed, 239 insertions, 4 deletions

diff --git a/app/controllers/backends/terraform_aws_controller.rb b/app/controllers/backends/terraform_aws_controller.rb
index e5b66d4..b09001d 100644
--- a/app/controllers/backends/terraform_aws_controller.rb
+++ b/app/controllers/backends/terraform_aws_controller.rb
@@ -84,6 +84,10 @@ class Backends::TerraformAwsController < ApplicationController
     redirect_to terraform_aws_backend_path(@backend)
   end
 
+  def in_use_store_paths
+    @backend = Backends::TerraformAws.find(params['id'])
+  end
+
   private
 
   def create_params
diff --git a/app/controllers/backends/terraform_libvirt_controller.rb b/app/controllers/backends/terraform_libvirt_controller.rb
index 4421146..1bf37cd 100644
--- a/app/controllers/backends/terraform_libvirt_controller.rb
+++ b/app/controllers/backends/terraform_libvirt_controller.rb
@@ -82,6 +82,10 @@ class Backends::TerraformLibvirtController < ApplicationController
     redirect_to terraform_libvirt_backend_path(@backend)
   end
 
+  def in_use_store_paths
+    @backend = Backends::TerraformLibvirt.find(params['id'])
+  end
+
   private
 
   def create_params
diff --git a/app/jobs/govuk_guix/create_data_snapshot_job.rb b/app/jobs/govuk_guix/create_data_snapshot_job.rb
index 88a5ba4..bcc8aa1 100644
--- a/app/jobs/govuk_guix/create_data_snapshot_job.rb
+++ b/app/jobs/govuk_guix/create_data_snapshot_job.rb
@@ -87,6 +87,8 @@ class GovukGuix::CreateDataSnapshotJob < Que::Job
       from_remote_host: remote_host
     )
 
+    backend.add_in_use_store_path(build_output)
+
     GovukGuix::DataSnapshot.transaction do
       GovukGuix::DataSnapshot.create!(data_snapshot_fields)
 
diff --git a/app/jobs/govuk_guix/fetch_revision_job.rb b/app/jobs/govuk_guix/fetch_revision_job.rb
index fd6e09c..7b1acdf 100644
--- a/app/jobs/govuk_guix/fetch_revision_job.rb
+++ b/app/jobs/govuk_guix/fetch_revision_job.rb
@@ -27,9 +27,11 @@ class GovukGuix::FetchRevisionJob < Que::Job
   def run(commit_hash, options = {})
     backend_type_and_id = options[:backend_type_and_id]
     if backend_type_and_id
-      remote_host = Backends.find_by_type_and_id(
+      backend = Backends.find_by_type_and_id(
         *backend_type_and_id.split('=')
-      ).build_remote_host
+      )
+
+      remote_host = backend.build_remote_host
     end
 
     sha = fetch_and_checkout(commit_hash, remote_host)
@@ -51,6 +53,8 @@ class GovukGuix::FetchRevisionJob < Que::Job
     store_path = output.last.strip
     logger.debug(self.class) { "store_path: #{store_path}" }
 
+    backend.add_in_use_store_path(store_path) if backend
+
     GovukGuix::Revision.transaction do
       GovukGuix::Revision.create(
         commit_hash: sha,
diff --git a/app/models/backends/terraform_aws/backend_methods.rb b/app/models/backends/terraform_aws/backend_methods.rb
index cc5ca62..d6665a1 100644
--- a/app/models/backends/terraform_aws/backend_methods.rb
+++ b/app/models/backends/terraform_aws/backend_methods.rb
@@ -80,6 +80,28 @@ module Backends::TerraformAws::BackendMethods
     end
   end
 
+  def in_use_store_paths
+    [
+      GovukGuix::Revision.pluck(:store_path),
+      available_data_snapshots.pluck(:store_path),
+      mini_environments.pluck(:backend_data).map { |x| x&.dig('build_output') }
+    ].flatten.compact
+  end
+
+  def update_guix_gcroots
+    GovukGuix::UpdateGcrootsDirectory.set_in_use_store_paths(
+      in_use_store_paths,
+      run_remotely_on_host: build_remote_host
+    )
+  end
+
+  def add_in_use_store_path(store_path)
+    GovukGuix::UpdateGcrootsDirectory.add_store_path(
+      store_path,
+      run_remotely_on_host: build_remote_host
+    )
+  end
+
   def within_backend_terraform_working_directory(&block)
     with_advisory_lock(
       "terraform"
diff --git a/app/models/backends/terraform_aws/mini_environment_methods.rb b/app/models/backends/terraform_aws/mini_environment_methods.rb
index d0a8ab5..63efcd7 100644
--- a/app/models/backends/terraform_aws/mini_environment_methods.rb
+++ b/app/models/backends/terraform_aws/mini_environment_methods.rb
@@ -22,7 +22,7 @@ module Backends::TerraformAws::MiniEnvironmentMethods
   def build(mini_environment)
     slug = mini_environment.name.parameterize
 
-    GovukGuix::BuildMiniEnvironment.build(
+    store_path = GovukGuix::BuildMiniEnvironment.build(
       mini_environment.id,
       services: mini_environment.services.map(&:build_argument_string),
       arguments: {
@@ -44,6 +44,8 @@ module Backends::TerraformAws::MiniEnvironmentMethods
       },
       run_remotely_on_host: mini_environment.backend.build_remote_host
     )
+
+    add_in_use_store_path(store_path)
   end
 
   def start(mini_environment)
diff --git a/app/models/backends/terraform_libvirt/backend_methods.rb b/app/models/backends/terraform_libvirt/backend_methods.rb
index d14affa..85cd427 100644
--- a/app/models/backends/terraform_libvirt/backend_methods.rb
+++ b/app/models/backends/terraform_libvirt/backend_methods.rb
@@ -54,6 +54,26 @@ module Backends::TerraformLibvirt::BackendMethods
     end
   end
 
+  def in_use_store_paths
+    [
+      GovukGuix::Revision.pluck(:store_path),
+      available_data_snapshots.pluck(:store_path),
+      mini_environments.pluck(:backend_data).map { |x| x.dig('build_output') }
+    ].flatten.compact
+  end
+
+  def update_guix_gcroots
+    GovukGuix::UpdateGcrootsDirectory.set_in_use_store_paths(
+      in_use_store_paths
+    )
+  end
+
+  def add_in_use_store_path(store_path)
+    GovukGuix::UpdateGcrootsDirectory.add_store_path(
+      store_path
+    )
+  end
+
   def within_backend_terraform_working_directory(&block)
     with_advisory_lock(
       "terraform"
diff --git a/app/models/backends/terraform_libvirt/mini_environment_methods.rb b/app/models/backends/terraform_libvirt/mini_environment_methods.rb
index ef2b4c1..582aebb 100644
--- a/app/models/backends/terraform_libvirt/mini_environment_methods.rb
+++ b/app/models/backends/terraform_libvirt/mini_environment_methods.rb
@@ -22,7 +22,7 @@ module Backends::TerraformLibvirt::MiniEnvironmentMethods
   def build(mini_environment)
     slug = mini_environment.name.parameterize
 
-    GovukGuix::BuildMiniEnvironment.build(
+    store_path = GovukGuix::BuildMiniEnvironment.build(
       mini_environment.id,
       services: mini_environment.services.map(&:build_argument_string),
       arguments: {
@@ -41,6 +41,8 @@ module Backends::TerraformLibvirt::MiniEnvironmentMethods
         data_snapshot: mini_environment.data_snapshot.try(:store_path)
       }.compact
     )
+
+    add_in_use_store_path(store_path)
   end
 
   def start(mini_environment)
diff --git a/app/services/govuk_guix/build_mini_environment.rb b/app/services/govuk_guix/build_mini_environment.rb
index 01920eb..9cada20 100644
--- a/app/services/govuk_guix/build_mini_environment.rb
+++ b/app/services/govuk_guix/build_mini_environment.rb
@@ -83,6 +83,8 @@ module GovukGuix::BuildMiniEnvironment
         build_output: build_output
       }
     )
+
+    build_output
   end
 
   def self.signon_user_to_sexp(signon_user)
diff --git a/app/services/govuk_guix/update_gcroots_directory.rb b/app/services/govuk_guix/update_gcroots_directory.rb
new file mode 100644
index 0000000..97fe703
--- /dev/null
+++ b/app/services/govuk_guix/update_gcroots_directory.rb
@@ -0,0 +1,62 @@
+# GOV.UK Mini Environment Admin
+# Copyright © 2018  Christopher Baines <mail@cbaines.net>
+#
+# This file is part of the GOV.UK Mini Environment Admin.
+#
+# The GOV.UK Mini Environment Admin is free software: you can
+# redistribute it and/or modify it under the terms of the GNU Affero
+# General Public License as published by the Free Software Foundation,
+# either version 3 of the License, or (at your option) any later
+# version.
+#
+# The GOV.UK Mini Environment Admin is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with the GOV.UK Mini Environment Admin.  If not, see
+# <http://www.gnu.org/licenses/>.
+
+module GovukGuix::UpdateGcrootsDirectory
+  extend ::ShellUtils
+
+  DIRECTORY = '/var/guix/gcroots/govuk-mini-environment-admin'
+
+  def self.set_in_use_store_paths(store_paths, options = {})
+    current_store_paths = list_store_paths(options)
+
+    (current_store_paths - store_paths).each do |store_path|
+      remove_store_path(store_path, options)
+    end
+
+    (store_paths - current_store_paths).each do |store_path|
+      add_store_path(store_path, options)
+    end
+  end
+
+  def self.add_store_path(store_path, options = {})
+    run_command(
+      'ln', '-s',
+      store_path,
+      File.join(DIRECTORY, store_path['/gnu/store/'.length..-1]),
+      run_remotely_on_host: options[:run_remotely_on_host]
+    )
+  end
+
+  def self.remove_store_path(store_path, options = {})
+    run_command(
+      'rm', File.join(DIRECTORY, store_path['/gnu/store/'.length..-1]),
+      run_remotely_on_host: options[:run_remotely_on_host]
+    )
+  end
+
+  def self.list_store_paths(options = {})
+    run_command(
+      'ls', '-1', DIRECTORY,
+      run_remotely_on_host: options[:run_remotely_on_host]
+    ).map do |store_path_suffix|
+      "/gnu/store/#{store_path_suffix.strip}"
+    end
+  end
+end
diff --git a/app/views/backends/terraform_aws/in_use_store_paths.html.erb b/app/views/backends/terraform_aws/in_use_store_paths.html.erb
new file mode 100644
index 0000000..c3b694c
--- /dev/null
+++ b/app/views/backends/terraform_aws/in_use_store_paths.html.erb
@@ -0,0 +1,40 @@
+<%#
+
+GOV.UK Mini Environment Admin
+Copyright © 2018  Christopher Baines <mail@cbaines.net>
+
+This file is part of the GOV.UK Mini Environment Admin.
+
+The GOV.UK Mini Environment Admin is free software: you can
+redistribute it and/or modify it under the terms of the GNU Affero
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later
+version.
+
+The GOV.UK Mini Environment Admin is distributed in the hope that it
+will be useful, but WITHOUT ANY WARRANTY; without even the implied
+warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
+the GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public
+License along with the GOV.UK Mini Environment Admin.  If not, see
+<http://www.gnu.org/licenses/>.
+
+%>
+
+<h1>Backend: <%= @backend.label %></h1>
+<% status = @backend.status %>
+
+<br>
+
+<h3>In use store paths</h3>
+
+<table class="table table-striped">
+  <% @backend.in_use_store_paths.each do |store_path| %>
+    <tr>
+      <td><%= store_path %></td>
+      <td>
+      </td>
+    </tr>
+  <% end %>
+</table>
diff --git a/app/views/backends/terraform_libvirt/in_use_store_paths.html.erb b/app/views/backends/terraform_libvirt/in_use_store_paths.html.erb
new file mode 100644
index 0000000..c3b694c
--- /dev/null
+++ b/app/views/backends/terraform_libvirt/in_use_store_paths.html.erb
@@ -0,0 +1,40 @@
+<%#
+
+GOV.UK Mini Environment Admin
+Copyright © 2018  Christopher Baines <mail@cbaines.net>
+
+This file is part of the GOV.UK Mini Environment Admin.
+
+The GOV.UK Mini Environment Admin is free software: you can
+redistribute it and/or modify it under the terms of the GNU Affero
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later
+version.
+
+The GOV.UK Mini Environment Admin is distributed in the hope that it
+will be useful, but WITHOUT ANY WARRANTY; without even the implied
+warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
+the GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public
+License along with the GOV.UK Mini Environment Admin.  If not, see
+<http://www.gnu.org/licenses/>.
+
+%>
+
+<h1>Backend: <%= @backend.label %></h1>
+<% status = @backend.status %>
+
+<br>
+
+<h3>In use store paths</h3>
+
+<table class="table table-striped">
+  <% @backend.in_use_store_paths.each do |store_path| %>
+    <tr>
+      <td><%= store_path %></td>
+      <td>
+      </td>
+    </tr>
+  <% end %>
+</table>
diff --git a/config/routes.rb b/config/routes.rb
index 8e55628..8242395 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -19,6 +19,7 @@ Rails.application.routes.draw do
                 only: %i[create new show update destroy] do
         member do
           post 'perform_action'
+          get 'in_use_store_paths'
         end
       end
 
@@ -28,6 +29,7 @@ Rails.application.routes.draw do
                 only: %i[create new show update destroy] do
         member do
           post 'perform_action'
+          get 'in_use_store_paths'
         end
       end
     end
diff --git a/lib/tasks/backend.rake b/lib/tasks/backend.rake
index c23a3df..19dc462 100644
--- a/lib/tasks/backend.rake
+++ b/lib/tasks/backend.rake
@@ -6,6 +6,13 @@ namespace :backend do
         .find(args.backend_id)
         .deploy_backend
     end
+
+    desc 'Update GC Roots'
+    task :update_guix_gcroots, [:backend_id] => :environment do |t, args|
+      Backends::TerraformAws
+        .find(args.backend_id)
+        .update_guix_gcroots
+    end
   end
 
   namespace :terraform_aws do
@@ -22,5 +29,12 @@ namespace :backend do
         .find(args.backend_id)
         .destroy_backend
     end
+
+    desc 'Update GC Roots'
+    task :update_guix_gcroots, [:backend_id] => :environment do |t, args|
+      Backends::TerraformAws
+        .find(args.backend_id)
+        .update_guix_gcroots
+    end
   end
 end
diff --git a/test/controllers/mini_environments_controller_test.rb b/test/controllers/mini_environments_controller_test.rb
new file mode 100644
index 0000000..8176143
--- /dev/null
+++ b/test/controllers/mini_environments_controller_test.rb
@@ -0,0 +1,13 @@
+require 'integration_test_helper'
+
+class MiniEnvironmentsControllerTest < ActionDispatch::IntegrationTest
+  setup do
+    login_as User.new
+  end
+
+  test 'show' do
+    MiniEnvironment.create
+
+    get mini_environment_path
+  end
+end
diff --git a/test/models/backends/terraform_aws_test.rb b/test/models/backends/terraform_aws_test.rb
index f9a34f3..d782f3c 100644
--- a/test/models/backends/terraform_aws_test.rb
+++ b/test/models/backends/terraform_aws_test.rb
@@ -28,6 +28,7 @@ class Backends::TerraformAwsTest < ActiveSupport::TestCase
   end
 
   test 'build' do
+    GovukGuix::UpdateGcrootsDirectory.stubs(:add_store_path)
     GovukGuix::BuildMiniEnvironment.expects(:build)
 
     @backend.build(@mini_environment)
diff --git a/test/models/backends/terraform_libvirt_test.rb b/test/models/backends/terraform_libvirt_test.rb
index 056a1c4..fb56dba 100644
--- a/test/models/backends/terraform_libvirt_test.rb
+++ b/test/models/backends/terraform_libvirt_test.rb
@@ -17,6 +17,7 @@ class Backends::TerraformLibvirtTest < ActiveSupport::TestCase
   end
 
   test 'build' do
+    GovukGuix::UpdateGcrootsDirectory.stubs(:add_store_path)
     GovukGuix::BuildMiniEnvironment.expects(:build)
 
     @backend.build(@mini_environment)