From 3459059ed1f9332a9e71a4a7be5dc5e22e7f6bae Mon Sep 17 00:00:00 2001 From: Christopher Baines Date: Mon, 31 Dec 2018 18:22:12 +0000 Subject: Start tracking which store paths are in use This will enable garbage collection of the Guix store, without removing things that are still in use. --- .../backends/terraform_aws_controller.rb | 4 ++ .../backends/terraform_libvirt_controller.rb | 4 ++ app/jobs/govuk_guix/create_data_snapshot_job.rb | 2 + app/jobs/govuk_guix/fetch_revision_job.rb | 8 ++- .../backends/terraform_aws/backend_methods.rb | 22 ++++++++ .../terraform_aws/mini_environment_methods.rb | 4 +- .../backends/terraform_libvirt/backend_methods.rb | 20 +++++++ .../terraform_libvirt/mini_environment_methods.rb | 4 +- app/services/govuk_guix/build_mini_environment.rb | 2 + .../govuk_guix/update_gcroots_directory.rb | 62 ++++++++++++++++++++++ .../terraform_aws/in_use_store_paths.html.erb | 40 ++++++++++++++ .../terraform_libvirt/in_use_store_paths.html.erb | 40 ++++++++++++++ config/routes.rb | 2 + lib/tasks/backend.rake | 14 +++++ .../mini_environments_controller_test.rb | 13 +++++ test/models/backends/terraform_aws_test.rb | 1 + test/models/backends/terraform_libvirt_test.rb | 1 + 17 files changed, 239 insertions(+), 4 deletions(-) create mode 100644 app/services/govuk_guix/update_gcroots_directory.rb create mode 100644 app/views/backends/terraform_aws/in_use_store_paths.html.erb create mode 100644 app/views/backends/terraform_libvirt/in_use_store_paths.html.erb create mode 100644 test/controllers/mini_environments_controller_test.rb diff --git a/app/controllers/backends/terraform_aws_controller.rb b/app/controllers/backends/terraform_aws_controller.rb index e5b66d4..b09001d 100644 --- a/app/controllers/backends/terraform_aws_controller.rb +++ b/app/controllers/backends/terraform_aws_controller.rb @@ -84,6 +84,10 @@ class Backends::TerraformAwsController < ApplicationController redirect_to terraform_aws_backend_path(@backend) end + def in_use_store_paths + @backend = Backends::TerraformAws.find(params['id']) + end + private def create_params diff --git a/app/controllers/backends/terraform_libvirt_controller.rb b/app/controllers/backends/terraform_libvirt_controller.rb index 4421146..1bf37cd 100644 --- a/app/controllers/backends/terraform_libvirt_controller.rb +++ b/app/controllers/backends/terraform_libvirt_controller.rb @@ -82,6 +82,10 @@ class Backends::TerraformLibvirtController < ApplicationController redirect_to terraform_libvirt_backend_path(@backend) end + def in_use_store_paths + @backend = Backends::TerraformLibvirt.find(params['id']) + end + private def create_params diff --git a/app/jobs/govuk_guix/create_data_snapshot_job.rb b/app/jobs/govuk_guix/create_data_snapshot_job.rb index 88a5ba4..bcc8aa1 100644 --- a/app/jobs/govuk_guix/create_data_snapshot_job.rb +++ b/app/jobs/govuk_guix/create_data_snapshot_job.rb @@ -87,6 +87,8 @@ class GovukGuix::CreateDataSnapshotJob < Que::Job from_remote_host: remote_host ) + backend.add_in_use_store_path(build_output) + GovukGuix::DataSnapshot.transaction do GovukGuix::DataSnapshot.create!(data_snapshot_fields) diff --git a/app/jobs/govuk_guix/fetch_revision_job.rb b/app/jobs/govuk_guix/fetch_revision_job.rb index fd6e09c..7b1acdf 100644 --- a/app/jobs/govuk_guix/fetch_revision_job.rb +++ b/app/jobs/govuk_guix/fetch_revision_job.rb @@ -27,9 +27,11 @@ class GovukGuix::FetchRevisionJob < Que::Job def run(commit_hash, options = {}) backend_type_and_id = options[:backend_type_and_id] if backend_type_and_id - remote_host = Backends.find_by_type_and_id( + backend = Backends.find_by_type_and_id( *backend_type_and_id.split('=') - ).build_remote_host + ) + + remote_host = backend.build_remote_host end sha = fetch_and_checkout(commit_hash, remote_host) @@ -51,6 +53,8 @@ class GovukGuix::FetchRevisionJob < Que::Job store_path = output.last.strip logger.debug(self.class) { "store_path: #{store_path}" } + backend.add_in_use_store_path(store_path) if backend + GovukGuix::Revision.transaction do GovukGuix::Revision.create( commit_hash: sha, diff --git a/app/models/backends/terraform_aws/backend_methods.rb b/app/models/backends/terraform_aws/backend_methods.rb index cc5ca62..d6665a1 100644 --- a/app/models/backends/terraform_aws/backend_methods.rb +++ b/app/models/backends/terraform_aws/backend_methods.rb @@ -80,6 +80,28 @@ module Backends::TerraformAws::BackendMethods end end + def in_use_store_paths + [ + GovukGuix::Revision.pluck(:store_path), + available_data_snapshots.pluck(:store_path), + mini_environments.pluck(:backend_data).map { |x| x&.dig('build_output') } + ].flatten.compact + end + + def update_guix_gcroots + GovukGuix::UpdateGcrootsDirectory.set_in_use_store_paths( + in_use_store_paths, + run_remotely_on_host: build_remote_host + ) + end + + def add_in_use_store_path(store_path) + GovukGuix::UpdateGcrootsDirectory.add_store_path( + store_path, + run_remotely_on_host: build_remote_host + ) + end + def within_backend_terraform_working_directory(&block) with_advisory_lock( "terraform" diff --git a/app/models/backends/terraform_aws/mini_environment_methods.rb b/app/models/backends/terraform_aws/mini_environment_methods.rb index d0a8ab5..63efcd7 100644 --- a/app/models/backends/terraform_aws/mini_environment_methods.rb +++ b/app/models/backends/terraform_aws/mini_environment_methods.rb @@ -22,7 +22,7 @@ module Backends::TerraformAws::MiniEnvironmentMethods def build(mini_environment) slug = mini_environment.name.parameterize - GovukGuix::BuildMiniEnvironment.build( + store_path = GovukGuix::BuildMiniEnvironment.build( mini_environment.id, services: mini_environment.services.map(&:build_argument_string), arguments: { @@ -44,6 +44,8 @@ module Backends::TerraformAws::MiniEnvironmentMethods }, run_remotely_on_host: mini_environment.backend.build_remote_host ) + + add_in_use_store_path(store_path) end def start(mini_environment) diff --git a/app/models/backends/terraform_libvirt/backend_methods.rb b/app/models/backends/terraform_libvirt/backend_methods.rb index d14affa..85cd427 100644 --- a/app/models/backends/terraform_libvirt/backend_methods.rb +++ b/app/models/backends/terraform_libvirt/backend_methods.rb @@ -54,6 +54,26 @@ module Backends::TerraformLibvirt::BackendMethods end end + def in_use_store_paths + [ + GovukGuix::Revision.pluck(:store_path), + available_data_snapshots.pluck(:store_path), + mini_environments.pluck(:backend_data).map { |x| x.dig('build_output') } + ].flatten.compact + end + + def update_guix_gcroots + GovukGuix::UpdateGcrootsDirectory.set_in_use_store_paths( + in_use_store_paths + ) + end + + def add_in_use_store_path(store_path) + GovukGuix::UpdateGcrootsDirectory.add_store_path( + store_path + ) + end + def within_backend_terraform_working_directory(&block) with_advisory_lock( "terraform" diff --git a/app/models/backends/terraform_libvirt/mini_environment_methods.rb b/app/models/backends/terraform_libvirt/mini_environment_methods.rb index ef2b4c1..582aebb 100644 --- a/app/models/backends/terraform_libvirt/mini_environment_methods.rb +++ b/app/models/backends/terraform_libvirt/mini_environment_methods.rb @@ -22,7 +22,7 @@ module Backends::TerraformLibvirt::MiniEnvironmentMethods def build(mini_environment) slug = mini_environment.name.parameterize - GovukGuix::BuildMiniEnvironment.build( + store_path = GovukGuix::BuildMiniEnvironment.build( mini_environment.id, services: mini_environment.services.map(&:build_argument_string), arguments: { @@ -41,6 +41,8 @@ module Backends::TerraformLibvirt::MiniEnvironmentMethods data_snapshot: mini_environment.data_snapshot.try(:store_path) }.compact ) + + add_in_use_store_path(store_path) end def start(mini_environment) diff --git a/app/services/govuk_guix/build_mini_environment.rb b/app/services/govuk_guix/build_mini_environment.rb index 01920eb..9cada20 100644 --- a/app/services/govuk_guix/build_mini_environment.rb +++ b/app/services/govuk_guix/build_mini_environment.rb @@ -83,6 +83,8 @@ module GovukGuix::BuildMiniEnvironment build_output: build_output } ) + + build_output end def self.signon_user_to_sexp(signon_user) diff --git a/app/services/govuk_guix/update_gcroots_directory.rb b/app/services/govuk_guix/update_gcroots_directory.rb new file mode 100644 index 0000000..97fe703 --- /dev/null +++ b/app/services/govuk_guix/update_gcroots_directory.rb @@ -0,0 +1,62 @@ +# GOV.UK Mini Environment Admin +# Copyright © 2018 Christopher Baines +# +# This file is part of the GOV.UK Mini Environment Admin. +# +# The GOV.UK Mini Environment Admin is free software: you can +# redistribute it and/or modify it under the terms of the GNU Affero +# General Public License as published by the Free Software Foundation, +# either version 3 of the License, or (at your option) any later +# version. +# +# The GOV.UK Mini Environment Admin is distributed in the hope that it +# will be useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public +# License along with the GOV.UK Mini Environment Admin. If not, see +# . + +module GovukGuix::UpdateGcrootsDirectory + extend ::ShellUtils + + DIRECTORY = '/var/guix/gcroots/govuk-mini-environment-admin' + + def self.set_in_use_store_paths(store_paths, options = {}) + current_store_paths = list_store_paths(options) + + (current_store_paths - store_paths).each do |store_path| + remove_store_path(store_path, options) + end + + (store_paths - current_store_paths).each do |store_path| + add_store_path(store_path, options) + end + end + + def self.add_store_path(store_path, options = {}) + run_command( + 'ln', '-s', + store_path, + File.join(DIRECTORY, store_path['/gnu/store/'.length..-1]), + run_remotely_on_host: options[:run_remotely_on_host] + ) + end + + def self.remove_store_path(store_path, options = {}) + run_command( + 'rm', File.join(DIRECTORY, store_path['/gnu/store/'.length..-1]), + run_remotely_on_host: options[:run_remotely_on_host] + ) + end + + def self.list_store_paths(options = {}) + run_command( + 'ls', '-1', DIRECTORY, + run_remotely_on_host: options[:run_remotely_on_host] + ).map do |store_path_suffix| + "/gnu/store/#{store_path_suffix.strip}" + end + end +end diff --git a/app/views/backends/terraform_aws/in_use_store_paths.html.erb b/app/views/backends/terraform_aws/in_use_store_paths.html.erb new file mode 100644 index 0000000..c3b694c --- /dev/null +++ b/app/views/backends/terraform_aws/in_use_store_paths.html.erb @@ -0,0 +1,40 @@ +<%# + +GOV.UK Mini Environment Admin +Copyright © 2018 Christopher Baines + +This file is part of the GOV.UK Mini Environment Admin. + +The GOV.UK Mini Environment Admin is free software: you can +redistribute it and/or modify it under the terms of the GNU Affero +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later +version. + +The GOV.UK Mini Environment Admin is distributed in the hope that it +will be useful, but WITHOUT ANY WARRANTY; without even the implied +warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See +the GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public +License along with the GOV.UK Mini Environment Admin. If not, see +. + +%> + +

Backend: <%= @backend.label %>

+<% status = @backend.status %> + +
+ +

In use store paths

+ + + <% @backend.in_use_store_paths.each do |store_path| %> + + + + + <% end %> +
<%= store_path %> +
diff --git a/app/views/backends/terraform_libvirt/in_use_store_paths.html.erb b/app/views/backends/terraform_libvirt/in_use_store_paths.html.erb new file mode 100644 index 0000000..c3b694c --- /dev/null +++ b/app/views/backends/terraform_libvirt/in_use_store_paths.html.erb @@ -0,0 +1,40 @@ +<%# + +GOV.UK Mini Environment Admin +Copyright © 2018 Christopher Baines + +This file is part of the GOV.UK Mini Environment Admin. + +The GOV.UK Mini Environment Admin is free software: you can +redistribute it and/or modify it under the terms of the GNU Affero +General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later +version. + +The GOV.UK Mini Environment Admin is distributed in the hope that it +will be useful, but WITHOUT ANY WARRANTY; without even the implied +warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See +the GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public +License along with the GOV.UK Mini Environment Admin. If not, see +. + +%> + +

Backend: <%= @backend.label %>

+<% status = @backend.status %> + +
+ +

In use store paths

+ + + <% @backend.in_use_store_paths.each do |store_path| %> + + + + + <% end %> +
<%= store_path %> +
diff --git a/config/routes.rb b/config/routes.rb index 8e55628..8242395 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -19,6 +19,7 @@ Rails.application.routes.draw do only: %i[create new show update destroy] do member do post 'perform_action' + get 'in_use_store_paths' end end @@ -28,6 +29,7 @@ Rails.application.routes.draw do only: %i[create new show update destroy] do member do post 'perform_action' + get 'in_use_store_paths' end end end diff --git a/lib/tasks/backend.rake b/lib/tasks/backend.rake index c23a3df..19dc462 100644 --- a/lib/tasks/backend.rake +++ b/lib/tasks/backend.rake @@ -6,6 +6,13 @@ namespace :backend do .find(args.backend_id) .deploy_backend end + + desc 'Update GC Roots' + task :update_guix_gcroots, [:backend_id] => :environment do |t, args| + Backends::TerraformAws + .find(args.backend_id) + .update_guix_gcroots + end end namespace :terraform_aws do @@ -22,5 +29,12 @@ namespace :backend do .find(args.backend_id) .destroy_backend end + + desc 'Update GC Roots' + task :update_guix_gcroots, [:backend_id] => :environment do |t, args| + Backends::TerraformAws + .find(args.backend_id) + .update_guix_gcroots + end end end diff --git a/test/controllers/mini_environments_controller_test.rb b/test/controllers/mini_environments_controller_test.rb new file mode 100644 index 0000000..8176143 --- /dev/null +++ b/test/controllers/mini_environments_controller_test.rb @@ -0,0 +1,13 @@ +require 'integration_test_helper' + +class MiniEnvironmentsControllerTest < ActionDispatch::IntegrationTest + setup do + login_as User.new + end + + test 'show' do + MiniEnvironment.create + + get mini_environment_path + end +end diff --git a/test/models/backends/terraform_aws_test.rb b/test/models/backends/terraform_aws_test.rb index f9a34f3..d782f3c 100644 --- a/test/models/backends/terraform_aws_test.rb +++ b/test/models/backends/terraform_aws_test.rb @@ -28,6 +28,7 @@ class Backends::TerraformAwsTest < ActiveSupport::TestCase end test 'build' do + GovukGuix::UpdateGcrootsDirectory.stubs(:add_store_path) GovukGuix::BuildMiniEnvironment.expects(:build) @backend.build(@mini_environment) diff --git a/test/models/backends/terraform_libvirt_test.rb b/test/models/backends/terraform_libvirt_test.rb index 056a1c4..fb56dba 100644 --- a/test/models/backends/terraform_libvirt_test.rb +++ b/test/models/backends/terraform_libvirt_test.rb @@ -17,6 +17,7 @@ class Backends::TerraformLibvirtTest < ActiveSupport::TestCase end test 'build' do + GovukGuix::UpdateGcrootsDirectory.stubs(:add_store_path) GovukGuix::BuildMiniEnvironment.expects(:build) @backend.build(@mini_environment) -- cgit v1.2.3