aboutsummaryrefslogtreecommitdiff
path: root/app
diff options
context:
space:
mode:
authorChristopher Baines <mail@cbaines.net>2018-12-31 18:22:12 +0000
committerChristopher Baines <mail@cbaines.net>2019-01-01 02:38:23 +0000
commit3459059ed1f9332a9e71a4a7be5dc5e22e7f6bae (patch)
tree477f63fbc4546ee2947f571b831fb9463e2025b2 /app
parent3e78356904b610aa149dd2e2c30e832408f37acc (diff)
downloadgovuk-mini-environment-admin-3459059ed1f9332a9e71a4a7be5dc5e22e7f6bae.tar
govuk-mini-environment-admin-3459059ed1f9332a9e71a4a7be5dc5e22e7f6bae.tar.gz
Start tracking which store paths are in use
This will enable garbage collection of the Guix store, without removing things that are still in use.
Diffstat (limited to 'app')
-rw-r--r--app/controllers/backends/terraform_aws_controller.rb4
-rw-r--r--app/controllers/backends/terraform_libvirt_controller.rb4
-rw-r--r--app/jobs/govuk_guix/create_data_snapshot_job.rb2
-rw-r--r--app/jobs/govuk_guix/fetch_revision_job.rb8
-rw-r--r--app/models/backends/terraform_aws/backend_methods.rb22
-rw-r--r--app/models/backends/terraform_aws/mini_environment_methods.rb4
-rw-r--r--app/models/backends/terraform_libvirt/backend_methods.rb20
-rw-r--r--app/models/backends/terraform_libvirt/mini_environment_methods.rb4
-rw-r--r--app/services/govuk_guix/build_mini_environment.rb2
-rw-r--r--app/services/govuk_guix/update_gcroots_directory.rb62
-rw-r--r--app/views/backends/terraform_aws/in_use_store_paths.html.erb40
-rw-r--r--app/views/backends/terraform_libvirt/in_use_store_paths.html.erb40
12 files changed, 208 insertions, 4 deletions
diff --git a/app/controllers/backends/terraform_aws_controller.rb b/app/controllers/backends/terraform_aws_controller.rb
index e5b66d4..b09001d 100644
--- a/app/controllers/backends/terraform_aws_controller.rb
+++ b/app/controllers/backends/terraform_aws_controller.rb
@@ -84,6 +84,10 @@ class Backends::TerraformAwsController < ApplicationController
redirect_to terraform_aws_backend_path(@backend)
end
+ def in_use_store_paths
+ @backend = Backends::TerraformAws.find(params['id'])
+ end
+
private
def create_params
diff --git a/app/controllers/backends/terraform_libvirt_controller.rb b/app/controllers/backends/terraform_libvirt_controller.rb
index 4421146..1bf37cd 100644
--- a/app/controllers/backends/terraform_libvirt_controller.rb
+++ b/app/controllers/backends/terraform_libvirt_controller.rb
@@ -82,6 +82,10 @@ class Backends::TerraformLibvirtController < ApplicationController
redirect_to terraform_libvirt_backend_path(@backend)
end
+ def in_use_store_paths
+ @backend = Backends::TerraformLibvirt.find(params['id'])
+ end
+
private
def create_params
diff --git a/app/jobs/govuk_guix/create_data_snapshot_job.rb b/app/jobs/govuk_guix/create_data_snapshot_job.rb
index 88a5ba4..bcc8aa1 100644
--- a/app/jobs/govuk_guix/create_data_snapshot_job.rb
+++ b/app/jobs/govuk_guix/create_data_snapshot_job.rb
@@ -87,6 +87,8 @@ class GovukGuix::CreateDataSnapshotJob < Que::Job
from_remote_host: remote_host
)
+ backend.add_in_use_store_path(build_output)
+
GovukGuix::DataSnapshot.transaction do
GovukGuix::DataSnapshot.create!(data_snapshot_fields)
diff --git a/app/jobs/govuk_guix/fetch_revision_job.rb b/app/jobs/govuk_guix/fetch_revision_job.rb
index fd6e09c..7b1acdf 100644
--- a/app/jobs/govuk_guix/fetch_revision_job.rb
+++ b/app/jobs/govuk_guix/fetch_revision_job.rb
@@ -27,9 +27,11 @@ class GovukGuix::FetchRevisionJob < Que::Job
def run(commit_hash, options = {})
backend_type_and_id = options[:backend_type_and_id]
if backend_type_and_id
- remote_host = Backends.find_by_type_and_id(
+ backend = Backends.find_by_type_and_id(
*backend_type_and_id.split('=')
- ).build_remote_host
+ )
+
+ remote_host = backend.build_remote_host
end
sha = fetch_and_checkout(commit_hash, remote_host)
@@ -51,6 +53,8 @@ class GovukGuix::FetchRevisionJob < Que::Job
store_path = output.last.strip
logger.debug(self.class) { "store_path: #{store_path}" }
+ backend.add_in_use_store_path(store_path) if backend
+
GovukGuix::Revision.transaction do
GovukGuix::Revision.create(
commit_hash: sha,
diff --git a/app/models/backends/terraform_aws/backend_methods.rb b/app/models/backends/terraform_aws/backend_methods.rb
index cc5ca62..d6665a1 100644
--- a/app/models/backends/terraform_aws/backend_methods.rb
+++ b/app/models/backends/terraform_aws/backend_methods.rb
@@ -80,6 +80,28 @@ module Backends::TerraformAws::BackendMethods
end
end
+ def in_use_store_paths
+ [
+ GovukGuix::Revision.pluck(:store_path),
+ available_data_snapshots.pluck(:store_path),
+ mini_environments.pluck(:backend_data).map { |x| x&.dig('build_output') }
+ ].flatten.compact
+ end
+
+ def update_guix_gcroots
+ GovukGuix::UpdateGcrootsDirectory.set_in_use_store_paths(
+ in_use_store_paths,
+ run_remotely_on_host: build_remote_host
+ )
+ end
+
+ def add_in_use_store_path(store_path)
+ GovukGuix::UpdateGcrootsDirectory.add_store_path(
+ store_path,
+ run_remotely_on_host: build_remote_host
+ )
+ end
+
def within_backend_terraform_working_directory(&block)
with_advisory_lock(
"terraform"
diff --git a/app/models/backends/terraform_aws/mini_environment_methods.rb b/app/models/backends/terraform_aws/mini_environment_methods.rb
index d0a8ab5..63efcd7 100644
--- a/app/models/backends/terraform_aws/mini_environment_methods.rb
+++ b/app/models/backends/terraform_aws/mini_environment_methods.rb
@@ -22,7 +22,7 @@ module Backends::TerraformAws::MiniEnvironmentMethods
def build(mini_environment)
slug = mini_environment.name.parameterize
- GovukGuix::BuildMiniEnvironment.build(
+ store_path = GovukGuix::BuildMiniEnvironment.build(
mini_environment.id,
services: mini_environment.services.map(&:build_argument_string),
arguments: {
@@ -44,6 +44,8 @@ module Backends::TerraformAws::MiniEnvironmentMethods
},
run_remotely_on_host: mini_environment.backend.build_remote_host
)
+
+ add_in_use_store_path(store_path)
end
def start(mini_environment)
diff --git a/app/models/backends/terraform_libvirt/backend_methods.rb b/app/models/backends/terraform_libvirt/backend_methods.rb
index d14affa..85cd427 100644
--- a/app/models/backends/terraform_libvirt/backend_methods.rb
+++ b/app/models/backends/terraform_libvirt/backend_methods.rb
@@ -54,6 +54,26 @@ module Backends::TerraformLibvirt::BackendMethods
end
end
+ def in_use_store_paths
+ [
+ GovukGuix::Revision.pluck(:store_path),
+ available_data_snapshots.pluck(:store_path),
+ mini_environments.pluck(:backend_data).map { |x| x.dig('build_output') }
+ ].flatten.compact
+ end
+
+ def update_guix_gcroots
+ GovukGuix::UpdateGcrootsDirectory.set_in_use_store_paths(
+ in_use_store_paths
+ )
+ end
+
+ def add_in_use_store_path(store_path)
+ GovukGuix::UpdateGcrootsDirectory.add_store_path(
+ store_path
+ )
+ end
+
def within_backend_terraform_working_directory(&block)
with_advisory_lock(
"terraform"
diff --git a/app/models/backends/terraform_libvirt/mini_environment_methods.rb b/app/models/backends/terraform_libvirt/mini_environment_methods.rb
index ef2b4c1..582aebb 100644
--- a/app/models/backends/terraform_libvirt/mini_environment_methods.rb
+++ b/app/models/backends/terraform_libvirt/mini_environment_methods.rb
@@ -22,7 +22,7 @@ module Backends::TerraformLibvirt::MiniEnvironmentMethods
def build(mini_environment)
slug = mini_environment.name.parameterize
- GovukGuix::BuildMiniEnvironment.build(
+ store_path = GovukGuix::BuildMiniEnvironment.build(
mini_environment.id,
services: mini_environment.services.map(&:build_argument_string),
arguments: {
@@ -41,6 +41,8 @@ module Backends::TerraformLibvirt::MiniEnvironmentMethods
data_snapshot: mini_environment.data_snapshot.try(:store_path)
}.compact
)
+
+ add_in_use_store_path(store_path)
end
def start(mini_environment)
diff --git a/app/services/govuk_guix/build_mini_environment.rb b/app/services/govuk_guix/build_mini_environment.rb
index 01920eb..9cada20 100644
--- a/app/services/govuk_guix/build_mini_environment.rb
+++ b/app/services/govuk_guix/build_mini_environment.rb
@@ -83,6 +83,8 @@ module GovukGuix::BuildMiniEnvironment
build_output: build_output
}
)
+
+ build_output
end
def self.signon_user_to_sexp(signon_user)
diff --git a/app/services/govuk_guix/update_gcroots_directory.rb b/app/services/govuk_guix/update_gcroots_directory.rb
new file mode 100644
index 0000000..97fe703
--- /dev/null
+++ b/app/services/govuk_guix/update_gcroots_directory.rb
@@ -0,0 +1,62 @@
+# GOV.UK Mini Environment Admin
+# Copyright © 2018 Christopher Baines <mail@cbaines.net>
+#
+# This file is part of the GOV.UK Mini Environment Admin.
+#
+# The GOV.UK Mini Environment Admin is free software: you can
+# redistribute it and/or modify it under the terms of the GNU Affero
+# General Public License as published by the Free Software Foundation,
+# either version 3 of the License, or (at your option) any later
+# version.
+#
+# The GOV.UK Mini Environment Admin is distributed in the hope that it
+# will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with the GOV.UK Mini Environment Admin. If not, see
+# <http://www.gnu.org/licenses/>.
+
+module GovukGuix::UpdateGcrootsDirectory
+ extend ::ShellUtils
+
+ DIRECTORY = '/var/guix/gcroots/govuk-mini-environment-admin'
+
+ def self.set_in_use_store_paths(store_paths, options = {})
+ current_store_paths = list_store_paths(options)
+
+ (current_store_paths - store_paths).each do |store_path|
+ remove_store_path(store_path, options)
+ end
+
+ (store_paths - current_store_paths).each do |store_path|
+ add_store_path(store_path, options)
+ end
+ end
+
+ def self.add_store_path(store_path, options = {})
+ run_command(
+ 'ln', '-s',
+ store_path,
+ File.join(DIRECTORY, store_path['/gnu/store/'.length..-1]),
+ run_remotely_on_host: options[:run_remotely_on_host]
+ )
+ end
+
+ def self.remove_store_path(store_path, options = {})
+ run_command(
+ 'rm', File.join(DIRECTORY, store_path['/gnu/store/'.length..-1]),
+ run_remotely_on_host: options[:run_remotely_on_host]
+ )
+ end
+
+ def self.list_store_paths(options = {})
+ run_command(
+ 'ls', '-1', DIRECTORY,
+ run_remotely_on_host: options[:run_remotely_on_host]
+ ).map do |store_path_suffix|
+ "/gnu/store/#{store_path_suffix.strip}"
+ end
+ end
+end
diff --git a/app/views/backends/terraform_aws/in_use_store_paths.html.erb b/app/views/backends/terraform_aws/in_use_store_paths.html.erb
new file mode 100644
index 0000000..c3b694c
--- /dev/null
+++ b/app/views/backends/terraform_aws/in_use_store_paths.html.erb
@@ -0,0 +1,40 @@
+<%#
+
+GOV.UK Mini Environment Admin
+Copyright © 2018 Christopher Baines <mail@cbaines.net>
+
+This file is part of the GOV.UK Mini Environment Admin.
+
+The GOV.UK Mini Environment Admin is free software: you can
+redistribute it and/or modify it under the terms of the GNU Affero
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later
+version.
+
+The GOV.UK Mini Environment Admin is distributed in the hope that it
+will be useful, but WITHOUT ANY WARRANTY; without even the implied
+warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
+the GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public
+License along with the GOV.UK Mini Environment Admin. If not, see
+<http://www.gnu.org/licenses/>.
+
+%>
+
+<h1>Backend: <%= @backend.label %></h1>
+<% status = @backend.status %>
+
+<br>
+
+<h3>In use store paths</h3>
+
+<table class="table table-striped">
+ <% @backend.in_use_store_paths.each do |store_path| %>
+ <tr>
+ <td><%= store_path %></td>
+ <td>
+ </td>
+ </tr>
+ <% end %>
+</table>
diff --git a/app/views/backends/terraform_libvirt/in_use_store_paths.html.erb b/app/views/backends/terraform_libvirt/in_use_store_paths.html.erb
new file mode 100644
index 0000000..c3b694c
--- /dev/null
+++ b/app/views/backends/terraform_libvirt/in_use_store_paths.html.erb
@@ -0,0 +1,40 @@
+<%#
+
+GOV.UK Mini Environment Admin
+Copyright © 2018 Christopher Baines <mail@cbaines.net>
+
+This file is part of the GOV.UK Mini Environment Admin.
+
+The GOV.UK Mini Environment Admin is free software: you can
+redistribute it and/or modify it under the terms of the GNU Affero
+General Public License as published by the Free Software Foundation,
+either version 3 of the License, or (at your option) any later
+version.
+
+The GOV.UK Mini Environment Admin is distributed in the hope that it
+will be useful, but WITHOUT ANY WARRANTY; without even the implied
+warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
+the GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public
+License along with the GOV.UK Mini Environment Admin. If not, see
+<http://www.gnu.org/licenses/>.
+
+%>
+
+<h1>Backend: <%= @backend.label %></h1>
+<% status = @backend.status %>
+
+<br>
+
+<h3>In use store paths</h3>
+
+<table class="table table-striped">
+ <% @backend.in_use_store_paths.each do |store_path| %>
+ <tr>
+ <td><%= store_path %></td>
+ <td>
+ </td>
+ </tr>
+ <% end %>
+</table>