Add locking around all Terraform actions

To prevent them conflicting. Terraform itself has a locking mechanism,
which isn't being used, but this should be sufficient.

7 files changed, 35 insertions, 16 deletions

diff --git a/app/models/backends/terraform_aws/backend_methods.rb b/app/models/backends/terraform_aws/backend_methods.rb
index 52875e6..7489325 100644
--- a/app/models/backends/terraform_aws/backend_methods.rb
+++ b/app/models/backends/terraform_aws/backend_methods.rb
@@ -84,10 +84,14 @@ module Backends::TerraformAws::BackendMethods
   end
 
   def within_backend_terraform_working_directory(&block)
-    TerraformWorkingDirectory.new(
-      terraform_state_id,
-      'terraform/aws/backend'
-    ).within_working_directory(&block)
+    with_advisory_lock(
+      "aws_backend_terraform_working_directory/#{id}"
+    ) do
+      TerraformWorkingDirectory.new(
+        terraform_state_id,
+        'terraform/aws/backend'
+      ).within_working_directory(&block)
+    end
   end
 
   def available_data_snapshots
diff --git a/app/models/backends/terraform_aws/mini_environment_methods.rb b/app/models/backends/terraform_aws/mini_environment_methods.rb
index 6049829..1885af7 100644
--- a/app/models/backends/terraform_aws/mini_environment_methods.rb
+++ b/app/models/backends/terraform_aws/mini_environment_methods.rb
@@ -89,10 +89,14 @@ module Backends::TerraformAws::MiniEnvironmentMethods
   end
 
   def within_terraform_working_directory(mini_environment, &block)
-    TerraformWorkingDirectory.new(
-      mini_environment_state_id(mini_environment),
-      'terraform/aws/mini_environment'
-    ).within_working_directory(&block)
+    with_advisory_lock(
+      "aws_mini_environment_terraform_working_directory/#{mini_environment.id}"
+    ) do
+      TerraformWorkingDirectory.new(
+        mini_environment_state_id(mini_environment),
+        'terraform/aws/mini_environment'
+      ).within_working_directory(&block)
+    end
   end
 
   def mini_environment_state_id(mini_environment)
diff --git a/app/models/backends/terraform_libvirt/backend_methods.rb b/app/models/backends/terraform_libvirt/backend_methods.rb
index 00ac432..06b89e4 100644
--- a/app/models/backends/terraform_libvirt/backend_methods.rb
+++ b/app/models/backends/terraform_libvirt/backend_methods.rb
@@ -55,10 +55,14 @@ module Backends::TerraformLibvirt::BackendMethods
   end
 
   def within_backend_terraform_working_directory(&block)
-    TerraformWorkingDirectory.new(
-      terraform_state_id,
-      'terraform/libvirt/backend'
-    ).within_working_directory(&block)
+    with_advisory_lock(
+      "libvirt_backend_terraform_working_directory/#{id}"
+    ) do
+      TerraformWorkingDirectory.new(
+        terraform_state_id,
+        'terraform/libvirt/backend'
+      ).within_working_directory(&block)
+    end
   end
 
   def backend_terraform_states
diff --git a/app/models/backends/terraform_libvirt/mini_environment_methods.rb b/app/models/backends/terraform_libvirt/mini_environment_methods.rb
index 4ed8835..64781df 100644
--- a/app/models/backends/terraform_libvirt/mini_environment_methods.rb
+++ b/app/models/backends/terraform_libvirt/mini_environment_methods.rb
@@ -78,10 +78,14 @@ module Backends::TerraformLibvirt::MiniEnvironmentMethods
   end
 
   def within_terraform_working_directory(mini_environment, &block)
-    TerraformWorkingDirectory.new(
-      mini_environment_state_id(mini_environment),
-      'terraform/libvirt/mini_environment'
-    ).within_working_directory(&block)
+    with_advisory_lock(
+      "libvirt_mini_environment_terraform_working_directory/#{mini_environment.id}"
+    ) do
+      TerraformWorkingDirectory.new(
+        mini_environment_state_id(mini_environment),
+        'terraform/libvirt/mini_environment'
+      ).within_working_directory(&block)
+    end
   end
 
   def mini_environment_state_id(mini_environment)
diff --git a/config/application.rb b/config/application.rb
index b78dfda..534981c 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -7,6 +7,7 @@ require 'govuk_admin_template'
 require 'plek'
 
 require 'web_console' if ENV['RAILS_ENV'] == 'development'
+require 'with_advisory_lock'
 
 module GovukMiniEnvironmentAdmin
   class Application < Rails::Application
diff --git a/config/boot.rb b/config/boot.rb
index 48d7d8b..d620a68 100644
--- a/config/boot.rb
+++ b/config/boot.rb
@@ -10,4 +10,5 @@ gem 'gds-sso'
 gem 'plek'
 gem 'que'
 gem 'ruby-terraform'
+gem 'with_advisory_lock'
 gem 'git'
diff --git a/guix.scm b/guix.scm
index 547ef69..9815b4c 100644
--- a/guix.scm
+++ b/guix.scm
@@ -41,6 +41,7 @@
                (base32
                 "0714zhc0rak2wrb2yivy7a1naqlcccwgd90lp2vpjs7aj3j23i2k"))))))
        ("ruby-que" ,ruby-que)
+       ("ruby-with-advisory-lock" ,ruby-with-advisory-lock)
        ("ruby-git" ,ruby-git)
        ("terraform" ,terraform)
        ("terraform-provider-libvirt"