Writes basic NATs creds file if UAA is not up

If for some reason NATs sync cannot authenticate to use the BOSH API,
then the user authentication file for NATs will be empty. This causes
problems when UAA is slow to start in some installations.

Our approach to fix this issue was to make NATS Sync write the
credentials for the Director and the Monitor even if the BOSH API is not
available to be consumed. We also made sure that NATs Sync is not going
to overwrite any credentials if they were already written.

[#183572544] Investigate possible NATS 2.0 race condition

Signed-off-by: Daniel Felipe Ochoa <[email protected]>

src/bosh-nats-sync/lib/nats_sync/users_sync.rb

@@ -14,14 +14,28 @@ def initialize(nats_config_file_path, bosh_config, nats_server_executable, nats_
 
     def execute_users_sync
       NATSSync.logger.info 'Executing NATS Users Synchronization'
-      vms_uuids = query_all_running_vms
-      current_file_hash = nats_file_hash
-      write_nats_config_file(vms_uuids, read_subject_file(@bosh_config['director_subject_file']),
-                             read_subject_file(@bosh_config['hm_subject_file']))
-      new_file_hash = nats_file_hash
-      UsersSync.reload_nats_server_config(@nats_server_executable, @nats_server_pid_file) unless current_file_hash == new_file_hash
+      vms_uuids = []
+      overwriteable_config_file = true
+      begin
+        vms_uuids = query_all_running_vms
+      rescue RuntimeError => e
+        NATSSync.logger.error "Could not query all running vms: #{e.message}"
+        overwriteable_config_file = is_the_user_file_overwritable?
+        if overwriteable_config_file
+          NATSSync.logger.info "NATS config file is empty, writing basic users config file."
+        else
+          NATSSync.logger.info "NATS config file is not empty, doing nothing."
+        end
+      end
+
+      if overwriteable_config_file
+        current_file_hash = nats_file_hash
+        write_nats_config_file(vms_uuids, read_subject_file(@bosh_config['director_subject_file']),
+                               read_subject_file(@bosh_config['hm_subject_file']))
+        new_file_hash = nats_file_hash
+        UsersSync.reload_nats_server_config(@nats_server_executable, @nats_server_pid_file) unless current_file_hash == new_file_hash
+      end
       NATSSync.logger.info 'Finishing NATS Users Synchronization'
-      vms_uuids
     end
 
     def self.reload_nats_server_config(nats_server_executable, nats_server_pid_file)
@@ -33,6 +47,12 @@ def self.reload_nats_server_config(nats_server_executable, nats_server_pid_file)
 
     private
 
+    def is_the_user_file_overwritable?
+      JSON.parse(File.read(@nats_config_file_path)).empty?
+    rescue
+      true
+    end
+
     def read_subject_file(file_path)
       return nil unless File.exist?(file_path)
       return nil if File.empty?(file_path)
@@ -45,10 +65,12 @@ def nats_file_hash
     end
 
     def call_bosh_api(endpoint)
+      auth_header = create_authentication_header
+      NATSSync.logger.debug 'auth_header is empty, next REST call could fail' if auth_header.nil? || auth_header.empty?
       response = RestClient::Request.execute(
         url: @bosh_config['url'] + endpoint,
         method: :get,
-        headers: { 'Authorization' => create_authentication_header },
+        headers: { 'Authorization' => auth_header },
         verify_ssl: false,
       )
       NATSSync.logger.debug(response.inspect)

src/bosh-nats-sync/spec/nats_sync/users_sync_spec.rb

@@ -6,7 +6,9 @@ module NATSSync
   describe UsersSync do
     before do
       allow(NATSSync).to receive(:logger).and_return(logger)
-      allow(logger).to receive :info
+      allow(logger).to receive(:debug)
+      allow(logger).to receive(:error)
+      allow(logger).to receive(:info)
       allow(Open3).to receive(:capture2e).and_return(["Success", capture_status])
     end
 
@@ -149,7 +151,64 @@ module NATSSync
           .to_return(status: 200, body: deployments_json)
         allow(auth_provider).to receive(:new).and_return(auth_provider_double)
         allow(auth_provider_double).to receive(:auth_header).and_return('Bearer xyz')
-        allow(logger).to receive(:debug)
+        File.open(nats_config_file_path, 'w') do |f|
+          f.write('{}')
+        end
+      end
+
+      describe 'when UAA is not deployed and the BOSH API is not available' do
+        before do
+          stub_request(:get, url + '/deployments')
+            .with(headers: { 'Authorization' => 'Bearer xyz' })
+            .to_return(status: 401, body: "Unauthorized")
+        end
+
+        describe "and the authentication file is empty" do
+          it 'should write the basic bosh configuration' do
+            expect(JSON.parse(File.read(nats_config_file_path)).empty?).to be true
+            subject.execute_users_sync
+            file = File.read(nats_config_file_path)
+            data_hash = JSON.parse(file)
+            expect(data_hash['authorization']['users'])
+              .to include(include('user' => director_subject))
+            expect(data_hash['authorization']['users'])
+              .to include(include('user' => hm_subject))
+            expect(data_hash['authorization']['users'].length).to eq(2)
+          end
+        end
+
+        describe "and the authentication file is corrupted" do
+          before do
+            File.open(nats_config_file_path, 'w') do |f|
+              f.write('{invalidchar')
+            end
+          end
+          it 'should write the basic bosh configuration' do
+            subject.execute_users_sync
+            file = File.read(nats_config_file_path)
+            data_hash = JSON.parse(file)
+            expect(data_hash['authorization']['users'])
+              .to include(include('user' => director_subject))
+            expect(data_hash['authorization']['users'])
+              .to include(include('user' => hm_subject))
+            expect(data_hash['authorization']['users'].length).to eq(2)
+          end
+        end
+
+        describe "and the authentication file is not empty" do
+          before do
+            File.open(nats_config_file_path, 'w') do |f|
+              f.write('{"authorization": {"users": [{"user": "foo"}]}}')
+            end
+          end
+          it 'should not overwrite the authentication file' do
+            subject.execute_users_sync
+            file = File.read(nats_config_file_path)
+            data_hash = JSON.parse(file)
+            expect(data_hash).to eq({ 'authorization' => { 'users' => [{ 'user' => 'foo' }] } })
+          end
+        end
+
       end
 
       describe 'when there are no deployments with running vms in Bosh' do