Fix upgrading Splunk Enterprise

When upgrading, there was an issue with accepting the license of the new version.
Thus symptoms like described in #339 happened. To ensure we have a clean upgrade
we need to re-enable boot-start after the upgrade.

This will also apply the correct systemd service file when upgrading, before it
was just the service file of the originally installed version. It never changed.

manifests/enterprise/service/nix.pp

@@ -16,6 +16,12 @@
       timeout => 0,
       notify  => Exec['enable_splunk'],
     }
+
+    exec { 'disable_splunk_boot_start':
+      command     => "${splunk::enterprise::enterprise_homedir}/bin/splunk disable boot-start --accept-license --answer-yes --no-prompt",
+      refreshonly => true,
+      subscribe   => Package[$splunk::enterprise::package_name],
+    }
     if $splunk::params::supports_systemd and $splunk::enterprise::splunk_user == 'root' {
       $user_args = ''
     } else {
@@ -28,7 +34,8 @@
       command     => "${splunk::enterprise::enterprise_homedir}/bin/splunk enable boot-start ${user_args} ${splunk::params::boot_start_args} --accept-license --answer-yes --no-prompt",
       refreshonly => true,
       before      => Service[$splunk::enterprise::service_name],
-      require     => Exec['stop_splunk'],
+      require     => [Exec['stop_splunk'], Exec['disable_splunk_boot_start']],
+      subscribe   => Package[$splunk::enterprise::package_name],
     }
   }
   # Commands to license, disable, and start Splunk Enterprise

spec/acceptance/splunk_enterprise_spec.rb

@@ -55,6 +55,79 @@ class { 'splunk::enterprise': }
     end
   end
 
+  context "Upgrading splunk enterprise from 9.1.0 to 9.2.0.1" do
+    after(:all) do
+      pp = <<-EOS
+        service { '#{service_name}': ensure => stopped }
+        package { 'splunk': ensure => purged }
+        file { '/opt/splunk': ensure => absent, force => true, require => Package['splunk'] }
+        file { '/etc/init.d/splunk': ensure => absent, require => Package['splunk'] }
+        file { '/etc/systemd/system/Splunkd.service': ensure => absent, require => Package['splunk'] }
+        EOS
+      apply_manifest(pp, catch_failures: true)
+    end
+
+    it 'works idempotently with no errors' do
+      pp = <<-EOS
+        class { 'splunk::params': version => '9.1.0', build => '1c86ca0bacc3' }
+        class { 'splunk::enterprise':  }
+
+        # See https://community.splunk.com/t5/Installation/Why-am-I-getting-an-error-to-start-a-fresh-Splunk-instance-in-my/m-p/336938
+        file_line { 'file_locking':
+          path => '/opt/splunk/etc/splunk-launch.conf',
+          line => 'OPTIMISTIC_ABOUT_FILE_LOCKING=1',
+          before => Exec['enable_splunk'],
+          require => Package['splunk'],
+        }
+      EOS
+
+      # Run it twice and test for idempotency
+      apply_manifest(pp, catch_failures: true)
+      apply_manifest(pp, catch_changes: true)
+      # give splunk some time to start
+      sleep(10)
+    end
+
+    describe package('splunk') do
+      it { is_expected.to be_installed }
+    end
+
+    describe service(service_name) do
+      it { is_expected.to be_enabled }
+      it { is_expected.to be_running }
+    end
+
+    it 'upgrades to 9.2.0.1' do
+      pp = <<-EOS
+        class { 'splunk::params': version => '9.2.0.1', build => 'd8ae995bf219' }
+        class { 'splunk::enterprise': package_ensure => latest  }
+
+        # See https://community.splunk.com/t5/Installation/Why-am-I-getting-an-error-to-start-a-fresh-Splunk-instance-in-my/m-p/336938
+        file_line { 'file_locking':
+          path => '/opt/splunk/etc/splunk-launch.conf',
+          line => 'OPTIMISTIC_ABOUT_FILE_LOCKING=1',
+          before => Exec['enable_splunk'],
+          require => Package['splunk'],
+        }
+      EOS
+
+      # Run it twice and test for idempotency
+      apply_manifest(pp, catch_failures: true)
+      apply_manifest(pp, catch_changes: true)
+      # give splunk some time to start
+      sleep(10)
+    end
+
+    describe package('splunk') do
+      it { is_expected.to be_installed }
+    end
+
+    describe service(service_name) do
+      it { is_expected.to be_enabled }
+      it { is_expected.to be_running }
+    end
+  end
+
   context 'default parameters' do
     # Using puppet_apply as a helper
     it 'works idempotently with no errors' do