Skip "file already closed" errors in generator storage instance test (#…

…3882)

* Ignore ErrClosed in generator storage instance test

Occasionally we see "file already closed" errors in our CI.  There
appears to be a race in the syscall to close the file.

* Stop the test ticker

* Check the contents of the error and not the chain

* Add a note to the registry commit

* Add registry note

* Fix nested assertions

* Cover additional appender.Commit() tests

* Check the context.Err right before commit

* Avoid error checking and only cancel early

* Update comment

modules/generator/registry/registry.go

@@ -244,6 +244,13 @@ func (r *ManagedRegistry) collectMetrics(ctx context.Context) {
 	maxActiveSeries := r.overrides.MetricsGeneratorMaxActiveSeries(r.tenant)
 	r.metricMaxActiveSeries.Set(float64(maxActiveSeries))
 
+	// Try to avoid committing after we have started the shutdown process.
+	if ctx.Err() != nil { // shutdown
+		return
+	}
+
+	// If the shutdown has started here, a "file already closed" error will be
+	// observed here.
 	err = appender.Commit()
 	if err != nil {
 		return

modules/generator/storage/instance_test.go

@@ -85,6 +85,7 @@ func TestInstance(t *testing.T) {
 	mockServer.refuseRequests.Store(false)
 
 	// Shutdown the instance - even though previous requests failed, remote.Storage should flush pending data
+	cancel()
 	err = instance.Close()
 	assert.NoError(t, err)
 
@@ -156,6 +157,7 @@ func TestInstance_multiTenancy(t *testing.T) {
 	})
 	require.NoError(t, err, "timed out while waiting for accepted requests")
 
+	cancel()
 	for _, instance := range instances {
 		// Shutdown the instance - remote write should flush pending data
 		err = instance.Close()
@@ -231,7 +233,8 @@ func TestInstance_remoteWriteHeaders(t *testing.T) {
 			return
 		}
 
-		assert.NoError(t, appender.Commit())
+		err = appender.Commit()
+		assert.NoError(t, err)
 	})
 
 	// Wait until remote.Storage has tried at least once to send data
@@ -247,6 +250,7 @@ func TestInstance_remoteWriteHeaders(t *testing.T) {
 	mockServer.refuseRequests.Store(false)
 
 	// Shutdown the instance - even though previous requests failed, remote.Storage should flush pending data
+	cancel()
 	err = instance.Close()
 	assert.NoError(t, err)
 
@@ -332,6 +336,7 @@ func (m *mockPrometheusRemoteWriteServer) close() {
 // poll executes f every interval until ctx is done or cancelled.
 func poll(ctx context.Context, interval time.Duration, f func()) {
 	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
 
 	for {
 		select {