diff --git a/.github/workflows/integration-test-containers.yml b/.github/workflows/integration-test-containers.yml index 21595d0e356..33bfe6d3710 100644 --- a/.github/workflows/integration-test-containers.yml +++ b/.github/workflows/integration-test-containers.yml @@ -50,7 +50,7 @@ jobs: run: | ansible-galaxy install -r ansible/requirements.yml - BUILD_MULTI_ARCH="${{ contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') || github.event_name == 'push' }}" + BUILD_MULTI_ARCH="${{ contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') || github.event_name == 'push' || github.event_name == 'schedule' }}" # build_multi_arch passed in as json to ensure boolean type ansible-playbook \ diff --git a/RELEASED_VERSIONS b/RELEASED_VERSIONS index 30780bffe68..e02be4ba71f 100644 --- a/RELEASED_VERSIONS +++ b/RELEASED_VERSIONS @@ -29,4 +29,5 @@ 3.13.4 3.74.7 # Rox release 3.74.7 by roxbot at Tue Oct 24 13:38:02 UTC 2023 3.15.2 4.1.5 # Rox release 4.1.5 by roxbot at Thu Nov 9 02:58:36 UTC 2023 3.17.0 4.3.0 # Rox release 4.3.0 by roxbot at Wed Nov 15 08:14:26 UTC 2023 -3.16.2 4.2.3 # Rox release 4.2.3 by roxbot at Wed Nov 22 00:02:20 UTC 2023 \ No newline at end of file +3.16.2 4.2.3 # Rox release 4.2.3 by roxbot at Wed Nov 22 00:02:20 UTC 2023 +3.17.0 4.3.1 # Rox release 4.3.1 by roxbot at Mon Dec 11 20:36:15 UTC 2023 diff --git a/collector/container/Dockerfile b/collector/container/Dockerfile index 3ab23771579..17d789c3689 100644 --- a/collector/container/Dockerfile +++ b/collector/container/Dockerfile @@ -30,6 +30,7 @@ COPY LICENSE-kernel-modules.txt /kernel-modules/LICENSE COPY container/libs/libsinsp-wrapper.so /usr/local/lib/ COPY container/bin/collector /usr/local/bin/ COPY container/bin/self-checks /usr/local/bin/self-checks +COPY container/status-check.sh /usr/local/bin/status-check.sh RUN echo '/usr/local/lib' > /etc/ld.so.conf.d/usrlocallib.conf && \ ldconfig && \ @@ -38,6 +39,14 @@ RUN echo '/usr/local/lib' > /etc/ld.so.conf.d/usrlocallib.conf && \ EXPOSE 8080 9090 +HEALTHCHECK \ + # health checks within the first 5s are not counted as failure + --start-period=5s \ + # perform health check every 5s + --interval=5s \ + # the command uses /ready API + CMD /usr/local/bin/status-check.sh + ENTRYPOINT ["/bootstrap.sh"] CMD collector-wrapper.sh \ diff --git a/collector/container/status-check.sh b/collector/container/status-check.sh new file mode 100755 index 00000000000..9a8d4fa5be1 --- /dev/null +++ b/collector/container/status-check.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# /ready API will return the following formatted response: +# { +# "collector" : { +# "drops" : 0, +# "events" : 9330070, +# "node" : "node.name", +# "preemptions" : 0 +# }, +# "status" : "ok" +# } +# +# Take the status line, split it by ":" and trim spaces and quotes. +STATUS=$(curl -s localhost:8080/ready | grep 'status' | awk -F ':' '{print $2}' | tr -d '"' | tr -d ' ') + +if [[ "${STATUS}" = "ok" ]]; then + exit 0 +else + exit 1 +fi diff --git a/collector/lib/CollectorConfig.cpp b/collector/lib/CollectorConfig.cpp index 24e006bdbae..bf5f9348c7c 100644 --- a/collector/lib/CollectorConfig.cpp +++ b/collector/lib/CollectorConfig.cpp @@ -162,6 +162,7 @@ CollectorConfig::CollectorConfig(CollectorArgs* args) { HandleAfterglowEnvVars(); HandleConnectionStatsEnvVars(); + HandleSinspEnvVars(); host_config_ = ProcessHostHeuristics(*this); } @@ -175,6 +176,15 @@ void CollectorConfig::HandleAfterglowEnvVars() { afterglow_period_micros_ = static_cast(atof(afterglow_period) * 1000000); } + const int64_t max_afterglow_period_micros = 300000000; // 5 minutes + + if (afterglow_period_micros_ > max_afterglow_period_micros) { + CLOG(WARNING) << "User set afterglow period of " << afterglow_period_micros_ / 1000000 + << "s is greater than the maximum allowed afterglow period of " << max_afterglow_period_micros / 1000000 << "s"; + CLOG(WARNING) << "Setting the afterglow period to " << max_afterglow_period_micros / 1000000 << "s"; + afterglow_period_micros_ = max_afterglow_period_micros; + } + if (enable_afterglow_ && afterglow_period_micros_ > 0) { CLOG(INFO) << "Afterglow is enabled"; return; @@ -239,6 +249,31 @@ void CollectorConfig::HandleConnectionStatsEnvVars() { } } +void CollectorConfig::HandleSinspEnvVars() { + const char* envvar; + + sinsp_cpu_per_buffer_ = DEFAULT_CPU_FOR_EACH_BUFFER; + sinsp_buffer_size_ = DEFAULT_DRIVER_BUFFER_BYTES_DIM; + + if ((envvar = std::getenv("ROX_COLLECTOR_SINSP_CPU_PER_BUFFER")) != NULL) { + try { + sinsp_cpu_per_buffer_ = std::stoi(envvar); + CLOG(INFO) << "Sinsp cpu per buffer: " << sinsp_cpu_per_buffer_; + } catch (...) { + CLOG(ERROR) << "Invalid cpu per buffer value: '" << envvar << "'"; + } + } + + if ((envvar = std::getenv("ROX_COLLECTOR_SINSP_BUFFER_SIZE")) != NULL) { + try { + sinsp_buffer_size_ = std::stoi(envvar); + CLOG(INFO) << "Sinsp buffer size: " << sinsp_buffer_size_; + } catch (...) { + CLOG(ERROR) << "Invalid buffer size value: '" << envvar << "'"; + } + } +} + bool CollectorConfig::TurnOffScrape() const { return turn_off_scrape_; } diff --git a/collector/lib/CollectorConfig.h b/collector/lib/CollectorConfig.h index 1bfc8e8c122..5b1555c78ed 100644 --- a/collector/lib/CollectorConfig.h +++ b/collector/lib/CollectorConfig.h @@ -76,6 +76,8 @@ class CollectorConfig { const std::vector& GetConnectionStatsQuantiles() const { return connection_stats_quantiles_; } double GetConnectionStatsError() const { return connection_stats_error_; } unsigned int GetConnectionStatsWindow() const { return connection_stats_window_; } + unsigned int GetSinspBufferSize() const { return sinsp_buffer_size_; } + unsigned int GetSinspCpuPerBuffer() const { return sinsp_cpu_per_buffer_; } std::shared_ptr grpc_channel; @@ -105,10 +107,16 @@ class CollectorConfig { double connection_stats_error_; unsigned int connection_stats_window_; + // One ring buffer will be initialized for this many CPUs + unsigned int sinsp_cpu_per_buffer_; + // Size of one ring buffer, in bytes + unsigned int sinsp_buffer_size_; + Json::Value tls_config_; void HandleAfterglowEnvVars(); void HandleConnectionStatsEnvVars(); + void HandleSinspEnvVars(); }; std::ostream& operator<<(std::ostream& os, const CollectorConfig& c); diff --git a/collector/lib/ConnTracker.cpp b/collector/lib/ConnTracker.cpp index ce9656bd2ae..9c036b0f8ce 100644 --- a/collector/lib/ConnTracker.cpp +++ b/collector/lib/ConnTracker.cpp @@ -92,7 +92,7 @@ IPNet ConnectionTracker::NormalizeAddressNoLock(const Address& address) const { } if (enable_external_ips_) { - return IPNet(address, 0, true); + return IPNet(address, address.length() * 8); } // Otherwise, associate it to "rest of the internet". diff --git a/collector/lib/KernelDriver.h b/collector/lib/KernelDriver.h index 6b0bebcdcb7..8ceadb2f927 100644 --- a/collector/lib/KernelDriver.h +++ b/collector/lib/KernelDriver.h @@ -43,7 +43,9 @@ class KernelDriverEBPF : public IKernelDriver { std::unordered_set ppm_sc; try { - inspector.open_bpf(SysdigService::kProbePath, DEFAULT_DRIVER_BUFFER_BYTES_DIM, ppm_sc, tp_set); + inspector.open_bpf(SysdigService::kProbePath, + config.GetSinspBufferSize(), + ppm_sc, tp_set); } catch (const sinsp_exception& ex) { CLOG(WARNING) << ex.what(); return false; @@ -99,8 +101,8 @@ class KernelDriverCOREEBPF : public IKernelDriver { } try { - inspector.open_modern_bpf(DEFAULT_DRIVER_BUFFER_BYTES_DIM, - DEFAULT_CPU_FOR_EACH_BUFFER, + inspector.open_modern_bpf(config.GetSinspBufferSize(), + config.GetSinspCpuPerBuffer(), true, ppm_sc, tp_set); } catch (const sinsp_exception& ex) { if (config.CoReBPFHardfail()) { diff --git a/collector/test/ConnTrackerTest.cpp b/collector/test/ConnTrackerTest.cpp index 6a19e5628ee..292e4dc0dbf 100644 --- a/collector/test/ConnTrackerTest.cpp +++ b/collector/test/ConnTrackerTest.cpp @@ -386,13 +386,13 @@ TEST(ConnTrackerTest, TestNormalizedEnableExternalIPs) { // from private network Connection conn1_normalized("xyz", Endpoint(IPNet(), 9999), Endpoint(IPNet(Address(10, 1, 1, 42), 0, true), 0), L4Proto::TCP, true); // from unmatched external IP - Connection conn2_normalized("xyz", Endpoint(IPNet(), 9999), Endpoint(IPNet(Address(35, 127, 0, 15), 0, true), 0), L4Proto::TCP, true); + Connection conn2_normalized("xyz", Endpoint(IPNet(), 9999), Endpoint(IPNet(Address(35, 127, 0, 15), 32, false), 0), L4Proto::TCP, true); // from matched external IP Connection conn3_normalized("xyz", Endpoint(IPNet(), 9999), Endpoint(IPNet(Address(35, 127, 1, 0), 24, false), 0), L4Proto::TCP, true); // to private network Connection conn4_normalized("xyz", Endpoint(), Endpoint(IPNet(Address(10, 1, 1, 42), 0, true), 9999), L4Proto::TCP, false); // to unmatched external IP - Connection conn5_normalized("xyz", Endpoint(), Endpoint(IPNet(Address(35, 127, 0, 15), 0, true), 54321), L4Proto::TCP, false); + Connection conn5_normalized("xyz", Endpoint(), Endpoint(IPNet(Address(35, 127, 0, 15), 32, false), 54321), L4Proto::TCP, false); // to matched external IP Connection conn6_normalized("xyz", Endpoint(), Endpoint(IPNet(Address(35, 127, 1, 0), 24, false), 54321), L4Proto::TCP, false); diff --git a/docs/references.md b/docs/references.md index 4d5a5a3f988..5f6b0c9665f 100644 --- a/docs/references.md +++ b/docs/references.md @@ -49,6 +49,16 @@ This is enabled by default. - `ROX_COLLECTOR_CONNECTION_STATS_WINDOW`: the length of the sliding time window in minutes. Default: `60` +* `ROX_COLLECTOR_SINSP_CPU_PER_BUFFER`: Allows to control how many sinsp +buffers are going to be allocated. The resulting number of buffers will be +calculated as the overall number of CPU cores available divided by this +value. The default value is 1, meaning one buffer for each CPU. The value 0 is +special, it instructs sinsp to allocate only one buffer no matter how many CPUs +are there. This parameter affects CO-RE BPF only. + +* `ROX_COLLECTOR_SINSP_BUFFER_SIZE`: Specifies the size of a sinsp buffer in +bytes. The default value is 16 MB. + NOTE: Using environment variables is a preferred way of configuring Collector, so if you're adding a new configuration knob, keep this in mind. diff --git a/integration-tests/suites/base.go b/integration-tests/suites/base.go index a0f8272c8a8..bf1059a7697 100644 --- a/integration-tests/suites/base.go +++ b/integration-tests/suites/base.go @@ -70,8 +70,40 @@ func (s *IntegrationTestSuiteBase) StartCollector(disableGRPC bool, options *com s.Require().NoError(s.Collector().Setup(options)) s.Require().NoError(s.Collector().Launch()) - // wait for self-check process to guarantee collector is started - s.Sensor().WaitProcessesN(s.Collector().ContainerID, 30*time.Second, 1) + // Verify if the image we test has a health check. There are some CI + // configurations, where it's not the case. If something went wrong and we + // get an error, treat it as no health check was found for the sake of + // robustness. + hasHealthCheck, err := s.findContainerHealthCheck("collector", + s.Collector().ContainerID) + + if hasHealthCheck && err == nil { + // Wait for collector to report healthy, includes initial setup and + // probes loading. It doesn't make sense to wait for very long, limit + // it to 1 min. + _, err := s.waitForContainerToBecomeHealthy( + "collector", + s.Collector().ContainerID, + defaultWaitTickSeconds, 1*time.Minute) + s.Require().NoError(err) + } else { + fmt.Println("No HealthCheck found, do not wait for collector to become healthy") + + // No way to figure out when all the services up and running, skip this + // phase. + } + + // wait for the canary process to guarantee collector is started + selfCheckOk := s.Sensor().WaitProcessesN( + s.Collector().ContainerID, 30*time.Second, 1, func() { + // Self-check process is not going to be sent via GRPC, instead + // create at least one canary process to make sure everything is + // fine. + fmt.Println("Spawn a canary process") + _, err = s.execContainer("collector", []string{"echo"}) + s.Require().NoError(err) + }) + s.Require().True(selfCheckOk) } // StopCollector will tear down the collector container and stop @@ -235,17 +267,33 @@ func (s *IntegrationTestSuiteBase) launchContainer(name string, args ...string) return outLines[len(outLines)-1], err } -func (s *IntegrationTestSuiteBase) waitForContainerToExit(containerName, containerID string, tickSeconds time.Duration) (bool, error) { +// Wait for a container to become a certain status. +// - tickSeconds -- how often to check for the status +// - timeoutThreshold -- the overall time limit for waiting, +// defaulting to 30 min if zero +// - filter -- description of the desired status +func (s *IntegrationTestSuiteBase) waitForContainerStatus( + containerName string, + containerID string, + tickSeconds time.Duration, + timeoutThreshold time.Duration, + filter string) (bool, error) { + cmd := []string{ common.RuntimeCommand, "ps", "-qa", "--filter", "id=" + containerID, - "--filter", "status=exited", + "--filter", filter, } start := time.Now() tick := time.Tick(tickSeconds) tickElapsed := time.Tick(1 * time.Minute) - timeout := time.After(30 * time.Minute) + + if timeoutThreshold == 0 { + timeoutThreshold = 30 * time.Minute + } + timeout := time.After(timeoutThreshold) + for { select { case <-tick: @@ -256,17 +304,76 @@ func (s *IntegrationTestSuiteBase) waitForContainerToExit(containerName, contain return true, nil } if err != nil { - fmt.Printf("Retrying waitForContainerToExit(%s, %s): Error: %v\n", containerName, containerID, err) + fmt.Printf("Retrying waitForContainerStatus(%s, %s): Error: %v\n", + containerName, containerID, err) } case <-timeout: - fmt.Printf("Timed out waiting for container %s to exit, elapsed Time: %s\n", containerName, time.Since(start)) - return false, nil + fmt.Printf("Timed out waiting for container %s to become %s, elapsed Time: %s\n", + containerName, filter, time.Since(start)) + return false, fmt.Errorf("Timeout waiting for container %s to become %s after %v", + containerName, filter, timeoutThreshold) case <-tickElapsed: - fmt.Printf("Waiting for container: %s, elapsed time: %s\n", containerName, time.Since(start)) + fmt.Printf("Waiting for container %s to become %s, elapsed time: %s\n", + containerName, filter, time.Since(start)) } } } +// Find a HealthCheck section in the specified container and verify it's what +// we expect. This function would be used to wait until the health check +// reports healthy, so be conservative and report true only if absolutely +// certain. +func (s *IntegrationTestSuiteBase) findContainerHealthCheck( + containerName string, + containerID string) (bool, error) { + + cmd := []string{ + common.RuntimeCommand, "inspect", "-f", + "'{{ .Config.Healthcheck }}'", containerID, + } + + output, err := s.Executor().Exec(cmd...) + if err != nil { + return false, err + } + + outLines := strings.Split(output, "\n") + lastLine := outLines[len(outLines)-1] + + // Clearly no HealthCheck section + if lastLine == "" { + return false, nil + } + + // If doesn't contain an expected command, do not consider it to be a valid + // health check + if strings.Contains(lastLine, "CMD-SHELL /usr/local/bin/status-check.sh") { + return true, nil + } else { + return false, nil + } +} + +func (s *IntegrationTestSuiteBase) waitForContainerToBecomeHealthy( + containerName string, + containerID string, + tickSeconds time.Duration, + timeoutThreshold time.Duration) (bool, error) { + + return s.waitForContainerStatus(containerName, containerID, tickSeconds, + timeoutThreshold, "health=healthy") +} + +func (s *IntegrationTestSuiteBase) waitForContainerToExit( + containerName string, + containerID string, + tickSeconds time.Duration, + timeoutThreshold time.Duration) (bool, error) { + + return s.waitForContainerStatus(containerName, containerID, tickSeconds, + timeoutThreshold, "status=exited") +} + func (s *IntegrationTestSuiteBase) execContainer(containerName string, command []string) (string, error) { cmd := []string{common.RuntimeCommand, "exec", containerName} cmd = append(cmd, command...) @@ -342,7 +449,7 @@ func (s *IntegrationTestSuiteBase) RunCollectorBenchmark() { containerID, err := s.launchContainer(benchmarkName, benchmarkArgs...) s.Require().NoError(err) - _, err = s.waitForContainerToExit(benchmarkName, containerID, defaultWaitTickSeconds) + _, err = s.waitForContainerToExit(benchmarkName, containerID, defaultWaitTickSeconds, 0) s.Require().NoError(err) benchmarkLogs, err := s.containerLogs("benchmark") diff --git a/integration-tests/suites/benchmark.go b/integration-tests/suites/benchmark.go index 0d079ae5af3..d9fbfd8a400 100644 --- a/integration-tests/suites/benchmark.go +++ b/integration-tests/suites/benchmark.go @@ -81,7 +81,7 @@ func (b *BenchmarkTestSuiteBase) RunInitContainer() { containerID, err := b.launchContainer("host-init", cmd...) require.NoError(b.T(), err) - if finished, _ := b.waitForContainerToExit("host-init", containerID, 5*time.Second); !finished { + if finished, _ := b.waitForContainerToExit("host-init", containerID, 5*time.Second, 0); !finished { logs, err := b.containerLogs("host-init") if err == nil { fmt.Println(logs) diff --git a/integration-tests/suites/image_json.go b/integration-tests/suites/image_json.go index 014cfec82b9..784d86be3c5 100644 --- a/integration-tests/suites/image_json.go +++ b/integration-tests/suites/image_json.go @@ -23,7 +23,7 @@ func (s *ImageLabelJSONTestSuite) TestRunImageWithJSONLabel() { containerID, err := s.launchContainer(name, image) s.Require().NoError(err) - _, err = s.waitForContainerToExit(name, containerID, defaultWaitTickSeconds) + _, err = s.waitForContainerToExit(name, containerID, defaultWaitTickSeconds, 0) s.Require().NoError(err) } diff --git a/integration-tests/suites/mock_sensor/expect_proc.go b/integration-tests/suites/mock_sensor/expect_proc.go index cd0511a7a44..b4e0bf21088 100644 --- a/integration-tests/suites/mock_sensor/expect_proc.go +++ b/integration-tests/suites/mock_sensor/expect_proc.go @@ -14,11 +14,11 @@ import ( func (s *MockSensor) ExpectProcessesN(t *testing.T, containerID string, timeout time.Duration, n int) []types.ProcessInfo { return s.waitProcessesN(func() { assert.FailNowf(t, "timed out", "found %d processes (expected %d)", len(s.Processes(containerID)), n) - }, containerID, timeout, n) + }, containerID, timeout, n, 0, func() {}) } -func (s *MockSensor) WaitProcessesN(containerID string, timeout time.Duration, n int) bool { - return len(s.waitProcessesN(func() {}, containerID, timeout, n)) == n +func (s *MockSensor) WaitProcessesN(containerID string, timeout time.Duration, n int, tickFn func()) bool { + return len(s.waitProcessesN(func() {}, containerID, timeout, n, 0, tickFn)) >= n } func (s *MockSensor) ExpectProcesses( @@ -100,14 +100,37 @@ func (s *MockSensor) ExpectLineages(t *testing.T, containerID string, timeout ti } -func (s *MockSensor) waitProcessesN(timeoutFn func(), containerID string, timeout time.Duration, n int) []types.ProcessInfo { - if len(s.Processes(containerID)) == n { +// Wait for expected number of processes to show up in a specified container. +// - timeoutFn: will be invoked after the timeout expiration +// - containerID: the target container for searching processes +// - timeout: maximum waiting time +// - n: expected number of processes +// - tickSeconds: how often to wake up within the timeout time, default is 1s +// - tickFn: what to do when ticking, could be used to trigger expected number +// of processes +func (s *MockSensor) waitProcessesN( + timeoutFn func(), + containerID string, + timeout time.Duration, + n int, + tickSeconds time.Duration, + tickFn func()) []types.ProcessInfo { + + if len(s.Processes(containerID)) >= n { return s.Processes(containerID) } + if tickSeconds == 0 { + tickSeconds = 1 * time.Second + } + + tick := time.Tick(tickSeconds) + loop: for { select { + case <-tick: + tickFn() case <-time.After(timeout): timeoutFn() return make([]types.ProcessInfo, 0) @@ -116,7 +139,7 @@ loop: continue loop } - if len(s.Processes(containerID)) == n { + if len(s.Processes(containerID)) >= n { return s.Processes(containerID) } } diff --git a/integration-tests/suites/perf_event_open.go b/integration-tests/suites/perf_event_open.go index 1e51f13816d..957a88d9563 100644 --- a/integration-tests/suites/perf_event_open.go +++ b/integration-tests/suites/perf_event_open.go @@ -30,7 +30,7 @@ func (s *PerfEventOpenTestSuite) TestReadingTracepoints() { containerID, err := s.launchContainer("perf-event-open", "--privileged", image, "", "STDOUT") s.Require().NoError(err) - if finished, _ := s.waitForContainerToExit("perf-event-open", containerID, 5*time.Second); finished { + if finished, _ := s.waitForContainerToExit("perf-event-open", containerID, 5*time.Second, 0); finished { logs, err := s.containerLogs("perf-event-open") if err != nil { fmt.Println(logs) diff --git a/kernel-modules/KERNEL_VERSIONS b/kernel-modules/KERNEL_VERSIONS index 9db5fc10ead..f0b25501883 100644 --- a/kernel-modules/KERNEL_VERSIONS +++ b/kernel-modules/KERNEL_VERSIONS @@ -510,6 +510,7 @@ 4.12.14-122.176-default 4.12.14-122.179-default 4.12.14-122.183-default +4.12.14-122.186-default 4.12.14-122.20-default 4.12.14-122.23-default 4.12.14-122.26-default @@ -2499,6 +2500,8 @@ 4.18.0-522.el8.x86_64 4.18.0-526.el8.x86_64 4.18.0-527.el8.x86_64 +4.18.0-529.el8.x86_64 +4.18.0-532.el8.x86_64 4.18.0-80.1.2.el8_0.x86_64 4.18.0-80.1.2.rt9.145.el8_0.x86_64 4.18.0-80.11.1.el8_0.x86_64 @@ -2906,6 +2909,7 @@ 4.4.0-1094-aws 4.4.0-1094-fips 4.4.0-1095-aws +4.4.0-1095-fips 4.4.0-1096-aws 4.4.0-1097-aws 4.4.0-1098-aws @@ -3632,6 +3636,8 @@ 5.10.0-25-cloud-amd64 5.10.0-26-amd64 5.10.0-26-cloud-amd64 +5.10.0-27-amd64 +5.10.0-27-cloud-amd64 5.10.0-3-amd64 5.10.0-3-cloud-amd64 5.10.0-4-amd64 @@ -3857,6 +3863,8 @@ 5.10.197-16919.404.17-cos 5.10.197-16919.404.19-cos 5.10.197-16919.404.21-cos +5.10.197-16919.404.26-cos +5.10.197-16919.404.31-cos 5.10.197-16919.404.4-cos 5.10.197-16919.404.9-cos 5.10.197-186.748.amzn2.x86_64 @@ -4424,8 +4432,12 @@ 5.14.0-284.43.1.rt14.328.el9_2.x86_64 5.14.0-284.44.1.el9_2.x86_64 5.14.0-284.44.1.rt14.329.el9_2.x86_64 +5.14.0-284.45.1.el9_2.x86_64 +5.14.0-284.45.1.rt14.330.el9_2.x86_64 5.14.0-3-amd64 5.14.0-3-cloud-amd64 +5.14.0-362.13.1.el9_3.x86_64 +5.14.0-362.13.1.el9_3.x86_64+rt 5.14.0-362.8.1.el9_3.x86_64 5.14.0-362.8.1.el9_3.x86_64+rt 5.14.0-4-amd64 @@ -4491,6 +4503,7 @@ 5.14.20-200.fc34.x86_64 5.14.20-300.fc35.x86_64 5.14.21-150400.22-default +5.14.21-150400.24.100-default 5.14.21-150400.24.11-default 5.14.21-150400.24.18-default 5.14.21-150400.24.21-default @@ -4518,6 +4531,7 @@ 5.14.21-150500.55.28-default 5.14.21-150500.55.31-default 5.14.21-150500.55.36-default +5.14.21-150500.55.39-default 5.14.21-150500.55.7-default 5.14.3-300.fc34.x86_64 5.14.3-300.fc35.x86_64 @@ -4982,6 +4996,8 @@ 5.15.133-17162.336.25-cos 5.15.133-17162.336.27-cos 5.15.133-17162.336.28-cos +5.15.133-17162.336.35-cos +5.15.133-17162.336.43-cos 5.15.133-17162.336.7-cos 5.15.133-17162.336.9-cos 5.15.133-17412.226.10-cos @@ -4996,11 +5012,14 @@ 5.15.133-17412.226.35-cos 5.15.133-17412.226.41-cos 5.15.133-17412.226.43-cos +5.15.133-17412.226.52-cos +5.15.133-17412.226.62-cos 5.15.133-flatcar 5.15.136-flatcar 5.15.138-flatcar 5.15.14-100.fc34.x86_64 5.15.14-200.fc35.x86_64 +5.15.142-flatcar 5.15.15-100.fc34.x86_64 5.15.15-200.fc35.x86_64 5.15.16-100.fc34.x86_64 @@ -6405,6 +6424,7 @@ 5.4.17-2136.324.5.3.el7uek.x86_64 5.4.17-2136.325.5.1.el7uek.x86_64 5.4.17-2136.325.5.el7uek.x86_64 +5.4.17-2136.326.6.el7uek.x86_64 5.4.170-1.el7.elrepo.x86_64 5.4.170-16108.604.1-cos 5.4.170-16108.604.11-cos @@ -6609,6 +6629,8 @@ 5.4.261-174.360.amzn2.x86_64 5.4.262-1.el7.elrepo.x86_64 5.4.263-1.el7.elrepo.x86_64 +5.4.264-1.el7.elrepo.x86_64 +5.4.265-1.el7.elrepo.x86_64 5.4.38-17.76.amzn2.x86_64 5.4.46-19.75.amzn2.x86_64 5.4.46-23.77.amzn2.x86_64 @@ -7004,7 +7026,11 @@ 6.1.0-14-generic 6.1.0-15-amd64 6.1.0-15-cloud-amd64 +6.1.0-16-amd64 +6.1.0-16-cloud-amd64 6.1.0-16-generic +6.1.0-17-amd64 +6.1.0-17-cloud-amd64 6.1.0-2-amd64 6.1.0-2-cloud-amd64 6.1.0-3-amd64 @@ -7082,6 +7108,8 @@ 6.1.58-17800.66.27-cos 6.1.58-17800.66.32-cos 6.1.58-17800.66.33-cos +6.1.58-17800.66.43-cos +6.1.58-17800.66.54-cos 6.1.58-flatcar 6.1.6-100.fc36.x86_64 6.1.6-200.fc37.x86_64 @@ -7092,6 +7120,9 @@ 6.1.62-flatcar 6.1.64-18080.0.0-cos 6.1.64-18091.0.0-cos +6.1.64-18106.0.0-cos +6.1.66-flatcar +6.1.69-18125.0.0-cos 6.1.7-100.fc36.x86_64 6.1.7-200.fc37.x86_64 6.1.8-100.fc36.x86_64 @@ -7369,3 +7400,9 @@ 6.6.5-200.fc39.x86_64 6.6.6-100.fc38.x86_64 6.6.6-200.fc39.x86_64 +6.6.7-100.fc38.x86_64 +6.6.7-200.fc39.x86_64 +6.6.8-100.fc38.x86_64 +6.6.8-200.fc39.x86_64 +6.6.9-100.fc38.x86_64 +6.6.9-200.fc39.x86_64