From 3de9f470faad68ab7a7805b0b0f81cfb42991f13 Mon Sep 17 00:00:00 2001 From: Peter Dekkers Date: Tue, 22 Aug 2023 22:39:08 +0200 Subject: [PATCH] Added features to loggers --- .../org/roboquant/charts/MetricsReport.kt | 4 +- .../roboquant/questdb/QuestDBMetricsLogger.kt | 35 +++++++++++++++- .../org/roboquant/questdb/extensions.kt | 23 +++++++++++ .../org/roboquant/questdb/QuestDBFeedTest.kt | 1 + .../org/roboquant/server/routes/details.kt | 2 +- .../org/roboquant/loggers/LastEntryLogger.kt | 21 ++++++---- .../org/roboquant/loggers/MemoryLogger.kt | 20 +++++---- .../org/roboquant/loggers/MetricsLogger.kt | 41 +++++++++++++++---- .../roboquant/loggers/LastEntryLoggerTest.kt | 12 +++--- .../org/roboquant/loggers/MemoryLoggerTest.kt | 6 +-- .../org/roboquant/loggers/SilentLoggerTest.kt | 2 +- .../roboquant/loggers/SkipWarmupLoggerTest.kt | 4 +- .../roboquant/metrics/ReturnsMetricTest.kt | 2 +- .../roboquant/strategies/EMAStrategyTest.kt | 2 +- 14 files changed, 130 insertions(+), 45 deletions(-) diff --git a/roboquant-charts/src/main/kotlin/org/roboquant/charts/MetricsReport.kt b/roboquant-charts/src/main/kotlin/org/roboquant/charts/MetricsReport.kt index b24e47e59..c754845ca 100644 --- a/roboquant-charts/src/main/kotlin/org/roboquant/charts/MetricsReport.kt +++ b/roboquant-charts/src/main/kotlin/org/roboquant/charts/MetricsReport.kt @@ -41,7 +41,7 @@ class MetricsReport( get() = roboquant.logger private val charts - get() = logger.metricNames.map { + get() = logger.getMetricNames().map { { val data = roboquant.logger.getMetric(it) val chart = TimeSeriesChart(data) @@ -92,7 +92,7 @@ class MetricsReport( private fun metricsToHTML(): String { - val metricsMap = logger.metricNames.map { it to logger.getMetric(it) } + val metricsMap = logger.getMetricNames().map { it to logger.getMetric(it) } val result = StringBuffer() for ((name, metrics) in metricsMap) { result += "
" diff --git a/roboquant-questdb/src/main/kotlin/org/roboquant/questdb/QuestDBMetricsLogger.kt b/roboquant-questdb/src/main/kotlin/org/roboquant/questdb/QuestDBMetricsLogger.kt index 87708dbca..664dcd4df 100644 --- a/roboquant-questdb/src/main/kotlin/org/roboquant/questdb/QuestDBMetricsLogger.kt +++ b/roboquant-questdb/src/main/kotlin/org/roboquant/questdb/QuestDBMetricsLogger.kt @@ -48,6 +48,13 @@ class QuestDBMetricsLogger(dbPath: Path = Config.home / "questdb-metrics" / "db" engine = CairoEngine(config) } + /** + * Load previous runs already in the database, so they are accessible via [getMetric] + */ + fun loadPreviousRuns() { + tables.addAll(engine.tables()) + } + override fun log(results: Map, time: Instant, run: String) { if (results.isEmpty()) return if (! tables.contains(run)) { @@ -67,9 +74,12 @@ class QuestDBMetricsLogger(dbPath: Path = Config.home / "questdb-metrics" / "db" } - override fun getMetric(name: String, run: String): TimeSeries { + /** + * Get a metric for a specific [run] + */ + override fun getMetric(metricName: String, run: String): TimeSeries { val result = mutableListOf() - engine.query("select * from '$run' where metric='$name'") { + engine.query("select * from '$run' where metric='$metricName'") { while (hasNext()) { val r = this.record val o = Observation(ofEpochMicro(r.getTimestamp(1)), r.getDouble(0)) @@ -79,11 +89,32 @@ class QuestDBMetricsLogger(dbPath: Path = Config.home / "questdb-metrics" / "db" return TimeSeries(result) } + /** + * get a specific metric for all runs + */ + override fun getMetric(metricName: String): Map { + val result = mutableMapOf() + for (table in tables) { + val v = getMetric(metricName, table) + if (v.isNotEmpty()) result[table] = v + } + return result + } + override fun start(run: String, timeframe: Timeframe) { // engine.update("drop table $run") tables.remove(run) } + + override fun getMetricNames(run: String): Set { + return engine.distictSymbol(run, "name").toSortedSet() + } + + + override val runs: Set + get() = engine.tables().toSet() + private fun createTable(name: String) { engine.update( """CREATE TABLE IF NOT EXISTS '$name' ( diff --git a/roboquant-questdb/src/main/kotlin/org/roboquant/questdb/extensions.kt b/roboquant-questdb/src/main/kotlin/org/roboquant/questdb/extensions.kt index 1406b74c2..1e56a5d5f 100644 --- a/roboquant-questdb/src/main/kotlin/org/roboquant/questdb/extensions.kt +++ b/roboquant-questdb/src/main/kotlin/org/roboquant/questdb/extensions.kt @@ -38,6 +38,29 @@ internal inline fun CairoEngine.query(query: String, block: RecordCursor.() -> U } +internal fun CairoEngine.distictSymbol(tableName: String, column: String) : Set{ + SqlExecutionContextImpl(this, 1).use { ctx -> + sqlCompiler.use { + val sql = "SELECT DISTINCT $column from '$tableName" + val fact = it.compile(sql, ctx).recordCursorFactory + fact.use { + val result = mutableSetOf() + fact.getCursor(ctx).use { cursor -> + while(cursor.hasNext()) { + val r= cursor.record + val s = r.getSym(0) + result.add(s.toString()) + } + } + return result + } + } + } +} + + + + internal fun CairoEngine.insert(tableName: String, block: TableWriter.() -> Unit) { SqlExecutionContextImpl(this, 1).use { ctx -> getWriter(ctx.getTableToken(tableName), tableName).use { diff --git a/roboquant-questdb/src/test/kotlin/org/roboquant/questdb/QuestDBFeedTest.kt b/roboquant-questdb/src/test/kotlin/org/roboquant/questdb/QuestDBFeedTest.kt index 4cf24a808..1c4c933a9 100644 --- a/roboquant-questdb/src/test/kotlin/org/roboquant/questdb/QuestDBFeedTest.kt +++ b/roboquant-questdb/src/test/kotlin/org/roboquant/questdb/QuestDBFeedTest.kt @@ -53,6 +53,7 @@ internal class QuestDBFeedTest { val feed1 = RandomWalkFeed(tf, nAssets = 1, template = Asset("ABC")) val feed2 = RandomWalkFeed(tf, nAssets = 1, template = Asset("XYZ")) + // Need to partition when adding out-of-order price actions recorder.record(feed1, "pricebars3", partition = "YEAR") recorder.record(feed2, "pricebars3", append = true) diff --git a/roboquant-server/src/main/kotlin/org/roboquant/server/routes/details.kt b/roboquant-server/src/main/kotlin/org/roboquant/server/routes/details.kt index 5afc5b988..54342dddb 100644 --- a/roboquant-server/src/main/kotlin/org/roboquant/server/routes/details.kt +++ b/roboquant-server/src/main/kotlin/org/roboquant/server/routes/details.kt @@ -62,7 +62,7 @@ private fun FlowContent.echarts(elemId: String, width: String = "100%", height: } private fun FlowContent.metricForm(target: String, run: String, info: RunInfo) { - val metricNames = info.roboquant.logger.metricNames + val metricNames = info.roboquant.logger.getMetricNames() form { hxPost = "/echarts" hxTarget = target diff --git a/roboquant/src/main/kotlin/org/roboquant/loggers/LastEntryLogger.kt b/roboquant/src/main/kotlin/org/roboquant/loggers/LastEntryLogger.kt index 978b3bb81..5e4c17eff 100644 --- a/roboquant/src/main/kotlin/org/roboquant/loggers/LastEntryLogger.kt +++ b/roboquant/src/main/kotlin/org/roboquant/loggers/LastEntryLogger.kt @@ -49,6 +49,9 @@ class LastEntryLogger(var showProgress: Boolean = false) : MetricsLogger { } } + override val runs: Set + get() = history.keys + override fun start(run: String, timeframe: Timeframe) { history.remove(run) if (showProgress) progressBar.start(run, timeframe) @@ -68,27 +71,29 @@ class LastEntryLogger(var showProgress: Boolean = false) : MetricsLogger { /** * Get the unique list of metric names that have been captured */ - override val metricNames: List - get() = history.values.map { it.keys }.flatten().distinct().sorted() + override fun getMetricNames(run: String) : Set { + val values = history[run] ?: return emptySet() + return values.map { it.key }.distinct().toSortedSet() + } /** - * Get results for the metric specified by its [name]. + * Get results for the metric specified by its [metricName]. */ - override fun getMetric(name: String): Map { + override fun getMetric(metricName: String): Map { val result = mutableMapOf() for (run in history.keys) { - val ts = getMetric(name, run) + val ts = getMetric(metricName, run) if (ts.isNotEmpty()) result[run] = ts } return result } /** - * Get results for the metric specified by its [name]. + * Get results for the metric specified by its [metricName]. */ - override fun getMetric(name: String, run: String): TimeSeries { + override fun getMetric(metricName: String, run: String): TimeSeries { val entries = history[run] ?: return TimeSeries(emptyList()) - val v = entries[name] + val v = entries[metricName] val result = if (v == null) emptyList() else listOf(v) return TimeSeries(result) } diff --git a/roboquant/src/main/kotlin/org/roboquant/loggers/MemoryLogger.kt b/roboquant/src/main/kotlin/org/roboquant/loggers/MemoryLogger.kt index 925a000d9..6d016344c 100644 --- a/roboquant/src/main/kotlin/org/roboquant/loggers/MemoryLogger.kt +++ b/roboquant/src/main/kotlin/org/roboquant/loggers/MemoryLogger.kt @@ -72,36 +72,38 @@ class MemoryLogger(var showProgress: Boolean = true) : MetricsLogger { /** * Get all the recorded runs in this logger */ - val runs: Set + override val runs: Set get() = history.keys.toSortedSet() /** * Get the unique list of metric names that have been captured */ - override val metricNames: List - get() = history.values.asSequence().flatten().map { it.metrics.keys }.flatten().distinct().sorted().toList() + override fun getMetricNames(run: String) : Set { + val values = history[run] ?: return emptySet() + return values.map { it.metrics.keys }.flatten().toSortedSet() + } /** - * Get results for a metric specified by its [name]. It will include all the runs for that metric. + * Get results for a metric specified by its [metricName]. It will include all the runs for that metric. */ - override fun getMetric(name: String): Map { + override fun getMetric(metricName: String): Map { val result = mutableMapOf() for (run in history.keys) { - val ts = getMetric(name, run) + val ts = getMetric(metricName, run) if (ts.isNotEmpty()) result[run] = ts } return result.toSortedMap() } /** - * Get results for a metric specified by its [name] for a single [run] + * Get results for a metric specified by its [metricName] for a single [run] */ - override fun getMetric(name: String, run: String): TimeSeries { + override fun getMetric(metricName: String, run: String): TimeSeries { val entries = history[run] ?: return TimeSeries(emptyList()) val values = mutableListOf() val times = mutableListOf() entries.forEach { - val e = it.metrics[name] + val e = it.metrics[metricName] if (e != null) { values.add(e) times.add(it.time) diff --git a/roboquant/src/main/kotlin/org/roboquant/loggers/MetricsLogger.kt b/roboquant/src/main/kotlin/org/roboquant/loggers/MetricsLogger.kt index 7a5f4797c..788b61f6d 100644 --- a/roboquant/src/main/kotlin/org/roboquant/loggers/MetricsLogger.kt +++ b/roboquant/src/main/kotlin/org/roboquant/loggers/MetricsLogger.kt @@ -39,27 +39,50 @@ interface MetricsLogger : Lifecycle { fun log(results: Map, time: Instant, run: String) /** - * Get all the logged data for a specific metric identified by its [name]. + * Get all the logged data for a specific [metricName]. * The result is a Map with the key being the run-name and the value being the [TimeSeries]. * * This is optional to implement for a MetricsLogger since not all metric-loggers store metrics. * Use [metricNames] to see which metrics are available. */ - fun getMetric(name: String): Map = emptyMap() + fun getMetric(metricName: String): Map = buildMap { + runs.forEach { + val v = getMetric(metricName,it) + if (v.isNotEmpty()) put(it, v) + } + } /** - * Get the metric identified by its [name] for a single [run]. The result is a [TimeSeries]. + * Get the metric identified by its [metricName] for a single [run]. + * The result is a [TimeSeries]. * - * This is optional to implement for a MetricsLogger since not all metric-loggers store metrics. - * Use [metricNames] to see which metrics are available. + * This is optional to implement for a MetricsLogger since not all metric-loggers store metrics. + * Use [getMetricNames] to see which metrics are available. + */ + fun getMetric(metricName: String, run: String): TimeSeries = TimeSeries(emptyList()) + + /** + * The set of metric names that are available and can be retrieved with the [getMetric]. + * This across all runs and can be an extensive operation. + */ + fun getMetricNames(): Set = buildSet { + runs.forEach { + val v = getMetricNames(it) + addAll(v) + } + } + + + /** + * Get all available metric-names for a certain [run] */ - fun getMetric(name: String, run: String): TimeSeries = getMetric(name)[run] ?: TimeSeries(emptyList()) + fun getMetricNames(run: String): Set = emptySet() /** - * The list of metric names that are available and can be retrieved with the [getMetric]. + * The list of runs that are available and can be retrieved with the [getMetric]. */ - val metricNames: List - get() = emptyList() + val runs: Set + get() = emptySet() } diff --git a/roboquant/src/test/kotlin/org/roboquant/loggers/LastEntryLoggerTest.kt b/roboquant/src/test/kotlin/org/roboquant/loggers/LastEntryLoggerTest.kt index f4bb8a09e..58400d486 100644 --- a/roboquant/src/test/kotlin/org/roboquant/loggers/LastEntryLoggerTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/loggers/LastEntryLoggerTest.kt @@ -32,15 +32,15 @@ internal class LastEntryLoggerTest { logger.log(metrics, Instant.now(), "test") logger.end("test") - assertTrue(logger.metricNames.isNotEmpty()) - assertContains(logger.metricNames, metrics.keys.first()) + assertTrue(logger.getMetricNames().isNotEmpty()) + assertContains(logger.getMetricNames(), metrics.keys.first()) - val m1 = logger.metricNames.first() + val m1 = logger.getMetricNames().first() val m = logger.getMetric(m1).latestRun() assertTrue(m.isNotEmpty()) logger.reset() - assertTrue(logger.metricNames.isEmpty()) + assertTrue(logger.getMetricNames().isEmpty()) } @Test @@ -55,9 +55,9 @@ internal class LastEntryLoggerTest { } logger.end("test") - assertTrue(logger.metricNames.isNotEmpty()) + assertTrue(logger.getMetricNames().isNotEmpty()) - val m1 = logger.metricNames.first() + val m1 = logger.getMetricNames().first() val m = logger.getMetric(m1).latestRun() assertEquals(m.timeline.sorted(), m.timeline) } diff --git a/roboquant/src/test/kotlin/org/roboquant/loggers/MemoryLoggerTest.kt b/roboquant/src/test/kotlin/org/roboquant/loggers/MemoryLoggerTest.kt index 3b9ab936b..bf86dc905 100644 --- a/roboquant/src/test/kotlin/org/roboquant/loggers/MemoryLoggerTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/loggers/MemoryLoggerTest.kt @@ -34,15 +34,15 @@ internal class MemoryLoggerTest { @Test fun memoryLogger() { val logger = MemoryLogger(showProgress = false) - assertTrue(logger.metricNames.isEmpty()) + assertTrue(logger.getMetricNames().isEmpty()) val metrics = TestData.getMetrics() logger.start("test", Timeframe.INFINITE) logger.log(metrics, Instant.now(), "test") logger.end("test") - assertFalse(logger.metricNames.isEmpty()) - assertEquals(metrics.size, logger.metricNames.size) + assertFalse(logger.getMetricNames().isEmpty()) + assertEquals(metrics.size, logger.getMetricNames().size) val t = logger.getMetric(metrics.keys.first()).latestRun() assertEquals(1, t.size) diff --git a/roboquant/src/test/kotlin/org/roboquant/loggers/SilentLoggerTest.kt b/roboquant/src/test/kotlin/org/roboquant/loggers/SilentLoggerTest.kt index cdb9a2eac..a8b262153 100644 --- a/roboquant/src/test/kotlin/org/roboquant/loggers/SilentLoggerTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/loggers/SilentLoggerTest.kt @@ -29,7 +29,7 @@ internal class SilentLoggerTest { val logger = SilentLogger() logger.log(TestData.getMetrics(), Instant.now(), "test") assertEquals(1, logger.events) - assertTrue(logger.metricNames.isEmpty()) + assertTrue(logger.getMetricNames().isEmpty()) assertTrue(logger.getMetric("key1").isEmpty()) logger.log(TestData.getMetrics(), Instant.now(), "test") diff --git a/roboquant/src/test/kotlin/org/roboquant/loggers/SkipWarmupLoggerTest.kt b/roboquant/src/test/kotlin/org/roboquant/loggers/SkipWarmupLoggerTest.kt index c21e4aea8..c811b39ed 100644 --- a/roboquant/src/test/kotlin/org/roboquant/loggers/SkipWarmupLoggerTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/loggers/SkipWarmupLoggerTest.kt @@ -34,12 +34,12 @@ internal class SkipWarmupLoggerTest { repeat(9) { logger.log(metrics, Instant.now(), "test") } - assertTrue(logger.metricNames.isEmpty()) + assertTrue(logger.getMetricNames().isEmpty()) repeat(4) { logger.log(metrics, Instant.now(), "test") } - assertFalse(logger.metricNames.isEmpty()) + assertFalse(logger.getMetricNames().isEmpty()) } } \ No newline at end of file diff --git a/roboquant/src/test/kotlin/org/roboquant/metrics/ReturnsMetricTest.kt b/roboquant/src/test/kotlin/org/roboquant/metrics/ReturnsMetricTest.kt index 955e4cf6c..1dc1fdbd7 100644 --- a/roboquant/src/test/kotlin/org/roboquant/metrics/ReturnsMetricTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/metrics/ReturnsMetricTest.kt @@ -45,7 +45,7 @@ internal class ReturnsMetricTest { val feed = RandomWalkFeed.lastYears(2) val rq = Roboquant(EMAStrategy(), metric, logger = MemoryLogger(showProgress = false)) rq.run(feed) - assertContains(rq.logger.metricNames, "returns.sharperatio") + assertContains(rq.logger.getMetricNames(), "returns.sharperatio") } @Test diff --git a/roboquant/src/test/kotlin/org/roboquant/strategies/EMAStrategyTest.kt b/roboquant/src/test/kotlin/org/roboquant/strategies/EMAStrategyTest.kt index 484b94518..2c2a1eb05 100644 --- a/roboquant/src/test/kotlin/org/roboquant/strategies/EMAStrategyTest.kt +++ b/roboquant/src/test/kotlin/org/roboquant/strategies/EMAStrategyTest.kt @@ -33,7 +33,7 @@ internal class EMAStrategyTest { strategy.recording = true val roboquant = Roboquant(strategy, logger = MemoryLogger(false)) roboquant.run(TestData.feed, name = "test") - val names = roboquant.logger.metricNames + val names = roboquant.logger.getMetricNames() assertTrue(names.isNotEmpty()) val metrics = roboquant.logger.getMetric(names.first()).latestRun()
$name