Skip to content

Commit

Permalink
Merge early closes and late opens properly in MHDB. (#7712)
Browse files Browse the repository at this point in the history 
A more specific entry might contain some of the same early close/late open dates as the common entry but with different times. This makes sure this is handle and the more specific ones take precedence.
  • Loading branch information
@jhonabreul
jhonabreul authored Jan 22, 2024
1 parent 580a043 commit 1c93df8
Show file tree
Hide file tree
Showing 2 changed files with 180 additions and 2 deletions.
21 changes: 19 additions & 2 deletions Common/Util/MarketHoursDatabaseJsonConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,12 +305,12 @@ public MarketHoursDatabase.Entry Convert(MarketHoursDatabase.Entry underlyingEnt

if (marketEntry.ExchangeHours.EarlyCloses.Count > 0 )
{
earlyCloses = earlyCloses.Union(marketEntry.ExchangeHours.EarlyCloses).ToDictionary();
earlyCloses = MergeLateOpensAndEarlyCloses(marketEntry.ExchangeHours.EarlyCloses, earlyCloses);
}

if (marketEntry.ExchangeHours.LateOpens.Count > 0)
{
lateOpens = lateOpens.Union(marketEntry.ExchangeHours.LateOpens).ToDictionary();
lateOpens = MergeLateOpensAndEarlyCloses(marketEntry.ExchangeHours.LateOpens, lateOpens);
}
}

Expand All @@ -330,6 +330,23 @@ private void SetSegmentsForDay(SecurityExchangeHours hours, DayOfWeek day, out L
segments = new List<MarketHoursSegment>();
}
}

/// <summary>
/// Merges the late opens or early closes from the common entry (with wildcards) with the specific entry
/// (e.g. Indices-usa-[*] with Indices-usa-VIX).
/// The specific entry takes precedence.
/// </summary>
private static Dictionary<DateTime, TimeSpan> MergeLateOpensAndEarlyCloses(IReadOnlyDictionary<DateTime, TimeSpan> common,
IReadOnlyDictionary<DateTime, TimeSpan> specific)
{
var result = common.ToDictionary();
foreach (var (key, value) in specific)
{
result[key] = value;
}

return result;
}
}
}
}
161 changes: 161 additions & 0 deletions Tests/Common/Util/MarketHoursDatabaseJsonConverterTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@

using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NodaTime;
using NUnit.Framework;
using QuantConnect.Securities;
Expand Down Expand Up @@ -59,6 +61,165 @@ public void HandlesRoundTrip()
}
}

public static void TestOverriddenEarlyClosesAndLateOpens(IReadOnlyDictionary<DateTime, TimeSpan> commonDateTimes,
IReadOnlyDictionary<DateTime, TimeSpan> specificDateTimes,
JObject jsonDatabase,
string testKey,
string specificEntryKey)
{
var datesWereOverriden = false;

foreach (var date in commonDateTimes.Keys)
{
// All early open or late close dates in the common entry should have been added to the specific entry.
// e.g. Index-usa-[*] early closes and late opens should have been added to Index-usa-VIX
Assert.IsTrue(specificDateTimes.ContainsKey(date));

// If common entry and specific entry have different times for the same date, the specific entry should
// have the correct time.
if (commonDateTimes[date] != specificDateTimes[date])
{
var dateStr = date.ToStringInvariant("MM/dd/yyyy");
var timeStr = jsonDatabase["entries"][specificEntryKey][testKey][dateStr].Value<string>();
var time = TimeSpan.Parse(timeStr, CultureInfo.InvariantCulture);
Assert.AreEqual(time, specificDateTimes[date]);

datesWereOverriden = true;
}
}

Assert.IsTrue(datesWereOverriden);
}

[Test]
public void HandlesOverridingLateOpens()
{
var jsonDatabase = JObject.Parse(SampleMHDBWithOverriddenEarlyOpens);
var database = jsonDatabase.ToObject<MarketHoursDatabase>();

var googEntry = database.GetEntry(Market.USA, "GOOG", SecurityType.Equity);
var equityCommonEntry = database.GetEntry(Market.USA, "", SecurityType.Equity);

TestOverriddenEarlyClosesAndLateOpens(equityCommonEntry.ExchangeHours.LateOpens,
googEntry.ExchangeHours.LateOpens,
jsonDatabase,
"lateOpens",
"Equity-usa-GOOG");
}

[Test]
public void HandlesOverridingEarlyCloses()
{
var jsonDatabase = JObject.Parse(SampleMHDBWithOverriddenEarlyOpens);
var database = jsonDatabase.ToObject<MarketHoursDatabase>();

var googEntry = database.GetEntry(Market.USA, "GOOG", SecurityType.Equity);
var equityCommonEntry = database.GetEntry(Market.USA, "", SecurityType.Equity);

TestOverriddenEarlyClosesAndLateOpens(equityCommonEntry.ExchangeHours.EarlyCloses,
googEntry.ExchangeHours.EarlyCloses,
jsonDatabase,
"earlyCloses",
"Equity-usa-GOOG");
}

/// <summary>
/// Equity-usa-GOOG is more specific than Equity-usa-[*].
/// The early closes for GOOG should override the early closes for the common entry ([*]).
/// </summary>
public static string SampleMHDBWithOverriddenEarlyOpens => @"
{
""entries"": {
""Equity-usa-[*]"": {
""dataTimeZone"": ""America/New_York"",
""exchangeTimeZone"": ""America/New_York"",
""sunday"": [],
""monday"": [
{ ""start"": ""04:00:00"", ""end"": ""09:30:00"", ""state"": ""premarket"" },
{ ""start"": ""09:30:00"", ""end"": ""16:00:00"", ""state"": ""market"" },
{ ""start"": ""16:00:00"", ""end"": ""20:00:00"", ""state"": ""postmarket"" }
],
""tuesday"": [
{ ""start"": ""04:00:00"", ""end"": ""09:30:00"", ""state"": ""premarket"" },
{ ""start"": ""09:30:00"", ""end"": ""16:00:00"", ""state"": ""market"" },
{ ""start"": ""16:00:00"", ""end"": ""20:00:00"", ""state"": ""postmarket"" }
],
""wednesday"": [
{ ""start"": ""04:00:00"", ""end"": ""09:30:00"", ""state"": ""premarket"" },
{ ""start"": ""09:30:00"", ""end"": ""16:00:00"", ""state"": ""market"" },
{ ""start"": ""16:00:00"", ""end"": ""20:00:00"", ""state"": ""postmarket"" }
],
""thursday"": [
{ ""start"": ""04:00:00"", ""end"": ""09:30:00"", ""state"": ""premarket"" },
{ ""start"": ""09:30:00"", ""end"": ""16:00:00"", ""state"": ""market"" },
{ ""start"": ""16:00:00"", ""end"": ""20:00:00"", ""state"": ""postmarket"" }
],
""friday"": [
{ ""start"": ""04:00:00"", ""end"": ""09:30:00"", ""state"": ""premarket"" },
{ ""start"": ""09:30:00"", ""end"": ""16:00:00"", ""state"": ""market"" },
{ ""start"": ""16:00:00"", ""end"": ""20:00:00"", ""state"": ""postmarket"" }
],
""saturday"": [],
""holidays"": [],
""earlyCloses"": {
""11/25/2015"": ""13:00:00"",
""11/25/2016"": ""13:00:00"",
""11/25/2017"": ""13:00:00"",
""11/25/2018"": ""13:00:00""
},
""lateOpens"": {
""11/25/2015"": ""11:00:00"",
""11/25/2016"": ""11:00:00"",
""11/25/2017"": ""11:00:00"",
""11/25/2018"": ""11:00:00""
}
},
""Equity-usa-GOOG"": {
""dataTimeZone"": ""America/New_York"",
""exchangeTimeZone"": ""America/New_York"",
""sunday"": [],
""monday"": [
{ ""start"": ""04:00:00"", ""end"": ""09:30:00"", ""state"": ""premarket"" },
{ ""start"": ""09:30:00"", ""end"": ""16:00:00"", ""state"": ""market"" },
{ ""start"": ""16:00:00"", ""end"": ""20:00:00"", ""state"": ""postmarket"" }
],
""tuesday"": [
{ ""start"": ""04:00:00"", ""end"": ""09:30:00"", ""state"": ""premarket"" },
{ ""start"": ""09:30:00"", ""end"": ""16:00:00"", ""state"": ""market"" },
{ ""start"": ""16:00:00"", ""end"": ""20:00:00"", ""state"": ""postmarket"" }
],
""wednesday"": [
{ ""start"": ""04:00:00"", ""end"": ""09:30:00"", ""state"": ""premarket"" },
{ ""start"": ""09:30:00"", ""end"": ""16:00:00"", ""state"": ""market"" },
{ ""start"": ""16:00:00"", ""end"": ""20:00:00"", ""state"": ""postmarket"" }
],
""thursday"": [
{ ""start"": ""04:00:00"", ""end"": ""09:30:00"", ""state"": ""premarket"" },
{ ""start"": ""09:30:00"", ""end"": ""16:00:00"", ""state"": ""market"" },
{ ""start"": ""16:00:00"", ""end"": ""20:00:00"", ""state"": ""postmarket"" }
],
""friday"": [
{ ""start"": ""04:00:00"", ""end"": ""09:30:00"", ""state"": ""premarket"" },
{ ""start"": ""09:30:00"", ""end"": ""16:00:00"", ""state"": ""market"" },
{ ""start"": ""16:00:00"", ""end"": ""20:00:00"", ""state"": ""postmarket"" }
],
""saturday"": [],
""holidays"": [],
""earlyCloses"": {
""11/25/2017"": ""13:30:00"",
""11/25/2018"": ""13:30:00"",
""11/25/2019"": ""13:30:00""
},
""lateOpens"": {
""11/25/2017"": ""11:30:00"",
""11/25/2018"": ""11:30:00"",
""11/25/2019"": ""11:30:00""
}
}
}
}
";

[Test, Ignore("This is provided to make it easier to convert your own market-hours-database.csv to the new format")]
public void ConvertMarketHoursDatabaseCsvToJson()
{
Expand Down

0 comments on commit 1c93df8

Please sign in to comment.