Skip to content

Commit

Permalink
Handle tradier intraday history limitations
Browse files Browse the repository at this point in the history 
- Handle tradier history intraday request limitations, expanding unit
  tests
- Implement tick resolution history requests in chunks, else it blows up
  in tradier api side
  • Loading branch information
@Martin-Molinero
Martin-Molinero committed Mar 12, 2024
1 parent d4b5566 commit 3b0a3f3
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 50 deletions.
1 change: 1 addition & 0 deletions QuantConnect.TradierBrokerage.Tests/TestSetup.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ public static void ReloadConfiguration()
private static void SetUp()
{
Log.LogHandler = new CompositeLogHandler();
Log.DebuggingEnabled = Config.GetBool("debug-mode");
Log.Trace("TestSetup(): starting...");
ReloadConfiguration();
}
Expand Down
63 changes: 45 additions & 18 deletions QuantConnect.TradierBrokerage.Tests/TradierBrokerageHistoryProviderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,21 +46,38 @@ private static TestCaseData[] TestParameters
return new[]
{
// valid parameters
new TestCaseData(Symbols.AAPL, Resolution.Tick, false, -1, TickType.Trade),
new TestCaseData(Symbols.AAPL, Resolution.Second, false, -1, TickType.Trade),
new TestCaseData(Symbols.AAPL, Resolution.Minute, false, 60 + 1, TickType.Trade),
new TestCaseData(Symbols.AAPL, Resolution.Hour, false, 7 * 2, TickType.Trade),
new TestCaseData(Symbols.AAPL, Resolution.Daily, false, 6, TickType.Trade),
new TestCaseData(Symbols.AAPL, Resolution.Tick, false, false, 60 * 6 * 2, TickType.Trade, -1),
new TestCaseData(Symbols.AAPL, Resolution.Tick, false, false, 30, TickType.Trade, -1),
new TestCaseData(Symbols.AAPL, Resolution.Tick, false, true, 60 * 6 * 2, TickType.Trade, -1),
new TestCaseData(Symbols.AAPL, Resolution.Tick, false, true, 30, TickType.Trade, -1),

new TestCaseData(Symbols.AAPL, Resolution.Second, false, false, 60 * 6 * 10, TickType.Trade, 60 * 6 * 5),
new TestCaseData(Symbols.AAPL, Resolution.Second, false, false, 30, TickType.Trade, -1),
new TestCaseData(Symbols.AAPL, Resolution.Second, false, true, 60 * 6 * 2, TickType.Trade, -1),
new TestCaseData(Symbols.AAPL, Resolution.Second, false, true, 30, TickType.Trade, -1),

new TestCaseData(Symbols.AAPL, Resolution.Minute, false, false, 60 * 6 * 50, TickType.Trade, 60 * 6 * 15),
new TestCaseData(Symbols.AAPL, Resolution.Minute, false, false, 60 + 1, TickType.Trade, -1),
new TestCaseData(Symbols.AAPL, Resolution.Minute, false, true, 60 * 6 * 15, TickType.Trade, -1),
new TestCaseData(Symbols.AAPL, Resolution.Minute, false, true, 60 + 1, TickType.Trade, -1),

new TestCaseData(Symbols.AAPL, Resolution.Hour, false, false, 6 * 80, TickType.Trade, 6 * 30),
new TestCaseData(Symbols.AAPL, Resolution.Hour, false, false, 5, TickType.Trade, -1),
new TestCaseData(Symbols.AAPL, Resolution.Hour, false, true, 6 * 80, TickType.Trade, -1),
new TestCaseData(Symbols.AAPL, Resolution.Hour, false, true, 5, TickType.Trade, -1),

new TestCaseData(Symbols.AAPL, Resolution.Daily, false, false, 60, TickType.Trade, -1),
new TestCaseData(Symbols.AAPL, Resolution.Daily, false, false, 6, TickType.Trade, -1),

// invalid tick type, null result
new TestCaseData(Symbols.AAPL, Resolution.Minute, true, 0, TickType.Quote),
new TestCaseData(Symbols.AAPL, Resolution.Minute, true, 0, TickType.OpenInterest),
new TestCaseData(Symbols.AAPL, Resolution.Minute, true, false, 0, TickType.Quote, -1),
new TestCaseData(Symbols.AAPL, Resolution.Minute, true, false, 0, TickType.OpenInterest, -1),

// canonical symbol, null result
new TestCaseData(Symbols.SPY_Option_Chain, Resolution.Daily, true, 0, TickType.Trade),
new TestCaseData(Symbols.SPY_Option_Chain, Resolution.Daily, true, false, 0, TickType.Trade, -1),

// invalid security type, null result
new TestCaseData(Symbols.EURUSD, Resolution.Daily, true, 0, TickType.Trade)
new TestCaseData(Symbols.EURUSD, Resolution.Daily, true, false, 0, TickType.Trade, -1)
};
}
}
Expand All @@ -85,7 +102,7 @@ public void TearDown()
}

[Test, TestCaseSource(nameof(TestParameters))]
public void GetsHistory(Symbol symbol, Resolution resolution, bool unsupported, int expectedCount, TickType tickType)
public void GetsHistory(Symbol symbol, Resolution resolution, bool unsupported, bool extendedMarketHours, int expectedCount, TickType tickType, int adjustedExpectedCount = -1)
{
if (_useSandbox && (resolution == Resolution.Tick || resolution == Resolution.Second))
{
Expand All @@ -94,10 +111,10 @@ public void GetsHistory(Symbol symbol, Resolution resolution, bool unsupported,
}
var mhdb = MarketHoursDatabase.FromDataFolder().GetEntry(symbol.ID.Market, symbol, symbol.SecurityType);

GetStartEndTime(mhdb, resolution, expectedCount, out var startUtc, out var endUtc);
GetStartEndTime(mhdb, resolution, expectedCount, extendedMarketHours, out var startUtc, out var endUtc);

var request = new HistoryRequest(startUtc, endUtc, LeanData.GetDataType(resolution, tickType), symbol, resolution, mhdb.ExchangeHours,
mhdb.DataTimeZone, null, false, false, DataNormalizationMode.Adjusted, tickType);
mhdb.DataTimeZone, null, includeExtendedMarketHours: extendedMarketHours, false, DataNormalizationMode.Adjusted, tickType);

if (unsupported)
{
Expand All @@ -114,7 +131,17 @@ public void GetsHistory(Symbol symbol, Resolution resolution, bool unsupported,
}
else
{
Assert.AreEqual(expectedCount, count, $"Symbol: {request.Symbol.Value}. Resolution {request.Resolution}");
if (adjustedExpectedCount != -1)
{
// the request was over the tradier api limit so it get's reduced
expectedCount = adjustedExpectedCount;
}

// add some padding for extended market hours, cause it ain't precise
var delta = (request.IncludeExtendedMarketHours || adjustedExpectedCount != -1)
? expectedCount * 0.15
: 0;
Assert.AreEqual(expectedCount, count, delta, $"Symbol: {request.Symbol.Value}. Resolution {request.Resolution}");
}
}
}
Expand All @@ -123,7 +150,7 @@ public void GetsHistory(Symbol symbol, Resolution resolution, bool unsupported,
[TestCase(Resolution.Hour, 30)]
[TestCase(Resolution.Minute, 60 * 10)]
[TestCase(Resolution.Second, 60 * 10 * 5)]
[TestCase(Resolution.Tick, -1)]
[TestCase(Resolution.Tick, 30)]
public void GetsOptionHistory(Resolution resolution, int expectedCount)
{
if (_useSandbox && (resolution == Resolution.Tick || resolution == Resolution.Second))
Expand All @@ -134,7 +161,7 @@ public void GetsOptionHistory(Resolution resolution, int expectedCount)
var spy = Symbol.Create("SPY", SecurityType.Equity, Market.USA);
var mhdb = MarketHoursDatabase.FromDataFolder().GetEntry(spy.ID.Market, spy, spy.SecurityType);

GetStartEndTime(mhdb, resolution, expectedCount, out var startUtc, out var endUtc);
GetStartEndTime(mhdb, resolution, expectedCount, false, out var startUtc, out var endUtc);

var chain = _chainProvider.GetOptionContractList(spy, startUtc.ConvertFromUtc(mhdb.ExchangeHours.TimeZone)).ToList();

Expand Down Expand Up @@ -170,12 +197,12 @@ public void GetsOptionHistory(Resolution resolution, int expectedCount)
Assert.Greater(count, 15, $"Symbol: {request.Symbol.Value}. Resolution {request.Resolution}");
}

private void GetStartEndTime(MarketHoursDatabase.Entry entry, Resolution resolution, int expectedCount, out DateTime startTimeUtc, out DateTime endTimeUtc)
private void GetStartEndTime(MarketHoursDatabase.Entry entry, Resolution resolution, int expectedCount,
bool extendedMarketHours, out DateTime startTimeUtc, out DateTime endTimeUtc)
{
if (resolution == Resolution.Tick || resolution == Resolution.Second)
{
// for tick ask for X minutes worth of data
expectedCount = 30;
resolution = Resolution.Minute;
}

Expand All @@ -184,7 +211,7 @@ private void GetStartEndTime(MarketHoursDatabase.Entry entry, Resolution resolut
var endLocalTime = endTimeUtc.ConvertFromUtc(entry.ExchangeHours.TimeZone);
var resolutionSpan = resolution.ToTimeSpan();

var localStartTime = Time.GetStartTimeForTradeBars(entry.ExchangeHours, endLocalTime, resolutionSpan, expectedCount, false, entry.DataTimeZone);
var localStartTime = Time.GetStartTimeForTradeBars(entry.ExchangeHours, endLocalTime, resolutionSpan, expectedCount, extendedMarketHours, entry.DataTimeZone);
startTimeUtc = localStartTime.ConvertToUtc(entry.ExchangeHours.TimeZone);
}

Expand Down
26 changes: 6 additions & 20 deletions QuantConnect.TradierBrokerage/TradierBrokerage.HistoryProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public partial class TradierBrokerage
private bool _loggedTradierSupportsOnlyTradeBars;
private bool _loggedUnsupportedAssetForHistory;
private bool _loggedInvalidTimeRangeForHistory;
private bool _loggedInvalidStartTimeForHistory;

/// <summary>
/// Gets the history for the requested security
Expand Down Expand Up @@ -75,8 +76,8 @@ public override IEnumerable<BaseData> GetHistory(HistoryRequest request)
return null;
}

var start = request.StartTimeUtc.ConvertTo(DateTimeZone.Utc, TimeZones.NewYork);
var end = request.EndTimeUtc.ConvertTo(DateTimeZone.Utc, TimeZones.NewYork);
var start = request.StartTimeUtc.ConvertFromUtc(TimeZones.NewYork);
var end = request.EndTimeUtc.ConvertFromUtc(TimeZones.NewYork);

IEnumerable<BaseData> history;
switch (request.Resolution)
Expand Down Expand Up @@ -114,12 +115,7 @@ private IEnumerable<BaseData> GetHistoryTick(HistoryRequest request, DateTime st
{
var symbol = request.Symbol;
var exchangeTz = request.ExchangeHours.TimeZone;
var history = GetTimeSeries(symbol, start, end, TradierTimeSeriesIntervals.Tick);

if (history == null)
{
return Enumerable.Empty<BaseData>();
}
var history = GetTimeSeries(request, start, end, TradierTimeSeriesIntervals.Tick);

return history.Select(tick => new Tick
{
Expand Down Expand Up @@ -159,12 +155,7 @@ private IEnumerable<BaseData> GetHistoryMinute(HistoryRequest request, DateTime
var symbol = request.Symbol;
var exchangeTz = request.ExchangeHours.TimeZone;
var requestedBarSpan = request.Resolution.ToTimeSpan();
var history = GetTimeSeries(symbol, start, end, TradierTimeSeriesIntervals.OneMinute);

if (history == null)
{
return Enumerable.Empty<BaseData>();
}
var history = GetTimeSeries(request, start, end, TradierTimeSeriesIntervals.OneMinute);

return history.Select(bar => new TradeBar(bar.Time, symbol, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume, requestedBarSpan));
}
Expand All @@ -174,12 +165,7 @@ private IEnumerable<BaseData> GetHistoryHour(HistoryRequest request, DateTime st
var symbol = request.Symbol;
var exchangeTz = request.ExchangeHours.TimeZone;
var requestedBarSpan = request.Resolution.ToTimeSpan();
var history = GetTimeSeries(symbol, start, end, TradierTimeSeriesIntervals.FifteenMinutes);

if (history == null)
{
return Enumerable.Empty<BaseData>();
}
var history = GetTimeSeries(request, start, end, TradierTimeSeriesIntervals.FifteenMinutes);

var tradierBarSpan = TimeSpan.FromMinutes(15);
var result = history.Select(bar => new TradeBar(bar.Time, symbol, bar.Open, bar.High, bar.Low, bar.Close, bar.Volume, tradierBarSpan));
Expand Down
89 changes: 77 additions & 12 deletions QuantConnect.TradierBrokerage/TradierBrokerage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using QuantConnect.Api;
using QuantConnect.Configuration;
using QuantConnect.Data;
using QuantConnect.Interfaces;
using QuantConnect.Logging;
Expand Down Expand Up @@ -498,19 +497,85 @@ public List<TradierQuote> GetQuotes(List<string> symbols)
/// <summary>
/// Get the historical bars for this period
/// </summary>
private List<TradierTimeSeries> GetTimeSeries(Symbol symbol, DateTime start, DateTime end, TradierTimeSeriesIntervals interval)
private IEnumerable<TradierTimeSeries> GetTimeSeries(HistoryRequest historyRequest, DateTime start, DateTime end, TradierTimeSeriesIntervals interval)
{
// Create and send request
var ticker = _symbolMapper.GetBrokerageSymbol(symbol);
var request = new RestRequest("markets/timesales", Method.GET);
request.AddParameter("symbol", ticker, ParameterType.QueryString);
request.AddParameter("interval", GetEnumDescription(interval), ParameterType.QueryString);
request.AddParameter("start", start.ToStringInvariant("yyyy-MM-dd HH:mm"), ParameterType.QueryString);
request.AddParameter("end", end.ToStringInvariant("yyyy-MM-dd HH:mm"), ParameterType.QueryString);
var dataContainer = Execute<TradierTimeSeriesContainer>(request, TradierApiRequestType.Data, "series");
// Create and send request, take into account tradier limitations, else we get an error like:
// 'Invalid parameter, start: must be on or after 2024-01-15 00:00:00.'
// ref https://documentation.tradier.com/brokerage-api/markets/get-timesales
/*
Interval Data Available (Open) Data Available (All)
tick 5 days N/A
1min 20 days 10 days
5min 40 days 18 days
15min 40 days 18 days
*/
TimeSpan maximumTimeAgo;
if (interval == TradierTimeSeriesIntervals.FifteenMinutes || interval == TradierTimeSeriesIntervals.FiveMinutes)
{
maximumTimeAgo = TimeSpan.FromDays(40);
}
else if (interval == TradierTimeSeriesIntervals.OneMinute)
{
maximumTimeAgo = TimeSpan.FromDays(20);
}
else if (interval == TradierTimeSeriesIntervals.Tick)
{
maximumTimeAgo = TimeSpan.FromDays(5);
}
else
{
throw new ArgumentException($"Invalid TradierTimeSeriesIntervals value: {interval}");
}

// there could be no data the requested symbol and time, tradier will return null
return dataContainer?.TimeSeries ?? new List<TradierTimeSeries>();
var nyCurrentTime = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
if (nyCurrentTime - start > maximumTimeAgo)
{
if (!_loggedInvalidStartTimeForHistory)
{
_loggedInvalidStartTimeForHistory = true;
OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidStartTime", "Warning: Adjusting history request start time to fit Tradier limitations"));
}

start = nyCurrentTime.Add(-maximumTimeAgo);
if (start > end)
{
yield break;
}
}

var requestEnd = end;
var requestStart = start;

// we ask tick data in chunks, if not tradier API blows up
if (interval == TradierTimeSeriesIntervals.Tick && requestEnd > (requestStart + Time.OneHour))
{
requestEnd = requestStart + Time.OneHour;
}

var ticker = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol);
do
{
if (historyRequest.ExchangeHours.IsOpen(requestStart, requestEnd, historyRequest.IncludeExtendedMarketHours))
{
var request = new RestRequest("markets/timesales", Method.GET);
request.AddParameter("symbol", ticker, ParameterType.QueryString);
request.AddParameter("interval", GetEnumDescription(interval), ParameterType.QueryString);
request.AddParameter("start", requestStart.ToStringInvariant("yyyy-MM-dd HH:mm"), ParameterType.QueryString);
request.AddParameter("end", requestEnd.ToStringInvariant("yyyy-MM-dd HH:mm"), ParameterType.QueryString);
request.AddParameter("session_filter", historyRequest.IncludeExtendedMarketHours ? "all" : "open", ParameterType.QueryString);
var dataContainer = Execute<TradierTimeSeriesContainer>(request, TradierApiRequestType.Data, "series");

// there could be no data the requested symbol and time, tradier will return null
foreach (var point in dataContainer?.TimeSeries ?? Enumerable.Empty<TradierTimeSeries>())
{
yield return point;
}
}

requestStart += Time.OneHour;
requestEnd += Time.OneHour;
}
while (requestEnd < end);
}

/// <summary>
Expand Down

0 comments on commit 3b0a3f3

Please sign in to comment.