Skip to content

Commit

Permalink
Merge pull request #240 from tisseo-deploy/235-add-argument-to-Trunca…
Browse files Browse the repository at this point in the history
…teNewCalendarStatements

Add argument to TruncateNewCalendarStatements
  • Loading branch information
leonardehrenfried authored Dec 6, 2024
2 parents 09e6445 + 03b2b25 commit 3300b38
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 2 deletions.
41 changes: 41 additions & 0 deletions docs/onebusaway-gtfs-transformer-cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
* [Generate Stop Times](#generate-stop-times)
* [Extend Service Calendars](#extend-service-calendars)
* [Deduplicate Calendar Entries](#deduplicate-calendar-entries)
* [Truncate New Calendar Statements](#truncate-new-calendar-statements)
* [Merge Trips and Simplify Calendar Entries](#merge-trips-and-simplify-calendar-entries)
* [Shift Negative Stop Times](#shift-negative-stop-times)
* [Arbitrary Transform](#arbitrary-transform)
Expand Down Expand Up @@ -347,6 +348,46 @@ ids to a single service_id entry.
```
{"op":"deduplicate_service_ids"}
```

#### Truncate New Calendar Statements

This operation truncates calendar and calendar date entries based on the configuration attributes in the JSON transformer snippet:

* calendar_field: Specifies the unit of time for truncation. It can have one of the following values:
- `Calendar.YEAR` = 1
- `Calendar.MONTH` = 2 (default)
- `Calendar.DAY_OF_MONTH` = 5
- `Calendar.DAY_OF_YEAR` = 6

* calendar_amount: Specifies the number of units to truncate entries.
The value is an integer representing the amount (default = 1).

Both `calendar_field` and `calendar_amount` must be provided as integers in the JSON transformer.

If these parameters are not specified, the default behavior is truncation by 1 month.

Example :

Truncate calendar and calendar dates to the next 21 days:

```
{"op":"transform", "class":"org.onebusaway.gtfs_transformer.impl.TruncateNewCalendarStatements","calendar_field":6,"calendar_amount":21}
```

Truncate entries to the next 3 months:

```
{"op":"transform", "class":"org.onebusaway.gtfs_transformer.impl.TruncateNewCalendarStatements","calendar_field":2,"calendar_amount":3}
```

Additionally, after truncating the calendar entries, it is recommended to use a **retain operation** to ensure that only trips with valid calendar dates are retained.

Without this retain operation, the `trips.txt` file will contain trips with non-existent calendar dates, leading to invalid data.

```
{"op":"transform", "class":"org.onebusaway.gtfs_transformer.impl.TruncateNewCalendarStatements","calendar_field":6,"calendar_amount":21}
{"op":"retain", "match":{"file":"calendar_dates.txt"}, "retainBlocks":false}
```

#### Merge Trips and Simplify Calendar Entries

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package org.onebusaway.gtfs_transformer.impl;

import org.onebusaway.csv_entities.schema.annotations.CsvField;
import org.onebusaway.gtfs.model.ServiceCalendar;
import org.onebusaway.gtfs.model.ServiceCalendarDate;
import org.onebusaway.gtfs.services.GtfsMutableRelationalDao;
Expand All @@ -35,6 +36,31 @@
public class TruncateNewCalendarStatements implements GtfsTransformStrategy {

private final Logger _log = LoggerFactory.getLogger(TruncateNewCalendarStatements.class);

/*
* add two arguments used in the truncated transformation strategy
* calendarField --> calendar_field (json config file)
* Calendar.YEAR = 1
* Calendar.MONTH = 2
* Calendar.DAY_OF_MONTH = 5
* Calendar.DAY_OF_YEAR = 6
* calendarAmount --> calendar_amount (json config file)
*/
@CsvField(optional = true)
private int calendarField = Calendar.MONTH;

@CsvField(optional = true)
private int calendarAmount = 1;


public void setCalendarField(int calendarField) {
this.calendarField = calendarField;
}

public void setCalendarAmount(int calendarAmount) {
this.calendarAmount = calendarAmount;
}

@Override
public String getName() {
return this.getClass().getSimpleName();
Expand All @@ -43,9 +69,8 @@ public String getName() {
@Override
public void run(TransformContext transformContext, GtfsMutableRelationalDao gtfsMutableRelationalDao) {
RemoveEntityLibrary removeEntityLibrary = new RemoveEntityLibrary();
// TODO make this an argument -- default to one month from now
Calendar c = Calendar.getInstance();
c.roll(Calendar.MONTH, 1);
c.add(calendarField, calendarAmount);
java.util.Date oneMonthFromNow = c.getTime();
Set<ServiceCalendar> serviceCalendarsToRemove = new HashSet<ServiceCalendar>();
for (ServiceCalendar calendar: gtfsMutableRelationalDao.getAllCalendars()) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package org.onebusaway.gtfs_transformer.impl;

import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.onebusaway.gtfs.services.GtfsMutableRelationalDao;
import org.onebusaway.gtfs.services.MockGtfs;
import org.onebusaway.gtfs_transformer.services.TransformContext;

public class TruncateNewCalendarStatementsTest {

private TruncateNewCalendarStatements truncateNewCalendarStatements = new TruncateNewCalendarStatements();
private TransformContext _context = new TransformContext();
private MockGtfs _gtfs;

@BeforeEach
public void setup() throws IOException{

_gtfs = MockGtfs.create();
_gtfs.putAgencies(1);
_gtfs.putStops(1);
_gtfs.putRoutes(1);
_gtfs.putTrips(1, "r0", "sid0");
_gtfs.putStopTimes("t0", "s0");

// Set startDate to today's date and endDate to three weeks from today
String startDate = getCurrentDateFormatted(null);
String endDate = getCurrentDateFormatted(21);

// Define additional dates for testing purposes, relative to startDate
String threeDaysFromStartDate = getCurrentDateFormatted(5);
String sevenDaysFromStartDate = getCurrentDateFormatted(7);
String tenDaysFromStartDate = getCurrentDateFormatted(10);
String fifteenDaysFromStartDate = getCurrentDateFormatted(15);

// Insert a calendar entry with startDate set to today and endDate set to 3 weeks from today
_gtfs.putCalendars(
1,
"start_date="+startDate,
"end_date="+endDate
);

// Insert calendar dates entries
_gtfs.putCalendarDates("sid0="+startDate+","+threeDaysFromStartDate+","+
sevenDaysFromStartDate+","+tenDaysFromStartDate+","+fifteenDaysFromStartDate);
}

@Test
public void testTruncateCalendarStatementsWithinThreeDays() throws IOException {
GtfsMutableRelationalDao dao = _gtfs.read();
// Set calendarField 6 -> Calendar.DAY_OF_YEAR
truncateNewCalendarStatements.setCalendarField(6);
// Set calendarAmount : 3 -> 3 Days
truncateNewCalendarStatements.setCalendarAmount(3);
truncateNewCalendarStatements.run(_context, dao);

// Verify that GtfsMutableRelationalDao object contains exactly one calendar and one calendar date entry
assertEquals(1,dao.getAllCalendars().size());
assertEquals(1,dao.getAllCalendarDates().size());
}

@Test
public void testTruncateCalendarStatementsWithinSevenDays() throws IOException {
GtfsMutableRelationalDao dao = _gtfs.read();
// Set calendarField 6 -> Calendar.DAY_OF_YEAR
truncateNewCalendarStatements.setCalendarField(6);
// Set calendarAmount : 7 -> 7 Days
truncateNewCalendarStatements.setCalendarAmount(7);
truncateNewCalendarStatements.run(_context, dao);
// Verify that GtfsMutableRelationalDao object contains exactly one calendar and three calendar dates entries
assertEquals(1,dao.getAllCalendars().size());
assertEquals(3,dao.getAllCalendarDates().size());
}

// Helper function to get today's date in the required format
public static String getCurrentDateFormatted(Integer daysOffset) {
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd");
LocalDate date = LocalDate.now();
if (daysOffset != null){
date = date.plusDays(daysOffset);
}
// Format date as "yyyyMMdd"
return date.format(formatter);
}

}

0 comments on commit 3300b38

Please sign in to comment.