Skip to content

Commit

Permalink
Even better date handling
Browse files Browse the repository at this point in the history
Stumbled into some issues with the date handling in the code base. This
make it more usable.
  • Loading branch information
reyjrar committed Aug 9, 2023
1 parent 4c9fd82 commit d0410e9
Showing 1 changed file with 33 additions and 35 deletions.
68 changes: 33 additions & 35 deletions lib/App/ElasticSearch/Utilities.pm
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,6 @@ From App::ElasticSearch::Utilities:
--index Index to run commands against
--base For daily indexes, reference only those starting with "logstash"
(same as --pattern logstash-* or logstash-DATE)
--datesep Date separator, default '.' also (--date-separator)
--pattern Use a pattern to operate on the indexes
--days If using a pattern or base, how many days back to go, default: 1
Expand Down Expand Up @@ -252,6 +251,16 @@ my %DEF = ();
my %PATTERN_REGEX = (
'*' => qr/.*/,
ANY => qr/.*/,
DATE => qr/
(?<datestr>
(?<year>\d{4}) # Extract 4 digits for the year
(?:(?<datesep>[\-._]))? # Optionally, look for . - _ as a separator
(?<month>\d{2}) # Two digits for the month
(?:\g{datesep})? # Whatever the date separator was in the previous match
(?<day>\d{2}) # Two digits for the day
(?![a-zA-Z0-9]) # Zero width negative look ahead, not alphanumeric
)
/x,
);
my $PATTERN;

Expand Down Expand Up @@ -390,10 +399,6 @@ sub es_utils_initialize {
PATTERN => exists $opts->{pattern} ? $opts->{pattern} : '*',
DAYS => exists $opts->{days} ? $opts->{days}
: exists $_GLOBALS{days} ? $_GLOBALS{days} : 1,
DATESEP => exists $opts->{datesep} ? $opts->{datesep}
: exists $_GLOBALS{datesep} ? $_GLOBALS{datesep}
: exists $_GLOBALS{"date-separator"} ? $_GLOBALS{"date-separator"}
: '.',
# HTTP Basic Authentication
USERNAME => exists $opts->{'http-username'} ? $opts->{'http-username'}
: exists $_GLOBALS{'http-username'} ? $_GLOBALS{'http-username'}
Expand Down Expand Up @@ -425,20 +430,11 @@ sub es_utils_initialize {
delete $ENV{$_} for qw(http_proxy HTTP_PROXY);
}

# Setup Variables based on the config
%PATTERN_REGEX = (
'*' => qr/.*/,
DATE => qr/\d{4}(?:\Q$DEF{DATESEP}\E)?\d{2}(?:\Q$DEF{DATESEP}\E)?\d{2}/,
ANY => qr/.*/,
);
my @ordered = qw(* DATE ANY);

if( index($DEF{DATESEP},'-') >= 0 ) {
output({stderr=>1,color=>'yellow'}, "=== Using a '-' as your date separator may cause problems with other utilities. ===");
}

# Build the Index Pattern
$PATTERN = $DEF{PATTERN};

my @ordered = qw(* DATE ANY);
foreach my $literal ( @ordered ) {
$PATTERN =~ s/\Q$literal\E/$PATTERN_REGEX{$literal}/g;
}
Expand Down Expand Up @@ -1043,7 +1039,6 @@ sub es_indices {
}
}
elsif( $args{check_dates} && defined $DEF{DAYS} ) {

my $days_old = es_index_days_old( $index );
if( !defined $days_old ) {
debug({indent=>2,color=>'red'}, "! error locating date in string, skipping !");
Expand Down Expand Up @@ -1081,12 +1076,8 @@ sub es_index_strip_date {
es_utils_initialize() unless keys %DEF;

# Try the Date Pattern
if( $index =~ s/[-_]$PATTERN_REGEX{DATE}.*//o ) {
return $index;
}
# Fallback to matching thing-YYYY-MM-DD or thing-YYYY.MM.DD
elsif( $index =~ s/[-_]\d{4}([.-])\d{2}\g{1}\d{2}(?:[-_.]\d+)?$// ) {
return $index;
if( my $base = $index =~ s/[^a-z0-9]$PATTERN_REGEX{DATE}.*$//rio ) {
return $base;
}
return;
}
Expand Down Expand Up @@ -1132,32 +1123,39 @@ Return the number of days old this index is.
=cut

my $NOW = timegm(0,0,0,(gmtime)[3,4,5]);
sub es_index_days_old {
my ($index) = @_;

return unless defined $index;

es_utils_initialize() unless keys %DEF;

if( my ($dateStr) = ($index =~ /($PATTERN_REGEX{DATE})/) ) {
my @date=();
if(length $DEF{DATESEP}) {
@date = reverse map { int } split /\Q$DEF{DATESEP}\E/, $dateStr;
}
else {
for my $len (qw(4 2 2)) {
unshift @date, substr($dateStr,0,$len,'');
}
}

if( $index =~ /[^a-z0-9]$PATTERN_REGEX{DATE}/io ) {
# Build Date Array
my @date = map { int }
grep { length }
map { $+{$_} =~ s/^0//r } qw(day month year);
$date[1]--; # move 1-12 -> 0-11
# Validate
if( @date != 3 ) {
warn sprintf "es_index_days_old(%s) matched DATE(%s), but did not receive enough parts: %s",
$index,
$+{datestr},
join(', ', map { "'$_'" } @date);
return;
}

# Calculate Difference
my $now = timegm(0,0,0,(gmtime)[3,4,5]);
my $idx_time = eval { timegm( 0,0,0, @date ) };
return unless $idx_time;
my $diff = $NOW - $idx_time;
my $diff = $now - $idx_time;
$diff++; # Add one second
debug({color=>"yellow"}, sprintf "es_index_days_old(%s) - Time difference is %0.3f", $index, $diff/86400);
return int($diff / 86400);
}
verbose({color=>"red"}, "es_index_days_old($index) - date string not found");
return;
}

Expand Down

0 comments on commit d0410e9

Please sign in to comment.