Skip to content

Commit

Permalink
* Removed the 'history.states' table from ScanCore along with the ass…
Browse files Browse the repository at this point in the history 
…ociated SQL procedure and trigger. This table was growing very rapidly and served no purpose important enough to justify the complexity it was inducing.

* Updated DB->do_db_write() to check if an array of queries was >25k and, if so, split the array into 25k chunks to commit independently, thus reducing memory footprint.
* Moved ScanCore's 'check_ram_usage()' function to 'ScanCore.pm' and made it a method. Then called it between each scan agent invocation as well as after committing each 25k chunk in DB->do_db_write().
* Disabled perl's deep recursion warnings.
* Moved several 'update_db_X' functions to use log level 3.
* Updated scan-ipmitool to uses a dedicated function call to convert a given ipmitool_value_ipmitool_uuid into the ipmitool_value_id.
* Boosted ScanCore's default allowed RAM usage to 1 GiB.
* Bumped the release version to tc5.

Signed-off-by: Digimer <[email protected]>
  • Loading branch information
Digimer committed Oct 19, 2016
1 parent 3b15f27 commit 8a6184b
Show file tree
Hide file tree
Showing 13 changed files with 416 additions and 550 deletions.
94 changes: 73 additions & 21 deletions AN/Tools/DB.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use strict;
use warnings;
use DBI;
use Data::Dumper;
no warnings 'recursion';

our $VERSION = "0.1.001";
my $THIS_FILE = "DB.pm";
Expand Down Expand Up @@ -650,15 +651,17 @@ sub do_db_write
my $an = $self->parent;

# Setup my variables.
my $id = $parameter->{id} ? $parameter->{id} : "";
my $source = $parameter->{source} ? $parameter->{source} : "";
my $line = $parameter->{line} ? $parameter->{line} : "";
my $query = $parameter->{query} ? $parameter->{query} : "";
$an->Log->entry({log_level => 3, message_key => "an_variables_0004", message_variables => {
name1 => "id", value1 => $id,
name2 => "source", value2 => $source,
name3 => "line", value3 => $line,
name4 => "query", value4 => $query,
my $id = $parameter->{id} ? $parameter->{id} : "";
my $source = $parameter->{source} ? $parameter->{source} : "";
my $line = $parameter->{line} ? $parameter->{line} : "";
my $query = $parameter->{query} ? $parameter->{query} : "";
my $reenter = $parameter->{reenter} ? $parameter->{reenter} : "";
$an->Log->entry({log_level => 3, message_key => "an_variables_0005", message_variables => {
name1 => "id", value1 => $id,
name2 => "source", value2 => $source,
name3 => "line", value3 => $line,
name4 => "query", value4 => $query,
name5 => "reenter", value5 => $reenter,
}, file => $THIS_FILE, line => __LINE__});

# If I don't have a query, die.
Expand Down Expand Up @@ -690,35 +693,81 @@ sub do_db_write
}

# Sort out if I have one or many queries.
my $is_array = 0;
my @query;
my $limit = 25000;
my $count = 0;
my $query_set = [];
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "query", value1 => $query
}, file => $THIS_FILE, line => __LINE__});
if (ref($query) eq "ARRAY")
{
# Multiple things to enter.
$is_array = 1;
foreach my $this_query (@{$query})
$count = @{$query};

# If I am re-entering, then we'll proceed normally. If not, and if we have more than 10k
# queries, we'll split up the queries into 10k chunks and re-enter.
$an->Log->entry({log_level => 2, message_key => "an_variables_0003", message_variables => {
name1 => "count", value1 => $count,
name2 => "limit", value2 => $limit,
name3 => "reenter", value3 => $reenter,
}, file => $THIS_FILE, line => __LINE__});
if (($count > $limit) && (not $reenter))
{
my $i = 0;
my $next = $limit;
$an->Log->entry({log_level => 2, message_key => "an_variables_0002", message_variables => {
name1 => "i", value1 => $i,
name2 => "next", value2 => $next,
}, file => $THIS_FILE, line => __LINE__});
foreach my $this_query (@{$query})
{
push @{$query_set}, $this_query;
$i++;

if ($i > $next)
{
# Commit this batch.
foreach my $id (@db_ids)
{
# Commit this chunk to this DB.
$an->DB->do_db_write({id => $id, query => $query_set, source => $THIS_FILE, line => $line, reenter => 1});

# This can get memory intensive, so check our RAM usage and
# bail if we're eating too much.
$an->ScanCore->check_ram_usage();

# Wipe out the old set array, create it as a new anonymous array and reset 'i'.
undef $query_set;
$query_set = [];
$i = 0;
}
}
}
}
else
{
push @query, $this_query;
# Not enough to worry about or we're dealing with a chunk, proceed as normal.
foreach my $this_query (@{$query})
{
push @{$query_set}, $this_query;
}
}
}
else
{
push @query, $query;
push @{$query_set}, $query;
}
foreach my $id (@db_ids)
{
# Do the actual query(ies)
if ($is_array)
if ($count)
{
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "is_array", value1 => $is_array
$an->Log->entry({log_level => 2, message_key => "an_variables_0001", message_variables => {
name1 => "count", value1 => $count,
}, file => $THIS_FILE, line => __LINE__});
$an->data->{dbh}{$id}->begin_work;
}
foreach my $query (@query)
foreach my $query (@{$query_set})
{
# Record the query
if ($an->Log->db_transactions())
Expand Down Expand Up @@ -750,12 +799,15 @@ sub do_db_write
}

# Commit the changes.
if ($is_array)
if ($count)
{
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "is_array", value1 => $is_array
name1 => "count", value1 => $count
}, file => $THIS_FILE, line => __LINE__});
$an->data->{dbh}{$id}->commit();

# Free up some memory.
undef $query_set;
}
}

Expand Down
98 changes: 98 additions & 0 deletions AN/Tools/ScanCore.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@ use strict;
use warnings;
use Data::Dumper;
use Text::Diff;
no warnings 'recursion';

our $VERSION = "0.1.001";
my $THIS_FILE = "ScanCore.pm";

### Methods;
# check_ram_usage
# get_anvils
# get_dr_jobs
# get_dr_targets
Expand Down Expand Up @@ -81,6 +83,102 @@ sub parent
# Provided methods #
#############################################################################################################

# This checks the amount RAM used by ScanCore and exits if it exceeds a scancore::maximum_ram bytes. It looks
# for any process with our name and sums the RAM used.
sub check_ram_usage
{
my $self = shift;
my $parameter = shift;
my $an = $self->parent;
$an->Log->entry({log_level => 3, title_key => "tools_log_0001", title_variables => { function => "check_ram_usage" }, message_key => "tools_log_0002", file => $THIS_FILE, line => __LINE__});

# Read in how much RAM we're using.
my $used_ram = $an->Get->ram_used_by_program({program_name => $an->data->{sys}{program_name}});
$an->Log->entry({log_level => 2, message_key => "an_variables_0002", message_variables => {
name1 => "used_ram", value1 => "$used_ram",
name2 => "scancore::maximum_ram", value2 => $an->data->{scancore}{maximum_ram}
}, file => $THIS_FILE, line => __LINE__});

# Exit if I failed to read the amount of RAM in use.
if (not $used_ram)
{
$an->Alert->warning({message_key => "scancore_warning_0023", message_variables => { program_name => $an->data->{sys}{program_name} }, quiet => 1, file => $THIS_FILE, line => __LINE__});
$an->data->{sys}{'exit'} = 1;
}

# Records the RAM used.
my $query = "
SELECT
ram_used_bytes
FROM
ram_used
WHERE
ram_used_by = ".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{program_name})."
AND
ram_used_host_uuid = ".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{host_uuid})."
;";
$an->Log->entry({log_level => 2, message_key => "an_variables_0001", message_variables => {
name1 => "query", value1 => $query
}, file => $THIS_FILE, line => __LINE__});
my $ram_used_bytes = $an->DB->do_db_query({query => $query, source => $THIS_FILE, line => __LINE__})->[0]->[0]; # (->[row]->[column])
$an->Log->entry({log_level => 3, message_key => "an_variables_0001", message_variables => {
name1 => "ram_used_bytes", value1 => $ram_used_bytes
}, file => $THIS_FILE, line => __LINE__});
if (not $ram_used_bytes)
{
# Add this agent to the DB
my $query = "
INSERT INTO
ram_used
(
ram_used_host_uuid,
ram_used_by,
ram_used_bytes,
modified_date
) VALUES (
".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{host_uuid}).",
".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{program_name}).",
".$an->data->{sys}{use_db_fh}->quote($used_ram).",
".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{db_timestamp})."
);
";
$an->DB->do_db_write({query => $query, source => $THIS_FILE, line => __LINE__});
}
elsif ($ram_used_bytes ne $used_ram)
{
# It exists and the value has changed.
my $query = "
UPDATE
ram_used
SET
ram_used_bytes = ".$an->data->{sys}{use_db_fh}->quote($used_ram).",
modified_date = ".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{db_timestamp})."
WHERE
ram_used_by = ".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{program_name})."
AND
ram_used_host_uuid = ".$an->data->{sys}{use_db_fh}->quote($an->data->{sys}{host_uuid})."
;";
$an->DB->do_db_write({query => $query, source => $THIS_FILE, line => __LINE__});
}
else
{
# The amount of RAM used is unchanged.
#print __LINE__."; The amount of RAM used by ".$an->data->{sys}{program_name}." is unchanged.\n";
}

if ($used_ram > $an->data->{scancore}{maximum_ram})
{
# Much, too much, much music!
# err, too much RAM...
$an->Alert->error({title_key => "an_0003", message_key => "scancore_error_0013", message_variables => {
used_ram => $an->Readable->bytes_to_hr({'bytes' => $used_ram}),
maximum_ram => $an->Readable->bytes_to_hr({'bytes' => $an->data->{scancore}{maximum_ram}})
}, code => 5, file => $THIS_FILE, line => __LINE__});
}

return(0);
}

# Get a list of Anvil! systems as an array of hash references
sub get_anvils
{
Expand Down
Loading

0 comments on commit 8a6184b

Please sign in to comment.