Skip to content

Commit

Permalink
Update athena_integration_account_glue_table.yaml
Browse files Browse the repository at this point in the history 
Updated SgaInitializerAsync to nodejs18.x code and runtime. Resolve issue #132
  • Loading branch information
@Baza-86
Baza-86 authored Jan 1, 2025
1 parent 556273e commit 05b6c41
Showing 1 changed file with 72 additions and 65 deletions.
137 changes: 72 additions & 65 deletions templates/athena_integration_account_glue_table.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,14 +145,14 @@ Resources:
Properties:
Code:
ZipFile: |
const response = require('./cfn-response');
const AWS = require('aws-sdk');
const glue = new AWS.Glue();
const { send } = require('./cfn-response');
const { GlueClient, GetTableCommand, GetPartitionCommand, CreatePartitionCommand } = require("@aws-sdk/client-glue");
const glue = new GlueClient({});
exports.handler = async function(event, context) {
exports.handler = async function (event, context) {
let errs = [], status
if (event.RequestType === 'Delete') {
status = response.SUCCESS
status = "SUCCESS"
} else {
console.log("Parsing athena configs")
let rp = event.ResourceProperties
Expand All @@ -163,8 +163,6 @@ Resources:
let service = rp.service
let region = rp.region
let errs = []
for(let i = 0; i < confs.length; i++) {
let cnf = confs[i]
let tab = cnf['partitionTableName']
Expand All @@ -173,76 +171,85 @@ Resources:
let end = (cnf['partitionEndDate'] == undefined) ? new Date() : new Date(cnf['partitionEndDate'])
while(strt <= end) {
let table = await glue.getTable({
DatabaseName: db,
Name: tab,
}).promise()
try {
let command = new GetTableCommand({
DatabaseName: db,
Name: tab,
});
let table = await glue.send(command);
let strgDesc = table.Table['StorageDescriptor']
let Values
let newDate = new Date()
let strgDesc = table.Table['StorageDescriptor']
let Values
let newDate = new Date()
if(frq == "monthly") {
if(hive == "true") {
Values = ["aws-account-id=" + account_id, "aws-service=" + service, "aws-region=" + region, "year=" + String(strt.getFullYear()), "month=" + ("0" + (strt.getMonth() + 1)).slice(-2)]
} else {
Values = [String(strt.getFullYear()), ("0" + (strt.getMonth() + 1)).slice(-2)]
}
newDate = strt.setMonth(strt.getMonth() + 1);
} else if(frq == "hourly") {
if(hive == "true") {
Values = ["aws-account-id=" + account_id, "aws-service=" + service, "aws-region=" + region, "year=" + String(strt.getFullYear()), "month=" + ("0" + (strt.getMonth() + 1)).slice(-2), "day=" + ("0" + strt.getDate()).slice(-2), "hour=" + ("0" + strt.getHours()).slice(-2)]
} else {
Values = [String(strt.getFullYear()), ("0" + (strt.getMonth() + 1)).slice(-2), ("0" + strt.getDate()).slice(-2), ("0" + strt.getHours()).slice(-2)]
}
newDate.setHours(strt.getHours() + 1);
} else {
if(hive == "true") {
Values = ["aws-account-id=" + account_id, "aws-service=" + service, "aws-region=" + region, "year=" + String(strt.getFullYear()), "month=" + ("0" + (strt.getMonth() + 1)).slice(-2), "day=" + ("0" + strt.getDate()).slice(-2)]
if(frq == "monthly") {
if(hive == "true") {
Values = ["aws-account-id=" + account_id, "aws-service=" + service, "aws-region=" + region, "year=" + String(strt.getFullYear()), "month=" + ("0" + (strt.getMonth() + 1)).slice(-2)]
} else {
Values = [String(strt.getFullYear()), ("0" + (strt.getMonth() + 1)).slice(-2)]
}
newDate = strt.setMonth(strt.getMonth() + 1);
} else if(frq == "hourly") {
if(hive == "true") {
Values = ["aws-account-id=" + account_id, "aws-service=" + service, "aws-region=" + region, "year=" + String(strt.getFullYear()), "month=" + ("0" + (strt.getMonth() + 1)).slice(-2), "day=" + ("0" + strt.getDate()).slice(-2), "hour=" + ("0" + strt.getHours()).slice(-2)]
} else {
Values = [String(strt.getFullYear()), ("0" + (strt.getMonth() + 1)).slice(-2), ("0" + strt.getDate()).slice(-2), ("0" + strt.getHours()).slice(-2)]
}
newDate.setHours(strt.getHours() + 1);
} else {
Values = [String(strt.getFullYear()), ("0" + (strt.getMonth() + 1)).slice(-2), ("0" + strt.getDate()).slice(-2)]
if(hive == "true") {
Values = ["aws-account-id=" + account_id, "aws-service=" + service, "aws-region=" + region, "year=" + String(strt.getFullYear()), "month=" + ("0" + (strt.getMonth() + 1)).slice(-2), "day=" + ("0" + strt.getDate()).slice(-2)]
} else {
Values = [String(strt.getFullYear()), ("0" + (strt.getMonth() + 1)).slice(-2), ("0" + strt.getDate()).slice(-2)]
}
newDate = strt.setDate(strt.getDate() + 1);
}
newDate = strt.setDate(strt.getDate() + 1);
}
try {
let result = await glue.getPartition({
DatabaseName: db,
TableName: tab,
PartitionValues: Values
}).promise()
} catch (err) {
if(err.code === 'EntityNotFoundException'){
console.log(strgDesc)
let params = {
DatabaseName: db,
TableName: tab,
PartitionInput: {
StorageDescriptor: {
...strgDesc,
Location: `${strgDesc.Location}${Values.join('/')}/`
},
Values,
},
}
await glue.createPartition(params).promise()
} else {
errs.push(err)
try {
let command = new GetPartitionCommand({
DatabaseName: db,
TableName: tab,
PartitionValues: Values
});
await glue.send(command);
} catch (err) {
if(err.name === 'EntityNotFoundException'){
console.log(strgDesc)
let params = {
DatabaseName: db,
TableName: tab,
PartitionInput: {
StorageDescriptor: {
...strgDesc,
Location: `${strgDesc.Location}${Values.join('/')}/`
},
Values,
},
}
try{
let command = new CreatePartitionCommand(params)
await glue.send(command)
} catch(err) {
errs.push(err)
}
} else {
errs.push(err)
}
}
strt = new Date(newDate);
} catch(err) {
errs.push(err)
}
strt = new Date(newDate);
}
}
status = errs.length > 0 ? response.FAILED : response.SUCCESS
}
return new Promise(() => response.send(event, context, status,
errs.length > 0 ? { error: errs } : {}, event.LogicalResourceId));
status = errs.length > 0 ? "FAILED" : "SUCCESS"
}
return new Promise(() => send(event, context, status, errs.length > 0 ? { error: errs } : {}, event.LogicalResourceId));
}
Handler: 'index.handler'
Timeout: 60
Runtime: nodejs14.x
Runtime: nodejs18.x
ReservedConcurrentExecutions: 1
Role: !GetAtt VpcFlowLogsTableIntegrationLambdaExecutorRole.Arn

Expand Down Expand Up @@ -366,4 +373,4 @@ Outputs:
FlowLogBucket:
Value: !Ref VpcFlowLogsTable
Export:
Name: !Join ["-", ["sga", !Ref MemberAccountId, vpc-flow-logs-table]]
Name: !Join ["-", ["sga", !Ref MemberAccountId, vpc-flow-logs-table]]

0 comments on commit 05b6c41

Please sign in to comment.