Skip to content

Commit

Permalink
Add support for multiline json
Browse files Browse the repository at this point in the history
  • Loading branch information
@cardigliano
cardigliano committed Jul 12, 2023
1 parent b1d2519 commit 669a02c
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 35 deletions.
4 changes: 3 additions & 1 deletion src/include/ndpi_typedefs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1756,7 +1756,8 @@ typedef enum {
ndpi_serialization_format_unknown = 0,
ndpi_serialization_format_tlv,
ndpi_serialization_format_json,
ndpi_serialization_format_csv
ndpi_serialization_format_csv,
ndpi_serialization_format_multiline_json
} ndpi_serialization_format;

/* Note:
Expand Down Expand Up @@ -1821,6 +1822,7 @@ typedef struct {
ndpi_serialization_format fmt;
char csv_separator[2];
u_int8_t has_snapshot;
u_int8_t multiline_json_array;
ndpi_private_serializer_status snapshot;
} ndpi_private_serializer;

Expand Down
109 changes: 75 additions & 34 deletions src/lib/ndpi_serializer.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,8 +177,12 @@ void ndpi_reset_serializer(ndpi_serializer *_serializer) {
serializer->status.buffer.size_used = 0;
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;

/* Note: please keep a space at the beginning as it is used for arrays when an end-of-record is used */
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, " {}");
if (serializer->multiline_json_array) {
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "{}");
} else {
/* Note: please keep a space at the beginning as it is used for arrays when an end-of-record is used */
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, " {}");
}
} else if(serializer->fmt == ndpi_serialization_format_csv) {
serializer->status.header.size_used = 0;
serializer->status.buffer.size_used = 0;
Expand Down Expand Up @@ -215,6 +219,11 @@ int ndpi_init_serializer_ll(ndpi_serializer *_serializer,

memset(serializer, 0, sizeof(ndpi_private_serializer));

if (fmt == ndpi_serialization_format_multiline_json) {
fmt = ndpi_serialization_format_json;
serializer->multiline_json_array = 1;
}

serializer->fmt = fmt;

if (buffer_size < 3 ||
Expand Down Expand Up @@ -614,10 +623,18 @@ int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
if(serializer->fmt == ndpi_serialization_format_json) {
needed += 1;

if(serializer->status.buffer.size_used == 3) /* Empty buffer [{} */
serializer->status.buffer.size_used = 2; /* Remove {} */
else
needed += 2, add_comma = 1;
if (serializer->multiline_json_array) {
if(serializer->status.buffer.size_used == 2) /* Empty buffer {} */
serializer->status.buffer.size_used = 0; /* Remove {} */
else
needed += 2;

} else {
if(serializer->status.buffer.size_used == 3) /* Empty buffer [{} */
serializer->status.buffer.size_used = 2; /* Remove {} */
else
needed += 2, add_comma = 1;
}
}

if(buff_diff < needed) {
Expand All @@ -628,18 +645,22 @@ int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
}

if(serializer->fmt == ndpi_serialization_format_json) {
if(add_comma)
serializer->buffer.data[serializer->status.buffer.size_used-1] = ',';
else
serializer->status.buffer.size_used--;
if (!serializer->multiline_json_array) {
if (add_comma)
serializer->buffer.data[serializer->status.buffer.size_used-1] = ',';
else
serializer->status.buffer.size_used--;
}
}

memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], record, record_len);
serializer->status.buffer.size_used += record_len;

if(serializer->fmt == ndpi_serialization_format_json) {
serializer->buffer.data[serializer->status.buffer.size_used] = ']';
if(add_comma) serializer->status.buffer.size_used++;
if (!serializer->multiline_json_array) {
serializer->buffer.data[serializer->status.buffer.size_used] = ']';
if(add_comma) serializer->status.buffer.size_used++;
}
}

ndpi_serialize_end_of_record(_serializer);
Expand All @@ -665,20 +686,26 @@ int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) {
}

if(serializer->fmt == ndpi_serialization_format_csv) {
serializer->buffer.data[serializer->status.buffer.size_used] = '\n';
serializer->status.buffer.size_used += 1;
serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
serializer->buffer.data[serializer->status.buffer.size_used] = '\0';
serializer->status.flags |= NDPI_SERIALIZER_STATUS_HDR_DONE;
serializer->status.flags |= NDPI_SERIALIZER_STATUS_EOR;

} else if(serializer->fmt == ndpi_serialization_format_json) {
if(!(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)) {
serializer->buffer.data[0] = '[';
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used],
buff_diff, "]");
}

if(serializer->multiline_json_array) {
serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
serializer->buffer.data[serializer->status.buffer.size_used] = '\0';

} else {
if(!(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)) {
serializer->buffer.data[0] = '[';
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "]");
}
}
serializer->status.flags |= NDPI_SERIALIZER_STATUS_ARRAY | NDPI_SERIALIZER_STATUS_EOR;
serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_COMMA;

} else /* ndpi_serialization_format_tlv */ {
serializer->buffer.data[serializer->status.buffer.size_used++] = ndpi_serialization_end_of_record;
}
Expand Down Expand Up @@ -709,18 +736,26 @@ static inline void ndpi_serialize_json_pre(ndpi_serializer *_serializer) {
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;

if(serializer->status.flags & NDPI_SERIALIZER_STATUS_EOR) {
serializer->status.buffer.size_used--; /* Remove ']' */
serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_EOR;
serializer->buffer.data[serializer->status.buffer.size_used++] = ',';
if(serializer->multiline_json_array) {
serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
} else {
serializer->status.buffer.size_used--; /* Remove ']' */
serializer->buffer.data[serializer->status.buffer.size_used++] = ',';
}
serializer->buffer.data[serializer->status.buffer.size_used++] = '{';

} else {
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)
serializer->status.buffer.size_used--; /* Remove ']' */
if(!serializer->multiline_json_array) {
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)
serializer->status.buffer.size_used--; /* Remove ']' */
}

serializer->status.buffer.size_used--; /* Remove '}' */

if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
serializer->status.buffer.size_used--; /* Remove ']' */
if(!serializer->multiline_json_array)
serializer->status.buffer.size_used--; /* Remove ']' */
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_SOL)
serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_SOL;
else
Expand All @@ -739,20 +774,24 @@ static inline void ndpi_serialize_json_pre(ndpi_serializer *_serializer) {
static inline int ndpi_serialize_json_post(ndpi_serializer *_serializer) {
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;

if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
if(serializer->status.buffer.size_used >= serializer->buffer.size)
return -1;
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
if (!serializer->multiline_json_array) {
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
if(serializer->status.buffer.size_used >= serializer->buffer.size)
return -1;
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
}
}

if(serializer->status.buffer.size_used >= serializer->buffer.size)
return -1;
serializer->buffer.data[serializer->status.buffer.size_used++] = '}';

if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
if(serializer->status.buffer.size_used >= serializer->buffer.size)
return -1;
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
if (!serializer->multiline_json_array) {
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
if(serializer->status.buffer.size_used >= serializer->buffer.size)
return -1;
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
}
}

serializer->status.flags |= NDPI_SERIALIZER_STATUS_COMMA;
Expand Down Expand Up @@ -2391,9 +2430,11 @@ void ndpi_serializer_rollback_snapshot(ndpi_serializer *_serializer) {

if(serializer->fmt == ndpi_serialization_format_json) {
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
serializer->buffer.data[serializer->status.buffer.size_used-1] = ']';
if (!serializer->multiline_json_array)
serializer->buffer.data[serializer->status.buffer.size_used-1] = ']';
} else {
serializer->buffer.data[0] = ' ';
if (!serializer->multiline_json_array)
serializer->buffer.data[0] = ' ';
serializer->buffer.data[serializer->status.buffer.size_used-1] = '}';
}
}
Expand Down

0 comments on commit 669a02c

Please sign in to comment.