#!perl
sub
version {
print
"mojo_cape_submit_extend v. 0.1.0\n"
;
exit
255;
}
my
$extend_version
= 1;
my
$version
;
my
$help
;
my
$rewind_by
= 300;
my
$dont_compress
= 0;
my
$incoming_json_dir
=
'/malware/incoming-json/'
;
GetOptions(
'h'
=> \
$help
,
'help'
=> \
$help
,
'v'
=> \
$version
,
'version'
=> \
$version
,
'Z'
=> \
$dont_compress
,
'm=s'
=> \
$incoming_json_dir
,
);
if
(
$version
) {
version;
}
if
(
$help
) {
pod2usage(
-exitval
=> 255,
-verbose
=> 2,
-output
=> \
*STDOUT
, );
}
my
$t
=
localtime
;
my
$target_time
=
$t
->epoch -
$rewind_by
;
my
@files
= File::Find::Rule->file()->name(
qr/^[0-9]+\.json$/
)->ctime(
'>='
.
$target_time
)->in(
$incoming_json_dir
);
my
$data
= {
totals
=> {
hash_changed
=> 0,
size_min
=> 0,
size_mean
=> 0,
size_median
=> 0,
size_mode
=> 0,
size_max
=> 0,
size_stddev
=> 0,
size_sum
=> 0,
sub_count
=> 0,
app_proto
=> {},
app_protos
=> 0,
},
slugs
=> {},
changed_hashes
=> [],
};
my
@all_sizes
;
my
$slug_sizes
;
my
$errorString
=
''
;
my
$error
= 0;
foreach
my
$file
(
@files
) {
my
$slug
;
my
$hash_changed
= 0;
$data
->{totals}{sub_count}++;
my
$short_file
=
$file
;
$short_file
=~ s/.*\///;
eval
{
my
$json
= decode_json( read_file(
$file
) );
push
(
@all_sizes
,
$json
->{cape_submit}{size} );
$slug
=
$json
->{suricata_extract_submit}{slug};
if
( !
defined
(
$data
->{slugs}{
$slug
} ) ) {
$data
->{slugs}{
$slug
} = {
hash_changed
=> 0,
size_min
=> 0,
size_mean
=> 0,
size_median
=> 0,
size_mode
=> 0,
size_max
=> 0,
size_stddev
=> 0,
size_sum
=> 0,
sub_count
=> 0,
app_proto
=> {},
app_protos
=> 0,
};
$slug_sizes
->{
$slug
} = [];
}
push
( @{
$slug_sizes
->{
$slug
} },
$json
->{cape_submit}{size} );
if
(
$json
->{cape_submit}{sha256} ne
$json
->{suricata_extract_submit}{sha256} ) {
$hash_changed
= 1;
push
( @{
$data
->{changed_hashes} },
$short_file
);
}
if
(
defined
(
$data
->{slugs}{
$slug
} ) ) {
$data
->{slugs}{
$slug
}{hash_changed} +=
$hash_changed
;
}
$data
->{totals}{hash_changed} +=
$hash_changed
;
$data
->{slugs}{
$slug
}{sub_count}++;
if
(
defined
(
$json
->{app_proto} ) ) {
my
$app_proto
=
$json
->{app_proto};
if
( !
defined
(
$data
->{totals}{app_proto}{
$app_proto
} ) ) {
$data
->{totals}{app_protos}++;
$data
->{totals}{app_proto}{
$app_proto
} = 0;
}
$data
->{totals}{app_proto}{
$app_proto
}++;
if
( !
defined
(
$data
->{slugs}{
$slug
}{app_proto}{
$app_proto
} ) ) {
$data
->{slugs}{
$slug
}{app_protos}++;
$data
->{slugs}{
$slug
}{app_proto}{
$app_proto
} = 0;
}
$data
->{slugs}{
$slug
}{app_proto}{
$app_proto
}++;
}
};
if
($@) {
$errorString
=
$errorString
.
$short_file
.
': '
. $@;
$error
= 1;
}
}
if
(
defined
(
$all_sizes
[0] ) ) {
$data
->{totals}{size_min} = min(
@all_sizes
);
$data
->{totals}{size_max} = max(
@all_sizes
);
$data
->{totals}{size_mean} = mean(
@all_sizes
);
$data
->{totals}{size_median} = median(
@all_sizes
);
$data
->{totals}{size_mode} = mode(
@all_sizes
);
$data
->{totals}{size_stddev} = stddev(
@all_sizes
);
$data
->{totals}{size_sum} = sum(
@all_sizes
);
my
@slugs
=
keys
( %{
$slug_sizes
} );
foreach
my
$item
(
@slugs
) {
$data
->{slugs}{
$item
}{size_min} = min(
@all_sizes
);
$data
->{slugs}{
$item
}{size_max} = max(
@all_sizes
);
$data
->{slugs}{
$item
}{size_mean} = mean(
@all_sizes
);
$data
->{slugs}{
$item
}{size_median} = median(
@all_sizes
);
$data
->{slugs}{
$item
}{size_mode} = mode(
@all_sizes
);
$data
->{slugs}{
$item
}{size_stddev} = stddev(
@all_sizes
);
$data
->{slugs}{
$item
}{size_sum} = sum(
@all_sizes
);
}
}
my
$json_string
= encode_json(
{
data
=>
$data
,
version
=>
$extend_version
,
error
=>
$error
,
errorString
=>
$errorString
,
}
);
if
( !
$dont_compress
) {
my
$compressed
= encode_base64( gzip(
$json_string
) );
$compressed
=~ s/\n//g;
$compressed
=
$compressed
.
"\n"
;
if
(
length
(
$compressed
) >
length
(
$json_string
) ) {
print
$json_string
.
"\n"
;
}
else
{
print
$compressed
;
}
}
else
{
print
$json_string
.
"\n"
;
}