Verified Commit 72501b2a authored by Luc Didry's avatar Luc Didry

Allow to delete URLs and ban IPs from safebrowsingcheck

parent 01a7e2ae
Revision history for Perl application Lstu
0.21-0 2018-??-??
- Allow to delete URLs and ban IPs from safebrowsingcheck
0.20-2 2018-10-07
- Handle very long URLs
......
......@@ -6,6 +6,8 @@ use File::Spec qw(catfile);
use Term::ProgressBar::Quiet;
use Mojo::Util qw(getopt);
use Mojo::Collection 'c';
use Lstu::DB::URL;
use Lstu::DB::Ban;
has description => 'Checks all URLs in database against Google Safe Browsing database (local copy)';
has usage => sub { shift->extract_usage };
......@@ -56,7 +58,10 @@ sub run {
getopt \@args,
'u|url=s{1,}' => \my @urls_to_check,
's|seconds=i' => \my $delay;
's|seconds=i' => \my $delay,
'r|remove' => \my $remove,
'a|all' => \my $all,
'b|ban' => \my $ban;
if ($c->app->gsb) {
my $urls;
......@@ -77,7 +82,8 @@ sub run {
{ name => 'Scanning '.$urls->size.' URLs', count => $urls->size, ETA => 'linear' }
);
my (@bad, %bad_ips, @bad_from_ips);
my $gsb = $c->app->gsb;
my $gsb = $c->app->gsb;
my $deleted = 0;
$urls->each(sub {
my ($e, $num) = @_;
......@@ -88,23 +94,52 @@ sub run {
if (@matches) {
push @bad, $e->{short};
$bad_ips{$e->{created_by}} = 1 if $e->{created_by};
$deleted += Lstu::DB::URL->new(
app => $c->app,
short => $e->{short}
)->delete if $remove;
}
});
say sprintf('All URLs (%d) have been scanned.', $urls->size);
say sprintf('%d bad URLs detected.', scalar(@bad));
say sprintf("If you want to delete the detected bad URLs, please do:\n carton exec script/lstu url --remove %s", join(' ', @bad)) if @bad;
if ($remove) {
say sprintf('%d bad URLs deleted.', $deleted) if $deleted;
} else {
say sprintf("If you want to delete the detected bad URLs, please do:\n carton exec script/lstu url --remove %s", join(' ', @bad)) if @bad;
}
$deleted = 0;
for my $ip (keys %bad_ips) {
my $u = Lstu::DB::URL->new(app => $c->app)->search_creator($ip);
$u->each(sub {
my ($e, $num) = @_;
push @bad_from_ips, $e->{short};
$deleted += Lstu::DB::URL->new(
app => $c->app,
short => $e->{short}
)->delete if ($remove && $all);
});
}
say sprintf("Bad URLs creators' IP addresses: \n %s", join(", ", keys %bad_ips)) if (keys %bad_ips);
say sprintf("If you want to delete the URLs created by the same IPs than the detected bad URLs, please do:\n carton exec script/lstu url --remove %s", join(' ', @bad_from_ips)) if @bad_from_ips;
my @ips = keys %bad_ips;
say sprintf("Bad URLs creators' IP addresses: \n %s", join(", ", @ips)) if (@ips);
if ($ban) {
for my $ip (@ips) {
Lstu::DB::Ban->new(
app => $c->app,
ip => $ip
)->ban_ten_years;
}
say sprintf("%d banned IP addresses", scalar(@ips)) if (@ips);
}
if ($remove && $all) {
say sprintf('%d URLs from same IPs deleted.', $deleted) if $deleted;
} else {
say sprintf("If you want to delete the URLs created by the same IPs than the detected bad URLs, please do:\n carton exec script/lstu url --remove %s", join(' ', @bad_from_ips)) if @bad_from_ips;
}
} else {
say 'It seems that safebrowsing_api_key isn\'t set. Please, check your configuration';
}
......@@ -140,6 +175,11 @@ Lstu::Command::safebrowsing - Checks all URLs in database against Google Safe Br
carton exec script/lstu safebrowsingcheck -u|--url <short> <short> Checks the space-separated URLs against Google Safe Browsing database
carton exec script/lstu safebrowsingcheck -s|--seconds <xxx> Checks URLs created the last xxx seconds against Google Safe Browsing database
Options (available with all commands):
-r|--remove Remove bad URLs that have been found
-a|--all Remove all URLs created by the same IP addresses that created bad URLs (only in combination with the `-r|--remove` option)
-b|--ban Ban IP addresses that created bad URLs
=cut
1;
......@@ -6,6 +6,7 @@ use Mojo::Collection 'c';
has 'ip';
has 'until';
has 'strike' => 0;
has 'record' => 0;
has 'app';
=head1 NAME
......@@ -191,6 +192,31 @@ Update the database record if one already exists, create one otherwise.
=back
=cut
sub increment_ban_delay {
my $c = shift;
my $penalty = shift;
my $until = time + $penalty;
my $h = {
strike => 1
};
if ($c->record) {
$c->app->dbi->db->query('UPDATE ban SET until = ?, strike = strike + 1 WHERE ip = ?', $until, $c->ip);
$h = $c->app->dbi->db->query('SELECT strike FROM ban WHERE ip = ?', $c->ip)->hashes->first;
} else {
$c->app->dbi->db->query('INSERT INTO ban (ip, until, strike) VALUES (?, ?, 1)', $c->ip, $until);
$c->record(1);
}
$c->strike($h->{strike});
$c->until($until);
return $c;
}
=head2 clear
=over 1
......@@ -237,6 +263,44 @@ sub delete_all {
$c->app->dbi->db->query('DELETE FROM ban');
}
=head2 ban_ten_years
=over 1
=item B<Usage> : C<$c-E<gt>ban_ten_years>
=item B<Arguments> : none
=item B<Purpose> : ban an IP address forever
=item B<Returns> : nothing is expected
=back
=cut
sub ban_ten_years {
my $c = shift;
my $until = time + 315360000;
my $h = {
strike => time
};
if ($c->record) {
$c->app->dbi->db->query('UPDATE ban SET until = ?, strike = ? WHERE ip = ?', $until, time, $c->ip);
$h = $c->app->dbi->db->query('SELECT strike FROM ban WHERE ip = ?', $c->ip)->hashes->first;
} else {
$c->app->dbi->db->query('INSERT INTO ban (ip, until, strike) VALUES (?, ?, 1)', $c->ip, $until);
$c->record(1);
}
$c->strike($h->{strike});
$c->until($until);
return $c;
}
=head2 _slurp
=over 1
......
......@@ -2,8 +2,6 @@
package Lstu::DB::Ban::MySQL;
use Mojo::Base 'Lstu::DB::Ban';
has 'record' => 0;
sub new {
my $c = shift;
......@@ -14,27 +12,4 @@ sub new {
return $c;
}
sub increment_ban_delay {
my $c = shift;
my $penalty = shift;
my $until = time + $penalty;
my $h = {
strike => 1
};
if ($c->record) {
$c->app->dbi->db->query('UPDATE ban SET until = ?, strike = strike + 1 WHERE ip = ?', $until, $c->ip);
$h = $c->app->dbi->db->query('SELECT strike FROM ban WHERE ip = ?', $c->ip)->hashes->first;
} else {
$c->app->dbi->db->query('INSERT INTO ban (ip, until, strike) VALUES (?, ?, 1)', $c->ip, $until);
$c->record(1);
}
$c->strike($h->{strike});
$c->until($until);
return $c;
}
1;
......@@ -2,8 +2,6 @@
package Lstu::DB::Ban::Pg;
use Mojo::Base 'Lstu::DB::Ban';
has 'record' => 0;
sub new {
my $c = shift;
......@@ -36,4 +34,25 @@ sub increment_ban_delay {
return $c;
}
sub ban_ten_years {
my $c = shift;
my $until = time + 315360000;
my $h = {
strike => time
};
if ($c->record) {
$h = $c->app->dbi->db->query('UPDATE ban SET until = ?, strike = ? WHERE ip = ? RETURNING strike', $until, time, $c->ip)->hashes->first;
} else {
$c->app->dbi->db->query('INSERT INTO ban (ip, until, strike) VALUES (?, ?, ?)', $c->ip, $until, time);
$c->record(1);
}
$c->strike($h->{strike});
$c->until($until);
return $c;
}
1;
......@@ -2,8 +2,6 @@
package Lstu::DB::Ban::SQLite;
use Mojo::Base 'Lstu::DB::Ban';
has 'record' => 0;
sub new {
my $c = shift;
......@@ -14,27 +12,4 @@ sub new {
return $c;
}
sub increment_ban_delay {
my $c = shift;
my $penalty = shift;
my $until = time + $penalty;
my $h = {
strike => 1
};
if ($c->record) {
$c->app->dbi->db->query('UPDATE ban SET until = ?, strike = strike + 1 WHERE ip = ?', $until, $c->ip)->hashes->first;
$h = $c->app->dbi->db->query('SELECT strike FROM ban WHERE ip = ?', $c->ip)->hashes->first;
} else {
$c->app->dbi->db->query('INSERT INTO ban (ip, until, strike) VALUES (?, ?, 1)', $c->ip, $until);
$c->record(1);
}
$c->strike($h->{strike});
$c->until($until);
return $c;
}
1;
......@@ -4,6 +4,7 @@ use Mojo::Base -base;
has 'token';
has 'until';
has 'record' => 0;
has 'app';
=head1 NAME
......@@ -103,6 +104,21 @@ sub to_hash {
=cut
sub delete {
my $c = shift;
$c->app->dbi->db->query('DELETE FROM sessions WHERE token = ?', $c->token);
my $h = $c->app->dbi->db->query('SELECT * FROM sessions WHERE token = ?', $c->token)->hashes;
if ($h->size) {
# We found the session, it hasn't been deleted
return 0;
} else {
$c = Lstu::DB::Session->new(app => $c->app);
# We didn't found the session, it has been deleted
return 1;
}
}
=head2 write
=over 1
......
......@@ -2,8 +2,6 @@
package Lstu::DB::Session::MySQL;
use Mojo::Base 'Lstu::DB::Session';
has 'record' => 0;
sub new {
my $c = shift;
......@@ -14,19 +12,4 @@ sub new {
return $c;
}
sub delete {
my $c = shift;
$c->app->dbi->db->query('DELETE FROM sessions WHERE token = ?', $c->token);
my $h = $c->app->dbi->db->query('SELECT * FROM sessions WHERE token = ?', $c->token)->hashes;
if ($h->size) {
# We found the session, it hasn't been deleted
return 0;
} else {
$c = Lstu::DB::Session->new(app => $c->app);
# We didn't found the session, it has been deleted
return 1;
}
}
1;
......@@ -2,8 +2,6 @@
package Lstu::DB::Session::Pg;
use Mojo::Base 'Lstu::DB::Session';
has 'record' => 0;
sub new {
my $c = shift;
......
......@@ -2,8 +2,6 @@
package Lstu::DB::Session::SQLite;
use Mojo::Base 'Lstu::DB::Session';
has 'record' => 0;
sub new {
my $c = shift;
......@@ -14,19 +12,4 @@ sub new {
return $c;
}
sub delete {
my $c = shift;
$c->app->dbi->db->query('DELETE FROM sessions WHERE token = ?', $c->token);
my $h = $c->app->dbi->db->query('SELECT * FROM sessions WHERE token = ?', $c->token)->hashes;
if ($h->size) {
# We found the session, it hasn't been deleted
return 0;
} else {
$c = Lstu::DB::Session->new(app => $c->app);
# We didn't found the session, it has been deleted
return 1;
}
}
1;
......@@ -7,6 +7,7 @@ has 'url';
has 'counter' => 0;
has 'timestamp';
has 'created_by';
has 'record' => 0;
has 'app';
=head1 NAME
......@@ -106,6 +107,18 @@ sub to_hash {
=back
=cut
sub increment_counter {
my $c = shift;
$c->app->dbi->db->query('UPDATE lstu SET counter = counter + 1 WHERE short = ?', $c->short);
my $h = $c->app->dbi->db->query('SELECT counter FROM lstu WHERE short = ?', $c->short)->hashes->first;
$c->counter($h->{counter});
return $c;
}
=head2 write
=over 1
......@@ -149,6 +162,23 @@ sub write {
=back
=cut
sub delete {
my $c = shift;
$c->app->dbi->db->query('DELETE FROM lstu WHERE short = ?', $c->short);
my $h = $c->app->dbi->db->query('SELECT * FROM lstu WHERE short = ?', $c->short)->hashes;
if ($h->size) {
# We found the URL, it hasn't been deleted
return 0;
} else {
$c = Lstu::DB::URL->new(app => $c->app);
# We didn't found the URL, it has been deleted
return 1;
}
}
=head2 exist
=over 1
......
......@@ -2,8 +2,6 @@
package Lstu::DB::URL::MySQL;
use Mojo::Base 'Lstu::DB::URL';
has 'record' => 0;
sub new {
my $c = shift;
......@@ -14,29 +12,4 @@ sub new {
return $c;
}
sub increment_counter {
my $c = shift;
$c->app->dbi->db->query('UPDATE lstu SET counter = counter + 1 WHERE short = ?', $c->short);
my $h = $c->app->dbi->db->query('SELECT counter FROM lstu WHERE short = ?', $c->short)->hashes->first;
$c->counter($h->{counter});
return $c;
}
sub delete {
my $c = shift;
$c->app->dbi->db->query('DELETE FROM lstu WHERE short = ?', $c->short);
my $h = $c->app->dbi->db->query('SELECT * FROM lstu WHERE short = ?', $c->short)->hashes;
if ($h->size) {
# We found the URL, it hasn't been deleted
return 0;
} else {
$c = Lstu::DB::URL->new(app => $c->app);
# We didn't found the URL, it has been deleted
return 1;
}
}
1;
......@@ -2,8 +2,6 @@
package Lstu::DB::URL::Pg;
use Mojo::Base 'Lstu::DB::URL';
has 'record' => 0;
sub new {
my $c = shift;
......
......@@ -2,8 +2,6 @@
package Lstu::DB::URL::SQLite;
use Mojo::Base 'Lstu::DB::URL';
has 'record' => 0;
sub new {
my $c = shift;
......@@ -14,29 +12,4 @@ sub new {
return $c;
}
sub increment_counter {
my $c = shift;
$c->app->dbi->db->query('UPDATE lstu SET counter = counter + 1 WHERE short = ?', $c->short);
my $h = $c->app->dbi->db->query('SELECT counter FROM lstu WHERE short = ?', $c->short)->hashes->first;
$c->counter($h->{counter});
return $c;
}
sub delete {
my $c = shift;
$c->app->dbi->db->query('DELETE FROM lstu WHERE short = ?', $c->short);
my $h = $c->app->dbi->db->query('SELECT * FROM lstu WHERE short = ?', $c->short)->hashes;
if ($h->size) {
# We found the URL, it hasn't been deleted
return 0;
} else {
$c = Lstu::DB::URL->new(app => $c->app);
# We didn't found the URL, it has been deleted
return 1;
}
}
1;
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment