Verified Commit c06ced5f authored by Luc Didry's avatar Luc Didry

Merge branch 'development'

parents d1fc25ed f87390d3
.cpan
.cpanm
.git
docker-compose.yml
local
log/*
lstu.conf
lstu.db
\ No newline at end of file
.cpan
.cpanm
.*_history
local/*
cover_db/*
safebrowsing_db/*
......
......@@ -34,7 +34,7 @@ variables:
services:
- name: rroemhild/test-openldap
alias: rroemhild-test-openldap
coverage: '/Total .*\d+\.\d+$/'
coverage: '/Total .*(\d+\.\d+)$/'
.pg_template: &pg_definition
stage: tests
retry: 2
......@@ -45,7 +45,7 @@ variables:
alias: postgres
- name: rroemhild/test-openldap
alias: rroemhild-test-openldap
coverage: '/Total .*\d+\.\d+$/'
coverage: '/Total .*(\d+\.\d+)$/'
.mysql_template: &mysql_definition
stage: tests
retry: 2
......@@ -56,7 +56,7 @@ variables:
alias: mariadb
- name: rroemhild/test-openldap
alias: rroemhild-test-openldap
coverage: '/Total .*\d+\.\d+$/'
coverage: '/Total .*(\d+\.\d+)$/'
### Podcheck
##
......
......@@ -13,3 +13,4 @@
* Pierre Rudloff
* Belvar (Gwenn M) (breton translation)
* Ira W. Snyder
* Arnaud de Mouhy (docker port)
\ No newline at end of file
Revision history for Perl application Lstu
0.20-0 2018-??-??
- Docker port, thanks to Arnaud de Mouhy
- Improve safebrowsingcheck CLI:
- displays creators' IP addresses + other URLs from thoses IPs
- allow to specify URLs to check
- allow to check URLs created <argument> seconds ago
0.19-3 2018-09-08
- Update translations
......
FROM alpine:3.7
ARG BUILD_DATE
ARG VCS_REF
ARG VERSION
LABEL org.label-schema.build-date=$BUILD_DATE \
org.label-schema.name="Let's Shorten That URL" \
org.label-schema.url="https://lstu.fr/" \
org.label-schema.vcs-ref=$VCS_REF \
org.label-schema.vcs-url="https://git.framasoft.org/luc/lstu" \
org.label-schema.vendor="Luc Didry" \
org.label-schema.version=$VERSION \
org.label-schema.schema-version="1.0"
RUN adduser -D lstu
COPY --chown=lstu:lstu . /home/lstu
WORKDIR /home/lstu
RUN apk --update add ca-certificates perl perl-netaddr-ip perl-io-socket-ssl perl-dbd-pg mariadb-client-libs libpng zlib \
&& apk add --virtual .build-deps build-base perl-utils perl-dev make sudo zlib-dev libpng-dev postgresql-dev mariadb-dev \
&& cpan Carton \
&& sudo -u lstu carton install --deployment --without=test \
&& perl -MCPAN -e 'install inc::latest' \
&& perl -MCPAN -e 'install Config::FromHash' \
&& apk del .build-deps \
&& rm -rf /var/cache/apk/*
USER lstu
ENTRYPOINT ["/bin/sh", "/home/lstu/docker/entrypoint.sh"]
\ No newline at end of file
......@@ -12,6 +12,8 @@ Lstu is licensed under the terms of the WTFPL. See the LICENSE file.
Please, see the [wiki](https://framagit.org/luc/lstu/wikis/home).
Or you can see [usage with Docker](https://framagit.org/luc/lstu/wikis/usage-with-docker).
## How many URLs can it handle ?
By default, there are 8 361 453 672 available combinations.
......
version: '3.3'
services:
app_dev:
build: .
ports:
- 8080:8080
volumes:
- .:/home/lstu
command: dev
mariadb_dev:
image: mariadb:10.2
environment:
MYSQL_ROOT_PASSWORD: password
MYSQL_DATABASE: lstu
postgres_dev:
image: postgres:10.3-alpine
environment:
POSTGRES_PASSWORD: password
POSTGRES_USER: lstu
POSTGRES_DB: lstu
memcached:
image: memcached:1.5-alpine
adminer:
image: dehy/adminer
ports:
- 8081:80
\ No newline at end of file
version: '3.3'
services:
app:
build: .
ports:
- 8080:8080
volumes:
- ./lstu.conf:/home/lstu/lstu.conf:ro
db:
image: mariadb:10.2
environment:
MYSQL_ROOT_PASSWORD: password
MYSQL_DATABASE: lstu
cache:
image: memcached:1.5-alpine
minion:
build: .
command: minion
volumes:
- ./lstu.conf:/home/lstu/lstu.conf:ro
minion_db:
image: mariadb:10.2
environment:
MYSQL_ROOT_PASSWORD: password
MYSQL_DATABASE: lstu_minion
\ No newline at end of file
version: '3.3'
services:
app:
image: aquinum/lstu
configs:
- source: lstu.conf
target: /home/lstu/lstu.conf
uid: '1000'
gid: '1000'
mode: 0440
deploy:
replicas: 1
db:
image: mariadb:10.2
environment:
MYSQL_ROOT_PASSWORD: <changeme>
MYSQL_DATABASE: lstu
cache:
image: memcached:1.5-alpine
minion:
image: aquinum/lstu
command: minion
configs:
- source: lstu.conf
target: /home/lstu/lstu.conf
uid: '1000'
gid: '1000'
mode: 0440
minion_db:
image: mariadb:10.2
environment:
MYSQL_ROOT_PASSWORD: <changeme>
MYSQL_DATABASE: lstu_minion
configs:
lstu.conf:
file: ./lstu.conf
\ No newline at end of file
#!/bin/sh
set -eu
cd ~lstu
if [ "${1:-}" == "dev" ]
then
echo ""
echo ""
echo "Container started in dev mode. Connect to the container with the following command:"
echo " docker-compose -f docker-compose.dev.yml exec -u root app_dev sh"
echo ""
echo ""
echo "You can then install the build dependencies with this command"
echo " apk --update add vim build-base perl-utils perl-dev make sudo zlib-dev libpng-dev postgresql-dev mariadb-dev"
tail -f /dev/null
exit 0
fi
# If MySQL/PostgreSQL, wait for database to be up
DB_TYPE=$(perl utilities/read_conf.pl dbtype)
DB_HOST=
DB_PORT=
if [ "$DB_TYPE" == "mysql" ]
then
DB_HOST=$(perl utilities/read_conf.pl mysqldb/host db)
DB_PORT=$(perl utilities/read_conf.pl mysqldb/port 3306)
fi
if [ "$DB_TYPE" == "postgresql" ]
then
DB_HOST=$(perl utilities/read_conf.pl pgdb/host db)
DB_PORT=$(perl utilities/read_conf.pl pgdb/port 5432)
fi
if [ -n "$DB_HOST" -a -n "$DB_PORT" ]
then
while ! nc -vz "${DB_HOST}" "${DB_PORT}"; do
echo "Waiting for database..."
sleep 1;
done
fi
if [ "${1:-}" == "minion" ]
then
exec carton exec script/application minion worker
fi
exec carton exec hypnotoad -f script/lstu
\ No newline at end of file
#!/bin/bash
# https://medium.com/microscaling-systems/labelling-automated-builds-on-docker-hub-f3d073fb8e1#.yldbwesu7
if [ -z "${IMAGE_NAME}" ]; then
echo ! "IMAGE_NAME" variable missing.
echo eg. IMAGE_NAME="aquinum/lstu:local" sh $0
exit 1
fi
function evil_git_dirty {
[[ $(git diff --shortstat 2> /dev/null | tail -n1) != "" ]] && echo "-dirty"
}
function last_tag {
git describe --abbrev=0
}
function version_number_plus {
[[ $(git rev-list --count ${1}..HEAD) -gt 0 ]] && echo "+"
}
VERSION=$(last_tag)
VERSION=${VERSION}$(version_number_plus ${VERSION})
VCS_REF=$(git rev-parse --short HEAD)$(evil_git_dirty)
BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
docker build --build-arg VCS_REF=${VCS_REF} \
--build-arg BUILD_DATE=${BUILD_DATE} \
--build-arg VERSION=${VERSION} \
-t $IMAGE_NAME .
......@@ -4,6 +4,8 @@ use Mojo::Base 'Mojolicious::Command';
use FindBin qw($Bin);
use File::Spec qw(catfile);
use Term::ProgressBar::Quiet;
use Mojo::Util qw(getopt);
use Mojo::Collection 'c';
has description => 'Checks all URLs in database against Google Safe Browsing database (local copy)';
has usage => sub { shift->extract_usage };
......@@ -52,13 +54,29 @@ sub run {
$c->app->plugin('Lstu::Plugin::Helpers');
getopt \@args,
'u|url=s{1,}' => \my @urls_to_check,
's|seconds=i' => \my $delay;
if ($c->app->gsb) {
my $urls = Lstu::DB::URL->new(app => $c->app)->get_all_urls;
my $urls;
if (@urls_to_check) {
$urls = c(get_shorts($c, @urls_to_check));
} elsif ($delay) {
$urls = Lstu::DB::URL->new(app => $c->app)->get_all_urls_created_ago($delay);
} else {
$urls = Lstu::DB::URL->new(app => $c->app)->get_all_urls;
}
unless ($urls->size) {
say 'No URLs to check.';
exit;
}
my $progress = Term::ProgressBar::Quiet->new(
{ name => 'Scanning '.$urls->size.' URLs', count => $urls->size, ETA => 'linear' }
);
my @bad;
my (@bad, %bad_ips, @bad_from_ips);
my $gsb = $c->app->gsb;
$urls->each(sub {
my ($e, $num) = @_;
......@@ -69,19 +87,46 @@ sub run {
if (@matches) {
push @bad, $e->{short};
$bad_ips{$e->{created_by}} = 1 if $e->{created_by};
}
});
say sprintf('All URLs (%d) have been scanned.', $urls->size);
say sprintf('%d bad URLs detected.', scalar(@bad));
say scalar($c->app->gsb->lookup(url => 'http://malware.testing.google.test/testing/malware/'));
say sprintf("If you want to delete the detected bad URLs, please do:\n carton exec script/lstu url --remove %s", join(' ', @bad)) if @bad;
for my $ip (keys %bad_ips) {
my $u = Lstu::DB::URL->new(app => $c->app)->search_creator($ip);
$u->each(sub {
my ($e, $num) = @_;
push @bad_from_ips, $e->{short};
});
}
say sprintf("Bad URLs creators' IP addresses: \n %s", join(", ", keys %bad_ips)) if (keys %bad_ips);
say sprintf("If you want to delete the URLs created by the same IPs than the detected bad URLs, please do:\n carton exec script/lstu url --remove %s", join(' ', @bad_from_ips)) if @bad_from_ips;
} else {
say 'It seems that safebrowsing_api_key isn\'t set. Please, check your configuration';
}
}
sub get_shorts {
my $c = shift;
my @shorts = @_;
my @results;
for my $short (@shorts) {
my $u = Lstu::DB::URL->new(app => $c->app, short => $short);
if ($u->url) {
push @results, $u->to_hash;
} else {
say sprintf('Sorry, unable to find an URL with short = %s', $short);
}
}
return @results;
}
=encoding utf8
=head1 NAME
......
......@@ -406,6 +406,29 @@ sub get_all_urls {
$c->app->dbi->db->select('lstu', undef, { url => { '!=', undef } })->hashes;
}
=head2 get_all_urls_created_ago
=over 1
=item B<Usage> : C<$c-E<gt>get_all_urls_created_ago($seconds)>
=item B<Arguments> : integer, number of seconds
=item B<Purpose> : return all non-empty records created less than $seconds agog
=item B<Returns> : a Mojo::Collection containing hashes of matching records
=back
=cut
sub get_all_urls_created_ago {
my $c = shift;
my $delay = shift;
$c->app->dbi->db->select('lstu', undef, { url => { '!=', undef }, timestamp => { '>=', time - $delay } })->hashes;
}
=head2 _slurp
=over 1
......
#!/usr/bin/env perl
# read_conf.pl - Outputs lstu's config value for given keypath
use strict;
use warnings;
use FindBin;
use Cwd qw(realpath);
use Config::FromHash;
sub print_help {
print <<END;
Usage:
$0 [-h] KEYPATH [DEFAULT] -- reads the lstu.conf file and outputs value at KEYPATH, or prints DEFAULT value
where:
-h prints this help message and exit
KEYPATH refers to the path in the hash, slash-separated (ie. mysqldb/host)
DEFAULT if keypath is undefined, return this value (ie. localhost)
END
return;
}
if (defined $ARGV[0] and $ARGV[0] eq "-h") {
print_help();
exit 0;
}
if (not defined $ARGV[0]) {
print_help();
exit 1;
}
my $filename = realpath("$FindBin::Bin/../lstu.conf");
if (! -r $filename) {
print STDERR "Error: unable to read config file \"$filename\"\n";
exit 1;
}
my $config = Config::FromHash->new(filename => $filename);
my $value = $config->get($ARGV[0]);
if (defined $value) {
print $value;
exit 0;
}
if (not defined $value and defined $ARGV[1]) {
print $ARGV[1];
exit 0;
}
print STDERR "Error: keypath \"$ARGV[0]\" not found in config, and no default value provided.\n";
exit 1;
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment