cmaffio/pannello
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Aggiunto sistema partizionamento DB per ottimizzare l'accesso ai dati

Browse Source
This commit is contained in:
cmaffio
2016-04-18 17:08:22 +02:00
parent e6e4b5d15b
commit 2085e68354
18 changed files with 1122 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
script/partitioning.pl
View File

@@ -9,55 +9,154 @@ use Time::Local;
our $messaggi = 0;
while( $_ = shift @ARGV ) {
$messaggi = 1 if /^\-v$/;
our $messaggi = 1 if /^\-v$/;
$start = 1 if /^\-i$/;
}
my $tabella = "proxy_dati";
my $campo = "data_destroy";
open CONFFILE, "< $Bin/../core/config.php" or die ("Manca file di configurazione\n");
while (<CONFFILE>) {
if(/^\$db_data_server = "(\S+)"/ ) {
$db_host = $1;
our $db_host = $1;
}
if(/^\$db_data_name = "(\S+)"/ ) {
$db_name = $1;
our $db_name = $1;
}
if(/^\$db_data_user = "(\S+)"/ ) {
$db_user = $1;
our $db_user = $1;
}
if(/^\$db_data_pwd = "(\S+)"/ ) {
$db_pass = $1;
our $db_pass = $1;
}
}
close CONFFILE;
printf "Inizio attivita' %s\n",strftime('%d-%m-%Y %H:%M:%S',localtime) if ($messaggi);
my $dbmysql = DBI->connect("DBI:mysql:;host=$db_host", "$db_user", "$db_pass") or die ($DBI::errstr);
$sts = $dbmysql->prepare("use $db_name");
$sts->execute ();
$part = "ALTER TABLE dati_traffico PARTITION BY RANGE ( dataunix ) (\n";
#$query = "SELECT DATE_FORMAT(data, \"%Y%m%d\") AS giorno, DATE_FORMAT(data, \"%Y-%m-%d\") AS giorno2 FROM dati_traffico GROUP BY giorno ORDER BY giorno";
$query = "SELECT DATE_FORMAT(data, \"%Y%m%d\") AS giorno, DATE_FORMAT(data, \"%Y\") AS a, DATE_FORMAT(data, \"%m\") AS m, DATE_FORMAT(data, \"%d\") AS g FROM dati_traffico GROUP BY giorno ORDER BY giorno";
$sts = $dbmysql->prepare($query);
$sts->execute ();
while (my $dato = $sts->fetchrow_hashref) {
my $giorno = $$dato{'giorno'};
my $a = $$dato{'a'};
my $m = $$dato{'m'};
my $g = $$dato{'g'};
# my $giorno2 = $$dato{'giorno2'};
# print "$giorno\n";
my $timest = timelocal (59, 59, 23, $g, $m-1, $a);
# $part .= "PARTITION p$giorno VALUES LESS THAN ( UNIX_TIMESTAMP('$giorno2 23:59:59') ),\n";
$part .= "PARTITION p$giorno VALUES LESS THAN ( $timest ),\n";
if ($start) {
inizializza($dbmysql, $tabella, $campo);
exit;
}
$part .= "PARTITION p99999999 VALUES LESS THAN (MAXVALUE)\n";
$part .= ");\n";
print $part;
aggiorna($dbmysql, $tabella, $campo);
exit;
printf "Inizio partizionamento %s\n",strftime('%d-%m-%Y %H:%M:%S',localtime) if ($messaggi);
$sts = $dbmysql->prepare($part);
$sts->execute ();
printf "Termine attivita' %s\n",strftime('%d-%m-%Y %H:%M:%S',localtime) if ($messaggi);
sub inizializza {
my $dbmysql = shift;
my $tabella = shift;
my $campo = shift;
printf "Inizio attivita' %s\n",strftime('%d-%m-%Y %H:%M:%S',localtime) if ($messaggi);
$part = "ALTER TABLE $tabella PARTITION BY RANGE COLUMNS ( $campo ) (\n";
$query = " SELECT
DATE_FORMAT($campo, \"%Y%m%d\") AS giorno,
DATE_FORMAT($campo, \"%Y-%m-%d 23:59:59\") AS partenza
FROM
$tabella
GROUP BY
giorno
ORDER BY
giorno
";
$sts = $dbmysql->prepare($query);
$sts->execute ();
while (my $dato = $sts->fetchrow_hashref) {
my $giorno = $$dato{'giorno'};
my $partenza = $$dato{'partenza'};
$part .= "\tPARTITION p$giorno VALUES LESS THAN ( '$partenza' ),\n";
}
$part .= "\tPARTITION p99999999 VALUES LESS THAN (MAXVALUE)\n";
$part .= ");\n";
print $part if ($messaggi);
printf "Inizio partizionamento %s\n",strftime('%d-%m-%Y %H:%M:%S',localtime) if ($messaggi);
$sts = $dbmysql->prepare($part);
$sts->execute ();
printf "Termine attivita' %s\n",strftime('%d-%m-%Y %H:%M:%S',localtime) if ($messaggi);
}
sub aggiorna {
my $dbmysql = shift;
my $tabella = shift;
my $campo = shift;
printf "Inizio attivita' %s\n",strftime('%d-%m-%Y %H:%M:%S',localtime) if ($messaggi);
$query = "SHOW CREATE TABLE $tabella";
$sts = $dbmysql->prepare($query);
$sts->execute ();
my @row = $sts->fetchrow_array();
my @risp = split '\n', $row[1];
my $ultimo = "";
foreach $riga (@risp) {
if ($riga =~ /^[\(\s]?PARTITION p(\d+) VALUES LESS THAN \(\'(\S+) (\S+)\'\) .*$/) {
next if ($1 eq '19700101');
$ultimo = $2;
my $diff = diffgg ($2);
if ($diff > 365) {
my $query = "ALTER TABLE $tabella DROP PARTITION p$1";
$sts = $dbmysql->prepare($query);
$sts->execute ();
}
}
}
if (diffgg($ultimo) > -5) {
my $prequery = "ALTER TABLE $tabella REORGANIZE PARTITION p99999999 INTO (\n";
my $postquery = "PARTITION p99999999 VALUES LESS THAN (MAXVALUE))\n";
my $inquery = "";
while (diffgg($ultimo) > -5) {
$ultimo = addgg ($ultimo);
my @tmp0 = split / /, $ultimo;
my @tmp = split /-/, $tmp0[0];
my $p = $tmp[0].$tmp[1].$tmp[2];
$inquery .= "PARTITION p$p VALUES LESS THAN ('$ultimo'),\n";
}
$query = "$prequery$inquery$postquery";
$sts = $dbmysql->prepare($query);
$sts->execute ();
}
printf "Termine attivita' %s\n",strftime('%d-%m-%Y %H:%M:%S',localtime) if ($messaggi);
}
sub diffgg {
my $data = shift;
my @today = localtime();
my $time = timelocal(@today);
my @part = split /-/, $data;
my @tempo = (59, 59, 23, $part[2], $part[1]-1, $part[0]-1900);
my $tempo = timelocal(@tempo);
return int(($time - $tempo) / 86400);
}
sub addgg {
my $data = shift;
my @part = split /-/, $data;
my @tempo = (59, 59, 23, $part[2], $part[1]-1, $part[0]-1900);
my $tempo = timelocal(@tempo) + 86400;
return strftime('%Y-%m-%d %H:%M:%S',localtime($tempo))
}