script for generating raw sql data (#34)

scripts/log2sql.pl

@@ -0,0 +1,123 @@
+#!/usr/bin/perl
+
+use warnings;
+use strict;
+use open qw(:std :utf8);
+use utf8;
+use Getopt::Long;
+use DBI;
+use File::Path;
+use File::Spec;
+use File::Basename;
+use Digest::MD5;
+
+
+
+my $outdir = '.';
+my (@logfiles,$dbuser,$dbhost,$dbdatabase,$dbpassword);
+my $dbport = 5432;
+my $ignore = qr/\.(gif|jpg|jpeg|tiff|png|js|css|eot|ico|svg)$/;
+my @services;
+
+
+GetOptions (
+            'in-files=s{1,}' => \@logfiles,
+            'out-dir=s' => \$outdir,
+            'db-user=s' => \$dbuser,
+            'db-host=s' => \$dbhost,
+            'db-database=s' => \$dbdatabase,
+            'db-password=s' => \$dbpassword,
+            'db-port=i' => \$dbport,
+        );
+
+
+my $dbi = DBI->connect("dbi:Pg:host=$dbhost;database=$dbdatabase;port=$dbport", $dbuser, $dbpassword, { RaiseError => 1, AutoCommit => 0, ReadOnly => 1 });
+
+unless($dbi){
+  die "Unable to connect to database\n";
+}
+
+File::Path::mkpath($outdir) unless -d $outdir;
+
+my $file_id = 1;
+
+my $sql='
+SELECT MAX(file_id) AS file_id
+FROM log_files;
+';
+
+my $sth = $dbi->prepare($sql);
+$sth->execute;
+if(my $result = $sth->fetchrow_hashref){
+  $file_id = $result->{file_id} + 1;
+}
+
+$sql='
+SELECT service_id, prefix
+FROM services
+ORDER BY service_id;
+';
+
+
+$sth = $dbi->prepare($sql);
+$sth->execute;
+while(my $result = $sth->fetchrow_hashref){
+  push @services, {id => $result->{service_id}, reg => $result->{prefix}}
+}
+
+for my $log_file_path (@logfiles) {
+  my $log_file = basename($log_file_path);
+  my $sql="SELECT 1 FROM log_files WHERE file_name='$log_file';";
+  my $sth = $dbi->prepare($sql);
+  $sth->execute;
+  if(my $result = $sth->fetchrow_hashref){
+    print STDERR "ERROR: skipping $log_file - file exists in database\n";
+    next;
+  }
+  $file_id++;
+  my ($file_name) = $log_file =~ m/^(.*)\./;
+  my $sql_file = "$file_name.sql";
+  my $dump_file = "$file_name.dump";
+  my ($first_line_checksum, $last_read_line_checksum, $lines_read, $lines_valid) = (undef, undef, 0, 0);
+  open LOG, "<$log_file_path" or die "Could not open $log_file_path: $!";
+  open DUMP, ">".File::Spec->catfile($outdir,$dump_file) or die "Could not open $dump_file: $!";
+  while(my $line = <LOG>){
+    $lines_read++;
+    $line =~ s/\n$//;
+    my ($remote_addr,$time_local, $remote_user, $method, $request, $protocol, $status, $body_bytes_sent,$http_referer, $http_user_agent, $unit) =
+        $line =~ /^([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+) - ([^\s]+) \[(\d{2})\/\w{3}\/\d{4}:\d{2}:\d{2}:\d{2} \+\d{4}\] +"([A-Z]+) ([^" ]*) ([^" ]*)" (2\d\d) ([-\d]*) "([^"]*)" "([^"]*)" [^\s]* [^\s]* [^\s](?: .*billing:infclen=(\d+))?/;
+    $unit //=0;
+    if($status && $request !~ /$ignore/){
+      $last_read_line_checksum = Digest::MD5::md5_hex($line);
+      $first_line_checksum //= $last_read_line_checksum;
+      my $service_id;
+      for my $service (@services){
+        if($request =~ $service->{reg}){
+          $service_id = $service->{id};
+          last if $service_id > 0;
+        }
+      }
+      next unless defined $service_id;
+      $lines_valid++;
+##      print STDERR "$service_id: $request\n";
+      print DUMP join("\t", ($file_id,$service_id,$lines_valid,$last_read_line_checksum,$remote_addr,$remote_user,$time_local, $method, $request, $protocol, $status, $body_bytes_sent,$http_referer, $http_user_agent, $unit)),"\n";
+    }
+  }
+  close DUMP;
+  close LOG;
+
+  open FILE, ">".File::Spec->catfile($outdir,$sql_file) or die "Could not open $sql_file: $!";
+  print FILE " -- $log_file sql dump
+INSERT
+INTO log_files(file_id, file_name, first_line_checksum, last_read_line_checksum, lines_read, lines_valid, tail)
+VALUES($file_id,'$log_file','$first_line_checksum', '$last_read_line_checksum', $lines_read, $lines_valid,FALSE);
+
+COPY log_file_entries(file_id, service_id, line_number, line_checksum, remote_addr, remote_user, time_local, method, request, protocol, status, body_bytes_sent, http_referer, http_user_agent, unit)
+FROM '$dump_file'
+WITH
+   DELIMITER AS '\\t';
+";
+  close FILE;
+}
+
+$dbi->disconnect();