diff --git a/MANIFEST b/MANIFEST index a13e73d..0d8bb7c 100644 --- a/MANIFEST +++ b/MANIFEST @@ -6,3 +6,9 @@ lib/MogileFS/Utils.pm mogtool mogadm mogstats +mogupload +mogdelete +mogfetch +mogfileinfo +moglistfids +moglistkeys diff --git a/Makefile.PL b/Makefile.PL index d596a58..2bfa1b9 100644 --- a/Makefile.PL +++ b/Makefile.PL @@ -8,7 +8,10 @@ WriteMakefile( VERSION_FROM => 'lib/MogileFS/Utils.pm', AUTHOR => 'Brad Fitzpatrick ', ABSTRACT => 'MogileFS utilities', - EXE_FILES => ['mogtool', 'mogadm', 'mogstats'], + EXE_FILES => ['mogtool', 'mogadm', 'mogstats', + 'mogupload', 'mogfetch', 'mogdelete', 'mogfileinfo', 'moglistkeys', + 'moglistfids', + ], PREREQ_PM => { 'LWP::Simple' => 0, 'Compress::Zlib' => 0, diff --git a/lib/MogileFS/Utils.pm b/lib/MogileFS/Utils.pm index 220a2d7..44ba134 100644 --- a/lib/MogileFS/Utils.pm +++ b/lib/MogileFS/Utils.pm @@ -3,6 +3,114 @@ package MogileFS::Utils; our $VERSION = '2.18'; +use Getopt::Long; +use MogileFS::Client; + +use fields ( + 'config' + ); + +# Helper object for the individual utilities. +sub new { + my MogileFS::Utils $self = shift; + $self = fields::new($self) unless ref $self; + $self->_init(@_); + + return $self; +} + +# Predefine some options via configuration. +sub _init { + my MogileFS::Utils $self = shift; + + $self->{config} = {}; +} + +sub _readconf { + my MogileFS::Utils $self = shift; + my $args = shift; + + # Liftedish from mogadm, but we can refactor mogadm to use this instead. + my @configs = ($args->{config}, $ENV{MOGUTILSCONF}, + "$ENV{HOME}/.mogilefs.conf", + "/etc/mogilefs/mogilefs.conf"); + my %opts = (); + for my $fn (reverse @configs) { + next unless $fn && -e $fn; + open my $file, "<$fn" + or die "unable to open $fn: $!"; + while (<$file>) { + s/\#.*//; + next unless m/^\s*(\w+)\s*=\s*(.+?)\s*$/; + $opts{$1} = $2 unless ( defined $opts{$1} ); + } + close $file; + } + + return \%opts; +} + +sub config { + my MogileFS::Utils $self = shift; + return $self->{config}; +} + +sub getopts { + my MogileFS::Utils $self = shift; + my $usage = shift; + my @want = @_; + + my %opts = (); + $self->abort_usage($usage) unless @ARGV; + GetOptions(\%opts, @want, qw/help trackers=s domain=s/) + or $self->abort_usage($usage); + my $config = $self->_readconf(\%opts); + + $self->{config} = {%$config, %opts}; + $self->_verify_config; + $self->abort_usage($usage) if $self->{config}->{help}; + + return $self->{config}; +} + +sub _verify_config { + my MogileFS::Utils $self = shift; + my $conf = $self->{config}; + + while (my ($k, $v) = each %$conf) { + if ($k =~ m/^trackers/) { + my @tr = split /,/, $v; + for (@tr) { + # Client is obnoxious about requiring a port. + if ($_ !~ m/:\d+/) { + $_ = $_ . ':7001'; + } + } + $conf->{$k} = \@tr; + } elsif ($k =~ m/class/) { + # "" means "default". Might have to remove this if people have + # been adding "default" classes, which I don't think is possible? + if ($v eq 'default') { + $conf->{$k} = ''; + } + } + } +} + +# Do we want to be fancier here? +sub abort_usage { + my MogileFS::Utils $self = shift; + my $usage = shift; + print "Usage: $0 $usage\n"; + exit; +} + +sub client { + my MogileFS::Utils $self = shift; + my $c = $self->{config}; + return MogileFS::Client->new(domain => $c->{domain}, + hosts => $c->{trackers}); +} =head1 NAME @@ -14,6 +122,18 @@ L L +L + +L + +L + +L + +L + +L + =head1 SUMMARY Please refer to the documentation for the tools included in this distribution. diff --git a/mogdelete b/mogdelete new file mode 100755 index 0000000..466ad44 --- /dev/null +++ b/mogdelete @@ -0,0 +1,18 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use lib './lib'; +use MogileFS::Utils; + +my $util = MogileFS::Utils->new; +my $usage = "--trackers=host --domain=foo --key='/hello.jpg'"; +my $c = $util->getopts($usage, 'key=s'); + +my $mogc = $util->client; + +$mogc->delete($c->{key}); +if ($mogc->errcode) { + print STDERR "Error deleting file: ", $mogc->errstr, "\n"; +} diff --git a/mogfetch b/mogfetch new file mode 100755 index 0000000..b8dfbab --- /dev/null +++ b/mogfetch @@ -0,0 +1,57 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use lib './lib'; +use MogileFS::Utils; + +my $util = MogileFS::Utils->new; +my $usage = "--trackers=host --domain=foo --key='/hello.jpg' --file='./output'"; +my $c = $util->getopts($usage, qw/key=s file=s/); + +my $mogc = $util->client; + +# Default to noverify, don't hang up the tracker. We'll try all paths. +my @paths = $mogc->get_paths($c->{key}, { noverify => 1 }); +if ($mogc->errcode) { + die "Error fetching paths: " . $mogc->errstr; +} + +die "No paths found or key does not exist" unless @paths; + +my $filename = $c->{file}; +my @resses; +for my $path (@paths) { + next unless $path; # overparanoid? + my $ua = LWP::UserAgent->new; + $ua->timeout(10); + + my $file; + if ($filename eq '-') { + $file = *STDOUT; + } else { + open($file, "> $filename") or die "Could not open " . $filename; + } + + my $writeout = sub { + print $file $_[0]; + }; + my $res = $ua->get($path, ':content_cb' => $writeout, + ':read_size_hint' => 32768); + + if ($res->is_success) { + last; + } else { + # print all the errors to be the most helpful + push(@resses, $res); + next; + } +} + +if (@resses) { + for my $res (@resses) { + print STDERR "Got errors while trying to fetch:\n"; + print STDERR $res->status_line, "\n"; + } +} diff --git a/mogfileinfo b/mogfileinfo new file mode 100755 index 0000000..f7d9f4a --- /dev/null +++ b/mogfileinfo @@ -0,0 +1,35 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use lib './lib'; +use MogileFS::Utils; + +my $util = MogileFS::Utils->new; +my $usage = "--trackers=host --domain=foo --key='/hello.jpg'"; +my $c = $util->getopts($usage, 'key=s'); + +my $mogc = $util->client; + +my $fid = $mogc->file_info($c->{key}); +if ($mogc->errcode) { + die "Error fetching file info: " . $mogc->errstr; +} +die "Key not found: " . $c->{key} unless $fid; + +# Might replace this with just fetching the devids from above... +my @paths = $mogc->get_paths($c->{key}, { noverify => 1, pathcount => 99 }); +if ($mogc->errcode) { + die "Error fetching paths: " . $mogc->errstr; +} +die "No paths found or key does not exist" unless @paths; + +print "- file: ", $c->{key}, "\n"; +for my $item (sort keys %$fid) { + printf(" %8s: %20s\n", $item, $fid->{$item}); +} + +for my $path (@paths) { + print " - ", $path, "\n"; +} diff --git a/moglistfids b/moglistfids new file mode 100755 index 0000000..674893f --- /dev/null +++ b/moglistfids @@ -0,0 +1,47 @@ +#!/usr/bin/perl +# Example for building into a backup program. + +use strict; +use warnings; + +use MogileFS::Admin; +use lib './lib'; +use MogileFS::Utils; + +my $util = MogileFS::Utils->new; +my $usage = "--trackers=host --fromfid=123 --count=5000"; +my $c = $util->getopts($usage, qw/fromfid=i count=i/); + +my $moga = MogileFS::Admin->new(hosts => $c->{trackers}); + +my $fromfid = $c->{fromfid} || 0; +my $count = $c->{count} || 100; + +while ($count) { + # Try to fetch the max, but we will likely get less. + my $fids_chunk = $moga->list_fids($fromfid, $count); + if ($moga->errcode) { + die "Error listing fids: ", $moga->errstr, "\n"; + } + my @fids = sort { $a <=> $b } keys %$fids_chunk; + last unless @fids; + $fromfid = $fids[-1]; + $count -= @fids; + for my $fid (@fids) { + my $file = $fids_chunk->{$fid}; + print "fid ", $fid, "\n"; + for my $key (sort keys %$file) { + my $val = $file->{$key}; + $val = _escape_url_string($val) if $key eq 'dkey'; + print $key, " ", $val, "\n"; + } + print "\n"; + } +} + +sub _escape_url_string { + my $str = shift; + $str =~ s/([^a-zA-Z0-9_\,\-.\/\\\: ])/uc sprintf("%%%02x",ord($1))/eg; + $str =~ tr/ /+/; + return $str; +} diff --git a/moglistkeys b/moglistkeys new file mode 100755 index 0000000..fa8a5d4 --- /dev/null +++ b/moglistkeys @@ -0,0 +1,23 @@ +#!/usr/bin/perl +# TODO: Add ways to limit # of keys displayed + +use strict; +use warnings; + +use lib './lib'; +use MogileFS::Utils; + +my $util = MogileFS::Utils->new; +my $usage = "--trackers=host --domain=foo --key_prefix='bar/'"; +my $c = $util->getopts($usage, 'key_prefix=s'); + +my $mogc = $util->client; + +$mogc->foreach_key(prefix => $c->{key_prefix}, sub { + my $key = shift; + print $key, "\n"; +}); + +if ($mogc->errcode) { + print STDERR "Error listing files: ", $mogc->errstr, "\n"; +} diff --git a/mogupload b/mogupload new file mode 100755 index 0000000..d12af8d --- /dev/null +++ b/mogupload @@ -0,0 +1,43 @@ +#!/usr/bin/perl + +use strict; +use warnings; + +use lib './lib'; +use MogileFS::Utils; + +my $util = MogileFS::Utils->new; +my $usage = "--trackers=host --domain=foo --key='/hello.jpg' --file='./hello.jpg'"; +my $c = $util->getopts($usage, qw/class=s key=s file=s/); + +my $mogc = $util->client; + +my $filename = $c->{file}; + +my $fh; +my $size; +if ($filename eq '-') { + # Can't upload files without a known filesize? + die "STDIN mode not yet supported"; + $fh = *STDIN; +} else { + $size = -s $filename; + die "Could not stat " . $filename unless defined $size; + open($fh, "< $filename") or die "Could not open " . $filename; +} + +my $mf = $mogc->new_file($c->{key}, $c->{class}, $size); +if ($mogc->errcode) { + die "Error opening MogileFS file: " . $mogc->errstr; +} + +my $buf; +while (my $read = read($fh, $buf, 32768)) { + die "error reading file" unless defined $read; + last if $read == 0; + print $mf $buf; +} + +unless ($mf->close) { + die "Error writing file: " . $mogc->errcode . ": " . $mogc->errstr; +}