#!/usr/bin/perl eval 'exec /usr/bin/perl -w -S $0 ${1+"$@"}' if 0; # not running under some shell # # MogileFS storage node daemon # (perlbal front-end) # # (c) 2004, Brad Fitzpatrick, <brad@danga.com> # (c) 2006-2007, Six Apart, Ltd. use strict; use lib 'lib'; use Mogstored::HTTPServer; use IO::Socket::INET; use POSIX qw(WNOHANG); use Perlbal 1.73; use FindBin qw($Bin $RealScript); use Mogstored::HTTPServer::Perlbal; use Mogstored::HTTPServer::Lighttpd; use Mogstored::HTTPServer::None; use Mogstored::HTTPServer::Apache; use Mogstored::HTTPServer::Nginx; use Mogstored::SideChannelListener; use Mogstored::SideChannelClient; my $selfexe = "$Bin/$RealScript"; # State: my %on_death; # pid -> subref (to run when pid dies) my %devnum_to_device; # mogile device number (eg. 'dev1' would be '1') -> os device path (eg. '/dev/rd0') my %osdevnum_to_device; # os device number (fetched via stat(file)[0]) -> os device path (ec. '/dev/rd0') my %iostat_listeners; # fd => SideChannel client: clients interested in iostat data. my $iostat_available = 1; # bool: iostat working. assume working to start. my ($iostat_pipe_r, $iostat_pipe_w); # pipes for talking to iostat process # Config: my $opt_skipconfig; my $opt_daemonize; my $opt_config; my $opt_iostat = 1; # default to on now my $max_conns = 10000; my $http_listen = "0.0.0.0:7500"; my $mgmt_listen = "0.0.0.0:7501"; my $docroot = "/var/mogdata"; my $default_config = "/etc/mogilefs/mogstored.conf"; my $server = $ENV{MOGSTORED_SERVER_TYPE} || "perlbal"; my $serverbin = ""; my $pidfile = undef; # Rename binary in process list to make init scripts saner $0 = "mogstored"; my %config_opts = ( 'iostat' => \$opt_iostat, 's|skipconfig' => \$opt_skipconfig, 'daemonize|d' => \$opt_daemonize, 'config=s' => \$opt_config, 'httplisten=s' => \$http_listen, 'mgmtlisten=s' => \$mgmt_listen, 'docroot=s' => \$docroot, 'maxconns=i' => \$max_conns, 'server=s' => \$server, 'serverbin=s' => \$serverbin, 'pidfile=s' => \$pidfile, ); usage() unless Getopt::Long::GetOptions(%config_opts); die "Unknown server type. Valid options: --server={perlbal,lighttpd,apache,nginx,none}" unless $server =~ /^perlbal|lighttpd|apache|nginx|none$/; $opt_config = $default_config if ! $opt_config && -e $default_config; load_config_file($opt_config => \%config_opts) if $opt_config && !$opt_skipconfig; # initialize basic required Perlbal machinery, for any HTTP server my $perlbal_init = qq{ CREATE SERVICE mogstored SET role = web_server SET docroot = $docroot # don't listen... this is just a stub service. CREATE SERVICE mgmt SET role = management ENABLE mgmt }; $perlbal_init .= "\nSERVER pidfile = $pidfile" if defined($pidfile); Perlbal::run_manage_commands($perlbal_init , sub { print STDERR "$_[0]\n"; }); # start HTTP server my $httpsrv_class = "Mogstored::HTTPServer::" . ucfirst($server); my $httpsrv = $httpsrv_class->new( listen => $http_listen, docroot => $docroot, maxconns => $max_conns, bin => $serverbin, ); # Configure Perlbal HTTP listener after daemonization since it can create a # kqueue on *BSD. kqueue descriptors are automatically invalidated on fork(), # making them unusable after daemonize. For non-Perlbal, starting the # server before daemonization improves error reporting as daemonization # redirects stdout/stderr to /dev/null. $httpsrv->start if $server ne "perlbal"; if ($opt_daemonize) { $httpsrv->pre_daemonize; Perlbal::daemonize(); } else { print "Running.\n"; } # It is now safe for Perlbal to create a kqueue $httpsrv->start if $server eq "perlbal"; $httpsrv->post_daemonize; # kill our children processes on exit: my $parent_pid = $$; $SIG{TERM} = $SIG{INT} = sub { return unless $$ == $parent_pid; # don't let this be inherited kill 'TERM', grep { $_ } keys %on_death; POSIX::_exit(0); }; setup_iostat_pipes(); start_disk_usage_process(); start_iostat_process() if $opt_iostat; harvest_dead_children(); # every 2 seconds, it reschedules itself setup_sidechannel_listener(); # now start the main loop Perlbal::run(); ############################################################################ sub usage { my $note = shift; $note = $note ? "NOTE: $note\n\n" : ""; die "${note}Usage: mogstored [OPTS] OPTS: --daemonize -d Daemonize --config=<file> Set config file (default is /etc/mogilefs/mogstored.conf) --httplisten=<ip:port> IP/Port HTTP server listens on --mgmtlisten=<ip:port> IP/Port management/sidechannel listens on --docroot=<path> Docroot above device mount points. Defaults to /var/mogdata --maxconns=<number> Number of simultaneous clients to serve. Default 10000 "; } # accessor for SideChannelClient: sub Mogstored::iostat_available { return $iostat_available; } sub load_config_file { my ($conffile, $opts) = @_; # parse the mogstored config file, which is just lines of comments and # "key = value" lines, where keys are just the same as command line # options. die "Config file $opt_config doesn't exist.\n" unless -e $conffile; open my $fh, $conffile or die "Couldn't open config file for reading: $!"; while (<$fh>) { s/\#.*//; next unless /\S/; if (/SERVER max_connect/i || /CREATE SERVICE/i) { usage("Your $opt_config file is the old syntax. The new format is simply lines of <key> = <value> where keys are the same as mogstored's command line options."); } die "Unknown config syntax: $_\n" unless /^\s*(\w+)\s*=\s*(.+?)\s*$/; my ($key, $val) = ($1, $2); my $dest; foreach my $ck (keys %$opts) { next unless $ck =~ /^$key\b/; $dest = $opts->{$ck}; } die "Unknown config setting: $key\n" unless $dest; $$dest = $val; } } sub harvest_dead_children { my $dead = waitpid(-1, WNOHANG); if ($dead > 0) { my $code = delete $on_death{$dead}; $code->() if $code; } Danga::Socket->AddTimer(2, \&harvest_dead_children); } sub Mogstored::on_pid_death { my ($class, $pid, $code) = @_; $on_death{$pid} = $code; } # returns $pid of child, if parent, else runs child. sub start_disk_usage_process { my $child = fork; unless (defined $child) { Perlbal::log('crit', "Fork error creating disk usage tracking process"); return undef; } # if we're the parent. if ($child) { $on_death{$child} = sub { start_disk_usage_process(); # start a new one }; return $child; } require Mogstored::ChildProcess::DiskUsage; my $class = "Mogstored::ChildProcess::DiskUsage"; $class->pre_exec_init; $class->exec; } sub Mogstored::iostat_subscribe { my ($class, $sock) = @_; $iostat_listeners{fileno($sock->sock)} = $sock; } sub Mogstored::iostat_unsubscribe { my ($class, $sock) = @_; my $fdno = fileno($sock->sock); return unless defined $fdno; delete $iostat_listeners{$fdno}; } # to be honest, I have no clue why this exists. I just had to move it # around for multi-server refactoring, and I felt better not # understanding it but preserving than killing it. in particular, why # is this "graceful"? (gets called from SideChannelClient's # die_gracefully) sub Mogstored::on_sidechannel_die_gracefully { if ($$ == $parent_pid) { kill 'TERM', grep { $_ } keys %on_death; } } sub setup_sidechannel_listener { Mogstored::SideChannelListener->new($mgmt_listen); } my $iostat_read_buf = ""; sub setup_iostat_pipes { pipe ($iostat_pipe_r, $iostat_pipe_w); IO::Handle::blocking($iostat_pipe_r, 0); IO::Handle::blocking($iostat_pipe_w, 0); Danga::Socket->AddOtherFds(fileno($iostat_pipe_r), sub { read_from_iostat_child(); }); } sub start_iostat_process { my $pid = fork; unless (defined $pid) { warn "Fork for iostat failed: $!"; return; } if ($pid) { # Parent $on_death{$pid} = sub { # Try a final read from data on the socket. # Note that this doesn't internally loop... so it might miss data. read_from_iostat_child(); # Kill any buffer so partial reads don't hurt us later. drain_iostat_child_pipe(); start_iostat_process(); }; return; } require Mogstored::ChildProcess::IOStat; my $class = "Mogstored::ChildProcess::IOStat"; $class->pre_exec_init; $class->exec; } sub Mogstored::get_iostat_writer_pipe { $iostat_pipe_w } sub drain_iostat_child_pipe { my $data; while (1) { last unless sysread($iostat_pipe_r, $data, 10240) > 0; } $iostat_read_buf = ''; } # (runs in parent event-loop process) sub read_from_iostat_child { my $data; my $rv = sysread($iostat_pipe_r, $data, 10240); return unless $rv && $rv > 0; $iostat_read_buf .= $data; # only write complete lines to sockets (in case for some reason we get # a partial read and child process dies...) while ($iostat_read_buf =~ s/(.+)\r?\n//) { my $line = $1; foreach my $out_sock (values %iostat_listeners) { # where $line will be like "dev53\t53.23" or a "." to signal end of a group of devices. # stop writing to the socket if the listener isn't picking it up. next if $out_sock->{write_buf_size}; $out_sock->write("$line\n"); } } } # Local Variables: # mode: perl # c-basic-indent: 4 # indent-tabs-mode: nil # End: __END__ =head1 NAME mogstored -- MogileFS storage daemon =head1 USAGE This is the MogileFS storage daemon, which is just an HTTP server that supports PUT, DELETE, etc. It's actually a wrapper around L<Perlbal>, doing all the proper Perlbal config for you. In addition, it monitors disk usage, I/O activity, etc, which are checked from the L<MogileFS tracker|mogilefsd>. =head1 AUTHORS Brad Fitzpatrick E<lt>brad@danga.comE<gt> Mark Smith E<lt>junior@danga.comE<gt> Jonathan Steinert E<lt>jsteinert@sixapart.comE<gt> =head1 ENVIRONMENT =over 4 =item PERLBAL_XS_HEADERS If defined and 0, Perlbal::XS::HTTPHeaders will not be used, if present. Otherwise, it will be enabled by default, if installed and loadable. =back =head1 COPYRIGHT Copyright 2004, Danga Interactive Copyright 2005-2006, Six Apart Ltd. =head1 LICENSE Same terms as Perl itself. Artistic/GPLv2, at your choosing. =head1 SEE ALSO L<MogileFS::Overview> -- high level overview of MogileFS L<mogilefsd> -- MogileFS daemon L<http://danga.com/mogilefs/>