| [vhffs-dev] [1256] Added the cron scheduler ! (It works but I don't know why, you have been warned :-) |
[ Thread Index |
Date Index
| More vhffs.org/vhffs-dev Archives
]
Revision: 1256
Author: gradator
Date: 2008-09-25 18:19:57 +0200 (Thu, 25 Sep 2008)
Log Message:
-----------
Added the cron scheduler ! (It works but I don't know why, you have been warned :-)
Modified Paths:
--------------
trunk/vhffs-backend/conf/vhffs.conf.dist.in
trunk/vhffs-robots/Makefile.am
Added Paths:
-----------
trunk/vhffs-robots/src/cron_scheduler.pl
Modified: trunk/vhffs-backend/conf/vhffs.conf.dist.in
===================================================================
--- trunk/vhffs-backend/conf/vhffs.conf.dist.in 2008-09-22 07:44:03 UTC (rev 1255)
+++ trunk/vhffs-backend/conf/vhffs.conf.dist.in 2008-09-25 16:19:57 UTC (rev 1256)
@@ -475,6 +475,26 @@
# Maximum execution time (in seconds)
max_execution_time = 7200
+
+ # System limits
+ <limits>
+ # Default configuration is for Linux 2.6
+ # You can use the following command to fetch the available limits on your system
+ # $ perl -e 'use BSD::Resource; print join( "\n" , keys %{get_rlimits()} )."\n"';
+ #
+ # resource = soft hard
+ RLIMIT_CPU = 900 900
+ RLIMIT_FSIZE = RLIM_INFINITY RLIM_INFINITY
+ RLIMIT_DATA = 104857600 104857600
+ RLIMIT_STACK = 4194304 4194304
+ RLIMIT_RSS = 33554432 33554432
+ RLIMIT_CORE = 0 0
+ RLIMIT_MEMLOCK = 0 0
+ RLIMIT_NPROC = 100 100
+ RLIMIT_NOFILE = 512 512
+ RLIMIT_LOCKS = 1024 1024
+ RLIMIT_AS = 67108864 67108864
+ </limits>
</cron>
</services>
Modified: trunk/vhffs-robots/Makefile.am
===================================================================
--- trunk/vhffs-robots/Makefile.am 2008-09-22 07:44:03 UTC (rev 1255)
+++ trunk/vhffs-robots/Makefile.am 2008-09-25 16:19:57 UTC (rev 1256)
@@ -71,7 +71,8 @@
src/web_delete.pl \
src/web_stats.pl \
src/cron_create.pl \
- src/cron_delete.pl
+ src/cron_delete.pl \
+ src/cron_scheduler.pl
# Define the substitution we need to point perl script at correct location
do_sed = $(SED) --in-place \
Added: trunk/vhffs-robots/src/cron_scheduler.pl
===================================================================
--- trunk/vhffs-robots/src/cron_scheduler.pl (rev 0)
+++ trunk/vhffs-robots/src/cron_scheduler.pl 2008-09-25 16:19:57 UTC (rev 1256)
@@ -0,0 +1,322 @@
+#!/usr/bin/perl -w
+# Copyright (c) vhffs project and its contributors
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in
+# the documentation and/or other materials provided with the
+# distribution.
+#3. Neither the name of vhffs nor the names of its contributors
+# may be used to endorse or promote products derived from this
+# software without specific prior written permission.
+#
+#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+#FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+#COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+#INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+#BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+#LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+#CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+use strict;
+use utf8;
+use POSIX;
+use Fcntl ':mode';
+use IO::Handle;
+use IO::Select;
+use BSD::Resource;
+#use Data::Dumper;
+
+use constant
+{
+ STATUS_CREATED => 0,
+ STATUS_RUNNING => 1,
+ STATUS_KILLED => 2,
+};
+
+use lib '%VHFFS_LIB_DIR%';
+use Vhffs::Main;
+use Vhffs::Services::Cron;
+use Vhffs::Robots::Cron;
+use Vhffs::Functions;
+
+#select(STDOUT);
+#$| = 1;
+
+my $vhffs = init Vhffs::Main;
+exit 1 unless defined $vhffs;
+
+my $cronconf = $vhffs->get_config->get_service('cron');
+my $chroot = $cronconf->{'chroot'};
+my $limits = $cronconf->{'limits'};
+my $rlimits = BSD::Resource::get_rlimits();
+my $mailcronfrom = $cronconf->{'mail_from'};
+my $maxexectime = $cronconf->{'max_execution_time'};
+
+my %jobs;
+my %fd2jobs;
+my $read_set = new IO::Select();
+my $prevrun = 0;
+
+while(1) {
+
+ if( time - $prevrun > int(rand 10)+5 ) {
+
+ my $crons = Vhffs::Robots::Cron::get_runnable_jobs( $vhffs );
+ foreach my $c ( @{$crons} ) {
+ if( exists ${%jobs}{ $c->get_cron_id } ) {
+ print scalar localtime().' ! '.$c->get_cron_id.' '.$c->get_cronpath."\n";
+ }
+ else {
+ new_job( $c );
+ }
+ }
+
+ $prevrun = time;
+ }
+
+ foreach ( keys %jobs ) {
+ my $job = ${%jobs}{$_};
+ if( $job->{'status'} == STATUS_CREATED ) {
+ run_job( $job ) if( time > $job->{'runat'} );
+ }
+ elsif ( $job->{'status'} == STATUS_RUNNING ) {
+ if( defined $maxexectime && $maxexectime > 0 && time - $job->{'startedat'} > $maxexectime ) {
+ kill 9, $job->{'pid'};
+ $job->{'status'} = STATUS_KILLED;
+ }
+ }
+ }
+
+ my ($rh_set) = IO::Select->select($read_set, undef, undef, 1);
+ foreach my $rh (@$rh_set) {
+ my $job = ${%fd2jobs}{$rh->fileno};
+ my $cron = $job->{'cron'};
+ my $buf = <$rh>;
+ if($buf) {
+ if( $job->{'inheaders'} ) {
+ if( (my $data) = ( $buf =~ /^ENV:(.*)$/ ) ) {
+ $job->{'env'} .= $data."\n";
+ }
+ elsif( $buf =~ /^$/ ) {
+ $job->{'inheaders'} = 0;
+ }
+ }
+ else {
+ $job->{'output'} .= $buf;
+ }
+ }
+ else {
+ # End of this job
+ waitpid( $job->{'pid'} , 0 );
+ my $returnvalue = $? >> 8;
+
+ $job->{'output'} .= "\n------KILLED------\n" if( $job->{'status'} == STATUS_KILLED );
+
+ if( defined $job->{'output'} && $job->{'output'} ne '' ) {
+
+ my $body = 'Exit value: '. $returnvalue."\n\n";
+ $body .= 'WARNING: This process was killed because it were still running after more than '.$maxexectime.' seconds'."\n\n" if( $job->{'status'} == STATUS_KILLED );
+ $body .= "\n--- Environment ---\n\n".$job->{'env'}."\n\n--- Stdout and stderr output ---\n\n".$job->{'output'}."\n";
+ sendmail_cron( $cron , $body );
+ }
+
+ $cron->quick_set_nextrundate( time() + $cron->get_interval );
+ $cron->quick_set_lastrun( $job->{'createdat'} , $returnvalue );
+ $cron->quick_dec_running();
+ destroy_job( $job );
+ }
+ }
+
+ #print 'Jobs: '.join(' ', sort { $a <=> $b } keys %jobs)."\n";
+ #print 'Fd2Jobs: '.join(' ', sort { $a <=> $b } keys %fd2jobs)."\n";
+}
+
+exit 0;
+
+
+sub new_job
+{
+ my $cron = shift;
+ my $cron_id = $cron->get_cron_id;
+ my $fullpath = $cron->get_cronpath;
+ $fullpath = $chroot.'/'.$fullpath if defined $chroot;
+
+ unless( -f $fullpath && -x $fullpath ) {
+ my $body = "CRITICAL: The path is neither a regular file nor an executable\n";
+ sendmail_cron( $cron , $body );
+ $cron->quick_set_nextrundate( time() + 600 );
+ return undef;
+ }
+
+ my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($fullpath);
+
+ unless( $gid == $cron->get_owner_gid ) {
+ my $body = "CRITICAL: GID of file don't match the owner GID of this object in the VHFFS database\n";
+ sendmail_cron( $cron , $body );
+ $cron->quick_set_nextrundate( time() + 600 );
+ return undef;
+ }
+
+ unless( $uid == $cron->get_owner_uid ) {
+ my $body = "CRITICAL: UID of file don't match the owner UID of this object in the VHFFS database\n";
+ sendmail_cron( $cron , $body );
+ $cron->quick_set_nextrundate( time() + 600 );
+ return undef;
+ }
+
+ if( $mode & S_IWOTH ) {
+ my $body = "CRITICAL: File is writeable by others, I am not going to execute that\n";
+ sendmail_cron( $cron , $body );
+ $cron->quick_set_nextrundate( time() + 600 );
+ return undef;
+ }
+
+ ${%jobs}{$cron_id}{'cron'} = $cron;
+ ${%jobs}{$cron_id}{'fullpath'} = $fullpath;
+ ${%jobs}{$cron_id}{'pid'} = undef;
+ ${%jobs}{$cron_id}{'pipe'} = undef;
+ ${%jobs}{$cron_id}{'output'} = undef;
+ ${%jobs}{$cron_id}{'env'} = '';
+ ${%jobs}{$cron_id}{'inheaders'} = 1;
+ ${%jobs}{$cron_id}{'createdat'} = time;
+ ${%jobs}{$cron_id}{'runat'} = ${%jobs}{$cron_id}{'createdat'} + int(rand 4) +2;
+ ${%jobs}{$cron_id}{'startedat'} = undef;
+ ${%jobs}{$cron_id}{'status'} = STATUS_CREATED;
+ $cron->quick_inc_running();
+
+ print scalar localtime().' + '.$cron_id.' '.$cron->get_cronpath."\n";
+ return ${%jobs}{$cron_id};
+}
+
+
+sub run_job
+{
+ my $job = shift;
+ my $cron = $job->{'cron'};
+ my $cron_id = $cron->get_cron_id;
+
+ my $running = $cron->quick_get_running();
+ return 1 unless defined $running;
+ if( $running > 1 ) {
+ print scalar localtime().' x '.$cron_id.' '.$cron->get_cronpath."\n";
+ $cron->quick_dec_running();
+ destroy_job( $job );
+ return 1;
+ }
+ elsif( $running < 0 ) {
+ #this should not happen, set running to 0 and abort
+ $cron->quick_set_running( 0 );
+ destroy_job( $job );
+ return 1;
+ }
+ print scalar localtime().' = '.$cron_id.' '.$cron->get_cronpath."\n";
+
+ my $par = new IO::Handle->new();
+ my $son = new IO::Handle->new();
+
+ unless( pipe( $par, $son ) ) {
+ print "pipe() failed\n";
+ return 1;
+ }
+
+ my $pid = fork();
+ unless( defined $pid ) {
+ print "fork() failed\n";
+ close $par;
+ close $son;
+ return 1;
+ }
+
+ if ($pid) {
+
+ # I am the parent
+ close $son;
+
+ ${%jobs}{$cron_id}{'pid'} = $pid;
+ ${%jobs}{$cron_id}{'pipe'} = $par;
+ ${%jobs}{$cron_id}{'startedat'} = time;
+ ${%jobs}{$cron_id}{'status'} = STATUS_RUNNING;
+ ${%fd2jobs}{$par->fileno} = ${%jobs}{$cron_id};
+ $read_set->add($par);
+ return 0;
+
+ }
+ elsif($pid == 0) {
+
+ # I am the child
+ close $par;
+ $son->autoflush(1);
+
+ close STDOUT;
+ open STDOUT, '>&'.$son->fileno;
+ STDOUT->autoflush(1);
+
+ close STDERR;
+ open STDERR, '>&'.$son->fileno;
+ STDERR->autoflush(1);
+
+ foreach (sort keys(%ENV)) {
+ print $son 'ENV:'.$_.'='.$ENV{$_}."\n";
+ }
+ print $son "\n";
+
+ chroot $chroot if defined $chroot;
+
+ foreach my $resource ( keys %{$limits} ) {
+ my ( $soft , $hard ) = ( $limits->{$resource} =~ /^\s*([\d\w]+)\s+([\d\w]+)\s*$/ );
+
+ $resource = $rlimits->{$resource};
+ $soft = eval $soft; # we get a string, we have to convert it to the
+ $hard = eval $hard; # integer constant if needed ( RLIM_INFINITY and such )
+
+ BSD::Resource::setrlimit( $resource , $soft, $hard ) if defined $resource and defined $soft and defined $hard;
+ }
+
+ POSIX::setgid $cron->get_owner_gid;
+ POSIX::setuid $cron->get_owner_uid;
+ return 1 unless( getuid() == $cron->get_owner_uid && getgid() == $cron->get_owner_gid );
+
+ exec $cron->get_cronpath;
+ exit 1;
+ }
+}
+
+
+sub destroy_job
+{
+ my $job = shift;
+ my $cron = $job->{'cron'};
+ my $cron_id = $cron->get_cron_id;
+
+ print scalar localtime().' - '.$cron->get_cron_id.' '.$cron->get_cronpath."\n";
+ delete ${%jobs}{$cron_id};
+
+ my $pipe = $job->{'pipe'};
+ if( defined $pipe ) {
+ delete ${%fd2jobs}{ $pipe->fileno };
+ $read_set->remove($pipe);
+ close($pipe);
+ }
+}
+
+
+sub sendmail_cron
+{
+ my $cron = shift;
+ my $body = shift;
+ return undef unless( defined $cron->get_reportmail && $cron->get_reportmail ne '' );
+ my $subject = 'VHFFS Cron '.$cron->get_cronpath;
+ Vhffs::Functions::send_mail( $mailcronfrom , $cron->get_reportmail , $vhffs->get_config->get_mailtag , $subject , $body );
+}
Property changes on: trunk/vhffs-robots/src/cron_scheduler.pl
___________________________________________________________________
Name: svn:executable
+ *