#!/usr/bin/perl
use warnings;
use strict;
my($app_svnid) = '$HeadURL$ $LastChangedRevision$';  ##  no critic (RequireInterpolationOfMetachars)
use lib substr(`ade-config ade_share_prefix`,0,-1) . '/include';  ## no critic (ProhibitBacktickOperators)
use ADE;
use lib substr(`js-config js_share_prefix`,0,-1) . '/include';  ## no critic (ProhibitBacktickOperators)
use JS;
#  see perlipc(1) for explanation of why this is needed
use POSIX ':sys_wait_h';
use Data::Dumper;
use experimental 'smartmatch';
use IPC::SysV qw(IPC_PRIVATE IPC_RMID IPC_CREAT S_IRUSR S_IWUSR);

my(%jsd_defined_errors) = (
    'jsd_err_misc' => { fmt => '%s' },
);

#  Options
my(%opt_resources);
my($opt_schedule_flag);
my($dbinit_file) = substr(`js-config js_share_prefix`,0,-1) . '/sql/jsd.sql';  ## no critic (ProhibitBacktickOperators)

#  Other globals
my(%exit_code_of_recently_reaped_pids, $js_shell);
my($sqlite_max_int) = 9223372036854775807;

sub jsd  ## no critic (ProhibitExcessComplexity)
{
    my($errstack_ref) = @_;
    my($rc, $db_file, $pid, $dbh, $sql_statement, $i, $quit_flag, @possible_job_ids, $possible_jobs_count);
    my($lock_file, $server_handle, $server_name, @sql_value);

    #  Set ADE options
    ADE::ade_err_registerdefderrs(\%jsd_defined_errors);

    #  Defaults for options
    #  (identical line in jsd, jss, jsq).
    $opt_schedule_flag = 1;

    #  Register options
    if (($rc=ADE::ade_opt_register($errstack_ref, 'r:s', 'resources:s,suspend', 'main::jsd_opt_handler_%s')) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    #  Register handler functions
    if (($rc=ADE::ade_msg_register($errstack_ref, \&jsd_usage, \&jsd_version, \&jsd_listpaths)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    #  Process options
    if (($rc=ADE::ade_opt_process($errstack_ref)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    #  Process arguments
    ADE::ade_msg_usage($errstack_ref, \&app_usage, 1) if ($#ARGV+1 != 0);

    #  Sanity checks and derivations
    if (($rc=JS::get_server_name($errstack_ref, \$server_name)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    $lock_file = "$ADE::ADE_TMP_DIR/jsd-$server_name.lock";
    $db_file = "$ADE::ADE_TMP_DIR/jsd-$server_name.sqlite";
    if (scalar keys %opt_resources == 0) {
        ADE::ade_err_error($errstack_ref, 'jsd_err_misc', 'no resources specified');
        return($ADE::ADE_ERR_FAIL);
    }
    foreach my $resource_id (keys %opt_resources) {
       if ($resource_id !~ /^[a-z][a-z0-9]*$/) {
           ADE::ade_err_error($errstack_ref, 'jsd_err_misc', "$resource_id: invalid resource name (hint: it should be word-like)");
           return($ADE::ADE_ERR_FAIL);
       }
       ADE::ade_err_debug($errstack_ref, 10, "jsd: resource_id=$resource_id, quantity_total=$opt_resources{$resource_id}{'quantity_total'}, affinous=$opt_resources{$resource_id}{'affinous'}");
       if ((not defined $opt_resources{$resource_id}{'quantity_total'}) or ($opt_resources{$resource_id}{'quantity_total'} !~ /^(?:0|[1-9]\d*)$/)) {
           ADE::ade_err_error($errstack_ref, 'jsd_err_misc', (defined $opt_resources{$resource_id}{'quantity_total'} ? $opt_resources{$resource_id}{'quantity_total'} : '') . ': invalid quantity (hint: should be an integer)');
           return($ADE::ADE_ERR_FAIL);
       }
       if (not defined $opt_resources{$resource_id}{'affinous'}) {
           $opt_resources{$resource_id}{'affinous'} = 'false';
       } elsif ($opt_resources{$resource_id}{'affinous'} ~~ ['t', 'true', '1', 'yes', 'y']) {
           $opt_resources{$resource_id}{'affinous'} = 'true';
       } elsif ($opt_resources{$resource_id}{'affinous'} ~~ ['f', 'false', '0', 'no', 'n']) {
           $opt_resources{$resource_id}{'affinous'} = 'false';
       } else {
           ADE::ade_err_error($errstack_ref, 'jsd_err_misc', "$opt_resources{$resource_id}{'affinous'}: invalid affinousness (hint: should be empty or a logical)");
           return($ADE::ADE_ERR_FAIL);
       }
       if ($opt_resources{$resource_id}{'quantity_total'} eq '0') {
           #  Don't allow infinite resources (indicated by quanity zero) to be affinous because
           #  the list of resource pool sets will be so big (i.e. 2^$sqlite_max_int). If we used
           #  a value smaller than $sqlite_max_int to be infinity, then perhaps we could allow it.
           if ($opt_resources{$resource_id}{'affinous'} eq 'true') {
               ADE::ade_err_error($errstack_ref, 'jsd_err_misc', "$opt_resources{$resource_id}: infinite resources cannot be affinous");
               return($ADE::ADE_ERR_FAIL);
           }
           ADE::ade_err_debug($errstack_ref, 10, "jsd: resource_id=$resource_id, quantity is zero; making infinite ...");
           $opt_resources{$resource_id}{'quantity_total'} = sprintf '%d', $sqlite_max_int;
       }
    }
    #  Set $js_shell
    if (defined $ENV{'JS_SHELL'} and not -x $ENV{'JS_SHELL'}) {
        ADE::ade_err_error($errstack_ref, 'jsd_err_misc', "JS_SHELL: invalid shell (hint: '$ENV{'JS_SHELL'}' was not found or is not executable");
        return($ADE::ADE_ERR_FAIL);
    } elsif (defined $ENV{'JS_SHELL'}) {
        $js_shell = $ENV{'JS_SHELL'};
    } else {
        $js_shell = '/bin/sh';
    }

    #  Lock
    if (($rc=ADE::ade_lck_lock_simple($errstack_ref, $lock_file)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    #  Define and initialise database
    if (-f $db_file) {
        ADE::ade_err_warning($errstack_ref, 'jsd_err_misc', "$db_file: removing old database ...");
        if (not unlink $db_file) {
            ADE::ade_err_internal($errstack_ref, 'jsd: failed to remove old database');
        }
    }
    ADE::ade_tmp_registerfile($errstack_ref, $db_file);
    ADE::ade_err_debug($errstack_ref, 10, "jsd: db_file=$db_file");
    #  Pass one arg to system() in order that it uses the shell, which is needed to use redirection,
    #  which is needed to get sqlite not to output stuff about loading ~/.sqliterc.
    if (system ("sqlite3 $db_file \".read $dbinit_file\" < /dev/null") != 0) {
        ADE::ade_err_error($errstack_ref, 'jsd_err_misc', 'failed to initialise database');
        return($ADE::ADE_ERR_FAIL);
    }
    #  Now connect to to the database for real work.
    $dbh = DBI->connect("dbi:SQLite:dbname=$db_file",'','');
    if (not defined $dbh) {
        #  This was $DBI::errstr, but perlcritic complained.
        ADE::ade_err_error($errstack_ref, 'jsd_err_misc', $dbh->errstr);
        return($ADE::ADE_ERR_FAIL);
    }
    $dbh->do('PRAGMA foreign_keys = ON');
    $dbh->sqlite_busy_timeout(10000);

    #  Populate resources table
    ADE::ade_err_debug($errstack_ref, 10, 'jsd: populating resources table ...');
    $sql_statement = 'INSERT INTO resources (resource_id, quantity_total, affinous) VALUES (?, ?, ?);';
    foreach my $resource_id (keys %opt_resources) {
        @sql_value     = ($resource_id, $opt_resources{$resource_id}{'quantity_total'}, $opt_resources{$resource_id}{'affinous'});
        if (($rc=sql_execute_qm($errstack_ref, $dbh, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
            return($rc);
        }
    }
    undef %opt_resources;                 #  let's be clear; this is no longer needed.
    #  Expand resources into resource pools. We assign arbitrary priorities to the pools. This
    #  will be used to help select which of the a resource pool sets that satisfies a job's
    #  requirements will be preferred.
    #
    #  Note that we use i+1 instead of i as the resource_pool_priority. If the pools have
    #  priority:
    #
    #      pool#0: 0
    #      pool#1: 1
    #
    #  then pool *set* #3 (consisting of pool#0 *and* pool#1) has the same priority as pool set #2.
    #  More specifically, if a job wants only one CPU, then pool #5 - *which consists of two CPUs*
    #  is equally attractive to pool #4, which consists of one CPU. Which of these sqlite chooses
    #  to show is not certain. For this reason we make the priority i+1.
    #
    $sql_statement = "INSERT INTO resource_pools (resource_id, resource_pool_id, resource_pool_priority, quantity_total)\n" .
                     "    SELECT resource_id, i AS resource_pool_id, i+1 AS resource_pool_priority, 1 AS quantity_total\n" .
                     "        FROM resources, i1000\n" .
                     "        WHERE affinous = 'true' and i < quantity_total\n" .
                     "    UNION SELECT resource_id, 0 AS resource_pool_id, 0 as resource_pool_priority, quantity_total\n" .
                     "        FROM resources\n" .
                     "        WHERE affinous = 'false';\n";
    if (($rc=ADE::ade_sql_execute($errstack_ref, $dbh, $sql_statement)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    #  Initialise the UNIX socket server
    JS::init_server($errstack_ref, $server_name, \$server_handle);

    #  Main loop
    local $SIG{'CHLD'} = \&sighandler_chld;
    $quit_flag = 0;
    while (1) {
        ADE::ade_err_debug($errstack_ref, 5, 'jsd: top of loop');

        #  Top priority: release job slots occupied by completed jobs!
        if (($rc=process_recently_reaped_pids($errstack_ref, $dbh)) != $ADE::ADE_ERR_OK) {
            return($rc);
        }

        #  Next priority start as many new jobs as possible but always going back
        #  and checking if any slots are releasable.
        if ($opt_schedule_flag) {
            ADE::ade_err_debug($errstack_ref, 5, 'jsd: checking for runnable jobs ...');
            if (($rc=get_job_ids($errstack_ref, $dbh, 'possible', \@possible_job_ids)) != $ADE::ADE_ERR_OK) {
                return($rc);
            }
            $possible_jobs_count = $#possible_job_ids+1;
            ADE::ade_err_debug($errstack_ref, 5, "jsd: $possible_jobs_count jobs are runnable ...");
            if ($possible_jobs_count > 0) {
                ADE::ade_err_debug($errstack_ref, 5, "jsd: starting job $possible_job_ids[0] ...");
                if (($rc=start_job_id($errstack_ref, $dbh, $possible_job_ids[0])) != $ADE::ADE_ERR_OK) {
                    return($rc);
                }
                next;
            }
        }

        #  Lowest priority: deal with incoming messages.
        ADE::ade_err_debug($errstack_ref, 5, 'jsd: waiting for a message ...');
        if (($rc=JS::read_message_str_and_send_reply_str($errstack_ref, $server_handle, \&process_message, [ $dbh, \$quit_flag, \$opt_schedule_flag ])) != $ADE::ADE_ERR_OK) {
            return($rc);
        }
        #  If the call to process_message() resulting in quit_flag being set then break out of loop.
        if ($quit_flag) {
            ADE::ade_err_debug($errstack_ref, 5, 'jsd: process_message() set quit_flag; breaking out of main loop ...');
            last;
        }
    }
    local $SIG{'CHLD'} = 'DEFAULT';

    #  Tidy up
    sleep 1;   #  give jsc time to read the reply we've sent to its instruction to quit.
    #msgctl($js_msgqueue_id, IPC_RMID, 0);
    $dbh->disconnect;
    if (not unlink $db_file) {
        ADE::ade_err_internal($errstack_ref, 'jsd: failed to remove old database');
    }
    ADE::ade_tmp_deregisterfile($errstack_ref, $db_file);
    if (($rc=ADE::ade_lck_unlock_simple($errstack_ref, $lock_file)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    return($ADE::ADE_ERR_OK);
}

sub jsd_opt_handler_r
{
    my($errstack_ref, $resource_specs_string) = @_;
    my($resource_id, $quantity_total, $affinous);

    foreach my $resource_spec (split /,/, $resource_specs_string) {
        ($resource_id, $quantity_total, $affinous) = split /:/, $resource_spec;
        $opt_resources{$resource_id}{'quantity_total'} = $quantity_total;
        $opt_resources{$resource_id}{'affinous'} = $affinous ? $affinous : 'n';
    }

    return $ADE::ADE_ERR_OK;
}

sub jsd_opt_handler_resources         ## no critic (Subroutines::RequireArgUnpacking)
{
    return(jsd_opt_handler_r(@_));
}

sub jsd_opt_handler_suspend
{
    my($errstack_ref) = @_;

    $opt_schedule_flag = 0;

    return $ADE::ADE_ERR_OK;
}

sub jsd_version
{
    my($errstack_ref, $version_text_ref) = @_;

    return(ADE::ade_smf_extractversionfromsvnstring($errstack_ref, $app_svnid, $version_text_ref));
}

sub jsd_listpaths
{
    my($errstack_ref, $pathlist_text_ref) = @_;
    my($rc);

    #${$pathlist_text_ref} = '';

    return($ADE::ADE_ERR_OK);
}

sub jsd_usage
{
    my($errstack_ref, $usage_text_ref, $passno) = @_;

    if ($passno == 1) {
        ${$usage_text_ref} = "\n";
    } elsif ($passno == 2) {
        ${$usage_text_ref} = "                     --suspend               start with scheduler suspended\n" .
                             "         -r ...    | --resources=...         set system resources\n";
    }

    return($ADE::ADE_ERR_OK);
}

sub process_recently_reaped_pids
{
    my($errstack_ref, $dbh) = @_;
    my($rc, $job_id, $sql_statement, @sql_value);

    foreach my $pid (keys %exit_code_of_recently_reaped_pids) {
        ADE::ade_err_debug($errstack_ref, 5, "process_recently_reaped_pids: processing recently reaped pid $pid ...");
        #  Translate exited PID to job number.
        if (($rc=get_job_id_by_pid($errstack_ref, $dbh, $pid, \$job_id)) != $ADE::ADE_ERR_OK) {
            return($rc);
        }

        $sql_statement = 'BEGIN EXCLUSIVE TRANSACTION;';
        if (($rc=ADE::ade_sql_execute($errstack_ref, $dbh, $sql_statement)) != $ADE::ADE_ERR_OK) {
            return($rc);
        }

        # Release the running demands (i.e. delete all entries related to this job from the running_job_resource_pools table).
        ADE::ade_err_debug($errstack_ref, 10, "process_recently_reaped_pids: $job_id: deleting running resources allocations ...");
        $sql_statement = "DELETE FROM running_job_resource_pools\n" .
                         "WHERE JOB_ID = ?;\n";
        @sql_value     = ( $job_id );
        if (($rc=sql_execute_qm($errstack_ref, $dbh, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
            return($rc);
        }

        #  Finally change the state of the job.
        ADE::ade_err_debug($errstack_ref, 10, "process_recently_reaped_pids: $job_id: changing job state to completed ...");
        if (($rc=set_job_state_pid_exit_code_by_job_id($errstack_ref, $dbh, $job_id, 'completed', 0, $exit_code_of_recently_reaped_pids{$pid})) != $ADE::ADE_ERR_OK) {
            return($rc);
        }

        #  And if we managed to do all of that then we can commit the transaction.
        ADE::ade_err_debug($errstack_ref, 10, "process_recently_reaped_pids: $job_id: committing transaction ...");
        $sql_statement = "END TRANSACTION;\n";
        if (($rc=ADE::ade_sql_execute($errstack_ref, $dbh, $sql_statement)) != $ADE::ADE_ERR_OK) {
            return($rc);
        }

        ADE::ade_err_debug($errstack_ref, 10, "process_recently_reaped_pids: $job_id: marking job as completed ...");
        delete $exit_code_of_recently_reaped_pids{$pid};
    }

    return($ADE::ADE_ERR_OK);
}

sub start_job_id
{
    my($errstack_ref, $dbh, $job_id) = @_;
    my($pid, $user, $rc, $i, @select_results);
    my($sql_statement, @wanted_resources, @available_resources);
    my($resource_id, $quantity_wanted, $affinous);
    my($resource_id2, $resource_pool_id, $quantity_available, $quantity_allocated_so_far, $quantity_taking);
    my(@taking_resources, @new_envs, @sql_value);
    my($command, $var);

    #  Get the job's command and user.
    $sql_statement = "SELECT command, [user]\n" .
                     "FROM jobs WHERE job_id = ?;\n";
    @sql_value = ( $job_id );
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@select_results, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    ($command, $user) = @{$select_results[0]};
    ADE::ade_err_debug($errstack_ref, 10, "start_job_id: job_id=$job_id, command=$command, user=$user");

    #  Determine which resources we'll use
    $sql_statement = "SELECT pending_job_resources.resource_id, pending_job_resources.quantity_wanted, resources.affinous\n" .
                     "FROM pending_job_resources, resources\n" .
                     "WHERE pending_job_resources.resource_id=resources.resource_id AND\n" .
                     "      pending_job_resources.job_id = ?\n";
    @sql_value     = ( $job_id );
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@wanted_resources, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    $sql_statement = "SELECT resource_id, resource_pool_id, quantity_available\n" .
                     "FROM pending_job_preferred_resource_pool_quantities_available\n" .
                     "WHERE job_id = ?\n";
    @sql_value     = ( $job_id );
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@available_resources, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    @taking_resources = ();
    @new_envs = ();
    foreach my $tuple_ref (@wanted_resources) {
        $quantity_allocated_so_far = 0;
        ($resource_id, $quantity_wanted, $affinous) = @{$tuple_ref};
        ADE::ade_err_debug($errstack_ref, 10, "start_job_id: resource_id=$resource_id, quantity_wanted=$quantity_wanted, affinous=$affinous");
        foreach my $tuple_ref2 (@available_resources) {
            ($resource_id2, $resource_pool_id, $quantity_available) = @{$tuple_ref2};
            next if ($resource_id2 ne $resource_id);
            ADE::ade_err_debug($errstack_ref, 10, "start_job_id: resource_id2=$resource_id2, resource_pool_id=$resource_pool_id, quantity_available=$quantity_available");
            $quantity_taking = ($quantity_available<$quantity_wanted-$quantity_allocated_so_far)?$quantity_available:$quantity_wanted-$quantity_allocated_so_far;
            ADE::ade_err_debug($errstack_ref, 10, "start_job_id: would take $quantity_taking");
            push @taking_resources, { resource_id=>$resource_id, resource_pool_id=>$resource_pool_id, quantity_taking=>$quantity_taking };
            $quantity_allocated_so_far += $quantity_taking;
        }
        if ($affinous eq 'true') {
            #  Since this code is executed inside the outer loop, the first time @taking_resources 
            #  will contain resources of only 1 type, but on subsequent loops it will contain also others.
            #  This assigment is split to avoid confusing perlcritic.
            $var = sprintf 'JS_%s_AFFINITY', uc $resource_id;
            push @new_envs, { var=>$var, val=>join ',', map { ($_->{'resource_id'} eq $resource_id) ? $_->{'resource_pool_id'} : () } @taking_resources };
        } else {
            #  This assigment is split to avoid confusing perlcritic.
            $var = sprintf 'JS_%s_LIMIT', uc $resource_id;
            push @new_envs, { var=>$var, val=>$quantity_wanted };
        }
    }
    push @new_envs, { var=>'JS_JOB_ID', val=>$job_id };
    push @new_envs, { var=>'JS_USER', val=>$user };

    #  Additional environment variables.

    #  Start the job
    ADE::ade_err_debug($errstack_ref, 10, "start_job_id: $job_id: forking ...");
    if (($pid=fork) < 0) {
        ADE::ade_err_error($errstack_ref, 'jsd_err_misc', 'fork: failed');
        return($ADE::ADE_ERR_FAIL);
    } elsif ($pid == 0) {
        ADE::ade_err_debug($errstack_ref, 10, "start_job_id: $job_id: child setting up environment and execing command ...");
        foreach my $new_env_ref (@new_envs) {
            #   we can't localise setting $ENV{...} because we need its effects in the scope of this 'elsif ... ' stanza, not
            #   within this inner 'foreach' stanza.
            $ENV{$new_env_ref->{'var'}} = $new_env_ref->{'val'}; ## no critic (RequireLocalizedPunctuationVars)
        }
        if (!exec $js_shell, '-c', $command) {
            ADE::ade_err_error($errstack_ref, 'jsd_err_misc', 'failed to exec');
            return($ADE::ADE_ERR_FAIL);
        }
    }
    ADE::ade_err_debug($errstack_ref, 5, "start_job_id: forked pid $pid");

    #  Take the resources
    foreach my $taking_resource (@taking_resources) {
        $sql_statement = "INSERT INTO running_job_resource_pools VALUES (?, ?, ?, ?);\n";
        @sql_value     = ($job_id, $taking_resource->{'resource_id'}, $taking_resource->{'resource_pool_id'}, $taking_resource->{'quantity_taking'});
        if (($rc=sql_execute_qm($errstack_ref, $dbh, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
            return($rc);
        }
    }
    # Release the pending demands (i.e. delete all entries related to this job from the pending_job_resources table).
    $sql_statement = "DELETE FROM pending_job_resources\n" .
                     "WHERE job_id = ?;\n";
    @sql_value     = ( $job_id );
    if (($rc=sql_execute_qm($errstack_ref, $dbh, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    #  Finally change the state of the job.
    if (($rc=set_job_state_pid_exit_code_by_job_id($errstack_ref, $dbh, $job_id, 'running', $pid, 0)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    return($ADE::ADE_ERR_OK);
}

sub sighandler_chld    ## no critic (RequireFinalReturn)
{
    my($sig) = @_;
    my($pid, @errstack_local, $rc, $job_id);

    @errstack_local = ();

    #  Reap and note exit codes. The main loop will update the database table with this info.
    while (($pid = waitpid -1, WNOHANG) > 0) {
        $exit_code_of_recently_reaped_pids{$pid} = $?;
        ADE::ade_err_debug(\@errstack_local, 5, "======> sighandler_chld: noting $pid exited with $exit_code_of_recently_reaped_pids{$pid} ...");
    }
}

sub set_job_state_pid_exit_code_by_job_id  ## no critic (ProhibitManyArgs)
{
    my($errstack_ref, $dbh, $job_id, $job_state, $pid, $exit_code) = @_;
    my($rc, $sql_statement, @sql_value);

    ADE::ade_err_debug($errstack_ref, 10, "set_job_state_pid_exit_code_by_job_id: sof (job_id=$job_id, job_state=$job_state, pid=$pid)");
    $sql_statement = "UPDATE jobs\n" .
                     "SET job_state=?, pid=?, exit_code=? WHERE job_id == ?;\n";
    @sql_value     = ( $job_state, $pid, $exit_code, $job_id );
    if (($rc=sql_execute_qm($errstack_ref, $dbh, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    return($ADE::ADE_ERR_OK);
}

sub get_job_id_by_pid
{
    my($errstack_ref, $dbh, $pid, $job_id_ref) = @_;
    my($rc, @select_results, $sql_statement, @sql_value);

    $sql_statement = "SELECT job_id\n" .
                     "FROM jobs WHERE pid == ?;\n";
    @sql_value     = ( $pid );
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@select_results, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    #  Extract the first column out of each row, assemble an array out of that
    #  and write it where to caller told us to write it.
    ${$job_id_ref} = ${$select_results[0]}[0];

    return($ADE::ADE_ERR_OK);
}

sub get_job_state_by_job_id
{
    my($errstack_ref, $dbh, $job_id, $job_state_ref) = @_;
    my($rc, @select_results, $sql_statement, @sql_value);

    $sql_statement = "SELECT job_state\n" .
                     "FROM jobs WHERE job_id == ?;\n";
    @sql_value     = ( $job_id );
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@select_results, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    #  Extract the first column out of each row, assemble an array out of that
    #  and write it where to caller told us to write it.
    ${$job_state_ref} = ${$select_results[0]}[0];

    return($ADE::ADE_ERR_OK);
}

sub get_current_resource_value_by_name
{
    my($errstack_ref, $dbh, $resource_id, $resource_value_ref) = @_;
    my(@select_results, $rc, $sql_statement, @sql_value);

    $sql_statement = "SELECT current\n" .
                     "FROM resources\n" .
                     "WHERE resource_id == ?;\n";
    @sql_value     = ($resource_id);
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@select_results, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    ${$resource_value_ref} = ${$select_results[0]}[0];
    return($ADE::ADE_ERR_OK);
}

sub get_demanded_resource_value_by_job_id
{
    my($errstack_ref, $dbh, $resource_id, $job_id, $demanded_resource_value_ref) = @_;
    my(@select_results, $rc, $sql_statement, @sql_value);

    ADE::ade_err_debug($errstack_ref, 10, "get_demanded_resource_value_by_job_id: sof (resource_id=$resource_id, job_id=$job_id)");

    $sql_statement = "SELECT $resource_id\n" .
                     "FROM jobs\n" .
                     "WHERE job_id == ?;\n";
    @sql_value     = ( $job_id );
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@select_results, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    ${$demanded_resource_value_ref} = ${$select_results[0]}[0];
    return($ADE::ADE_ERR_OK);
}

#sub dump_tables
#{
#    my($errstack_ref, $level, $dbh) = @_;
#    my($table, $sql_command, $sth, @row, $select_result, @select_results, $rc, $sql_statement);
#
#    foreach $table (qw(resources jobs)) {
#        ADE::ade_err_debug($errstack_ref, $level, "dump_tables: name=$table");
#        undef @select_results;
#        $sql_statement = "SELECT *\n;" .
#                         "FROM $table;\n";
#        if (($rc=ADE::ade_sql_select($errstack_ref, $dbh, \@select_results, $sql_statement)) != $ADE::ADE_ERR_OK) {
#            return($rc);
#        }
#        foreach $select_result (@select_results) {
#           ADE::ade_err_debug($errstack_ref, $level, sprintf 'dump_tables: ' . scalar '%-10s' x ($#{$select_result}+1), @{$select_result});
#        }
#    }
#}

sub get_job_ids
{
    my($errstack_ref, $dbh, $job_state, $job_ids_arrayref) = @_;
    my($rc, @select_results, $sql_statement, @sql_value);

    if ($job_state eq 'pending' or $job_state eq 'running' or $job_state eq 'completed' or $job_state eq 'cancelled') {
        $sql_statement = "SELECT job_id\n" .
                         "FROM jobs\n" .
                         "WHERE job_state == ?;\n";
        @sql_value     = ( $job_state );
        if (($rc=sql_select_qm($errstack_ref, $dbh, \@select_results, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
            return($rc);
        }
        #  Extract the first column out of each row, assemble an array out of that
        #  and write it where to caller told us to write it.
        @{$job_ids_arrayref} = map { ${$_}[0] } @select_results;

    } elsif ($job_state eq 'possible') {
        #  find the IDs of jobs whose count of *met* resource requirements are *equal*
        #  to the number of resource types on the system. (E.g. a job with *unmet* RAM
        #  requirement and *met* cores requirement is not eligble to run ('cos 1!=2).

        #sqlite> select * from next_runnable_job;
        #job_id
        #----------
        #1

        $sql_statement = "SELECT job_id\n" .
                         "FROM runnable_jobs;\n";
        if (($rc=ADE::ade_sql_select($errstack_ref, $dbh, \@select_results, $sql_statement)) != $ADE::ADE_ERR_OK) {
            return($rc);
        }
        #  Extract the first column out of each row, assemble an array out of that
        #  and write it where to caller told us to write it.
        @{$job_ids_arrayref} = map { ${$_}[0] } @select_results;

    } else {
       ADE::ade_err_internal($errstack_ref, "get_job_ids: $job_state: invalid job_state");
    }

    return($ADE::ADE_ERR_OK);
}

sub process_message
{
    my($errstack_ref, $callback_args_ref, $message_hash_ref, $reply_hash_ref) = @_;
    my($dbh, $quit_flag_ref, $schedule_flag_ref, $rc, $show_queue_output, %wanted_resource_counts);
    my($job_state, $sql_statement, @select_results, @sql_value, @show_job_states, $pid, %command_handlers);

    %command_handlers = (
       undef        => { 'handler' => \&command_handler_undef,      'reqpars' => [ ] },
       'quit'       => { 'handler' => \&command_handler_quit,       'reqpars' => [ ] },
       'suspend'    => { 'handler' => \&command_handler_suspend,    'reqpars' => [ ] },
       'resume'     => { 'handler' => \&command_handler_resume,     'reqpars' => [ ] },
       'ping'       => { 'handler' => \&command_handler_ping,       'reqpars' => [ ] },
       'show-queue' => { 'handler' => \&command_handler_show_queue, 'reqpars' => [ 'show-job-states' ] },
       'submit-job' => { 'handler' => \&command_handler_submit_job, 'reqpars' => [ 'job-command', 'resource-list' ] },
       'cancel-job' => { 'handler' => \&command_handler_cancel_job, 'reqpars' => [ 'job-id' ] },
    );

    #  Initialise client error stack.
    ${$reply_hash_ref}{'errstack-ref'} = [];

    #  Note use of 'scalar ~~ [ not_a_list_but_something_that_returns_a_list ]'. The [...] force
    #  keys to see array context. See https://stackoverflow.com/questions/18243584/.
    ADE::ade_err_debug($errstack_ref, 10, 'process_message: checking command is known ...');
    if (!(${$message_hash_ref}{'command'} ~~ [keys %command_handlers])) {
        ADE::ade_err_error(${$reply_hash_ref}{'errstack-ref'}, 'js_err_misc', "${$message_hash_ref}{'command'}: command unknown on server");
        ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_FAIL;
        return($ADE::ADE_ERR_OK);
    }

    ADE::ade_err_debug($errstack_ref, 10, 'process_message: checking required parameters are in message ...');
    foreach my $reqpar (@{$command_handlers{${$message_hash_ref}{'command'}}{'reqpars'}}) {
        if (not defined ${$message_hash_ref}{$reqpar}) {
            ADE::ade_err_error(${$reply_hash_ref}{'errstack-ref'}, 'js_err_misc', "${$message_hash_ref}{'command'} command has no $reqpar parameter set");
            ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_FAIL;
            return($ADE::ADE_ERR_OK);
        }
    }

    ADE::ade_err_debug($errstack_ref, 10, 'process_message: calling handler ...');
    if (($rc=$command_handlers{${$message_hash_ref}{'command'}}{'handler'}($errstack_ref, $callback_args_ref, $message_hash_ref, $reply_hash_ref)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    return($ADE::ADE_ERR_OK);
}

sub command_handler_undef
{
    my($errstack_ref, $callback_args_ref, $message_hash_ref, $reply_hash_ref) = @_;

    #  Guts

    #  Prepare reply to client
    ADE::ade_err_error(${$reply_hash_ref}{'errstack-ref'}, 'js_err_misc', 'message has no command');
    ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_OK;

    return($ADE::ADE_ERR_OK);
}

sub command_handler_quit
{
    my($errstack_ref, $callback_args_ref, $message_hash_ref, $reply_hash_ref) = @_;
    my($dbh, $quit_flag_ref, $schedule_flag_ref);

    #  Unpack callback argument list.
    ($dbh, $quit_flag_ref, $schedule_flag_ref) = @{$callback_args_ref};

    #  Guts
    ${$quit_flag_ref} = 1;

    #  Prepare reply to client
    ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_OK;

    return($ADE::ADE_ERR_OK);
}

sub command_handler_suspend
{
    my($errstack_ref, $callback_args_ref, $message_hash_ref, $reply_hash_ref) = @_;
    my($dbh, $quit_flag_ref, $schedule_flag_ref);

    #  Unpack callback argument list.
    ($dbh, $quit_flag_ref, $schedule_flag_ref) = @{$callback_args_ref};

    #  Guts
    ${$schedule_flag_ref} = 0;

    #  Prepare reply to client
    ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_OK;

    return($ADE::ADE_ERR_OK);
}

sub command_handler_resume
{
    my($errstack_ref, $callback_args_ref, $message_hash_ref, $reply_hash_ref) = @_;
    my($dbh, $quit_flag_ref, $schedule_flag_ref);

    #  Unpack callback argument list.
    ($dbh, $quit_flag_ref, $schedule_flag_ref) = @{$callback_args_ref};

    #  Guts
    ${$schedule_flag_ref} = 1;

    #  Prepare reply to client
    ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_OK;

    return($ADE::ADE_ERR_OK);
}

sub command_handler_ping
{
    my($errstack_ref, $callback_args_ref, $message_hash_ref, $reply_hash_ref) = @_;

    #  Guts
    ${$reply_hash_ref}{'stdout'} = "pong\n";

    #  Prepare reply to client
    ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_OK;

    return($ADE::ADE_ERR_OK);
}

sub command_handler_show_queue
{
    my($errstack_ref, $callback_args_ref, $message_hash_ref, $reply_hash_ref) = @_;
    my($dbh, $quit_flag_ref, $schedule_flag_ref, @show_job_states, $rc, $show_queue_output);
    my($sql_statement, $pending_jobs_count, $running_jobs_count, $cancelled_jobs_count, $completed_jobs_count);
    my(@select_results, $job_id, $job_state, $pid, $exit_code, $command, @sql_value);

    #  Unpack callback argument list.
    ($dbh, $quit_flag_ref, $schedule_flag_ref) = @{$callback_args_ref};

    #  Guts
    @show_job_states = split /,/, ${$message_hash_ref}{'show-job-states'};
    #  Initialise output string; after this we append.
    ${$reply_hash_ref}{'stdout'} = '';
    $sql_statement = "SELECT count(*)\n" .
                     "FROM jobs\n" .
                     "WHERE job_state == ?\n";
    @sql_value     = ('pending');
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@select_results, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    $pending_jobs_count = ${$select_results[0]}[0];
    @sql_value     = ('running');
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@select_results, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    $running_jobs_count = ${$select_results[0]}[0];
    @sql_value     = ('completed');
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@select_results, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    $completed_jobs_count = ${$select_results[0]}[0];
    @sql_value     = ('cancelled');
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@select_results, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    $cancelled_jobs_count = ${$select_results[0]}[0];
    ${$reply_hash_ref}{'stdout'} .= "jobs ($running_jobs_count running, $pending_jobs_count pending, $completed_jobs_count completed, $cancelled_jobs_count cancelled)\n----\n\n";
    ${$reply_hash_ref}{'stdout'} .= sprintf "%-4s %9s %5s %3s %s\n%s\n",'id','state','pid','rc','command','-'x78;
    $sql_statement = "SELECT job_id,job_state,pid,exit_code,command\n" .
                     "FROM jobs\n" .
                     "ORDER BY job_id;\n";
    if (($rc=ADE::ade_sql_select($errstack_ref, $dbh, \@select_results, $sql_statement)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    foreach my $select_result (@select_results) {
        ($job_id, $job_state, $pid, $exit_code, $command) = @{$select_result};
        next if not $job_state ~~ @show_job_states;
        ${$reply_hash_ref}{'stdout'} .= sprintf "%-4d %9s %5d %3d %s\n", $job_id, $job_state, $pid, $exit_code, $command;
    }
    ${$reply_hash_ref}{'stdout'} .= "\n";
    ${$reply_hash_ref}{'stdout'} .= "resources\n---------\n\n";
    ${$reply_hash_ref}{'stdout'} .= sprintf "%-8s %8s %4s\n%s\n",'id','max','curr','-'x78;
    $sql_statement = "SELECT resources.resource_id, quantity_total, sum(coalesce(quantity_used,0)) as quantity_used\n" .
                     "FROM resources LEFT OUTER JOIN running_job_resource_pools\n" .
                     "ON resources.resource_id = running_job_resource_pools.resource_id\n" .
                     "GROUP BY resources.resource_id\n" .
                     "ORDER BY resources.resource_id;\n";
    if (($rc=ADE::ade_sql_select($errstack_ref, $dbh, \@select_results, $sql_statement)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    foreach my $select_result (@select_results) {
        ${$reply_hash_ref}{'stdout'} .= sprintf "%-8s %8d %4d\n", @{$select_result};
    }
    ${$reply_hash_ref}{'stdout'} .= "\n";

    #  Prepare reply to client
    ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_OK;

    return($ADE::ADE_ERR_OK);
}

sub command_handler_submit_job
{
    my($errstack_ref, $callback_args_ref, $message_hash_ref, $reply_hash_ref) = @_;
    my($command, $rc, $sql_statement, @select_results, $job_id);
    my($dbh, $quit_flag_ref, $schedule_flag_ref, %wanted_resource_counts);
    my($wanted_resource_name, $wanted_resource_count, @sql_value);

    #  Unpack callback argument list.
    ($dbh, $quit_flag_ref, $schedule_flag_ref) = @{$callback_args_ref};

    #  Guts
    ADE::ade_err_debug($errstack_ref, 10, 'process_message: chopping up wanted resource list ...');
    foreach my $wanted_resource (split /,/, ${$message_hash_ref}{'resource-list'}) {
        ($wanted_resource_name, $wanted_resource_count) = split /:/, $wanted_resource;
        $wanted_resource_counts{$wanted_resource_name} = $wanted_resource_count;
    }
    ADE::ade_err_debug($errstack_ref, 10, sprintf 'process_message: wanted resource list is [%s]', join ',', map { sprintf '%s=>%s', $_, $wanted_resource_counts{$_} } keys %wanted_resource_counts);
    #  Get the list of resources jsd manages.
    $sql_statement = "SELECT resource_id FROM resources;\n";
    if (($rc=ADE::ade_sql_select($errstack_ref, $dbh, \@select_results, $sql_statement)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    #  If that differs from what's been passed then error. This can be done more succinctly
    #  with the Array::Diff module but introduces that extra dependency. Instead we check
    #  the count and then, item by item, the sorted lists.
    if ($#select_results+1 != scalar keys %wanted_resource_counts) {
        ADE::ade_err_error(${$reply_hash_ref}{'errstack-ref'}, 'js_err_misc', 'resource count mismatch');
        ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_FAIL;
        return($ADE::ADE_ERR_OK);
    }
    foreach my $i (0..$#select_results) {
        if ((sort map { $select_results[$_][0] } (0..$#select_results))[$i] ne (sort keys %wanted_resource_counts)[$i]) {
            ADE::ade_err_error(${$reply_hash_ref}{'errstack-ref'}, 'js_err_misc', 'resource list mismatch');
            ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_FAIL;
            return($ADE::ADE_ERR_OK);
        }
    }
    #  Let the database decide the job id but do the whole thing in one transaction
    $sql_statement = "BEGIN TRANSACTION;\n";
    if (($rc=ADE::ade_sql_execute($errstack_ref, $dbh, $sql_statement)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    $sql_statement = 'INSERT INTO jobs (command, [user]) VALUES(?, ?);';
    @sql_value = ( ${$message_hash_ref}{'job-command'}, getpwuid $< );
    if (($rc=sql_execute_qm($errstack_ref, $dbh, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        $sql_statement = "ROLLBACK TRANSACTION;\n";
        ADE::ade_sql_execute($errstack_ref, $dbh, $sql_statement);
        return($rc);
    }
    $job_id = $dbh->last_insert_id('','','','');
    ADE::ade_err_debug($errstack_ref, 10, "submit_job: job_id=$job_id");
    foreach my $resource_id (keys %wanted_resource_counts) {
    ADE::ade_err_debug($errstack_ref, 10, "submit_job: resource_id=$resource_id");
        $sql_statement = 'INSERT INTO pending_job_resources VALUES (?, ?, ?);';
        @sql_value = ( $job_id, $resource_id, $wanted_resource_counts{$resource_id} );
        if (($rc=sql_execute_qm($errstack_ref, $dbh, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
            $sql_statement = 'ROLLBACK TRANSACTION;';
            ADE::ade_sql_execute($errstack_ref, $dbh, $sql_statement);
            return($rc);
        }
    }
    $sql_statement = "END TRANSACTION;\n";
    if (($rc=ADE::ade_sql_execute($errstack_ref, $dbh, $sql_statement)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }

    #  Prepare reply to client
    ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_OK;
    ${$reply_hash_ref}{'stdout'} = "job submitted with job id $job_id\n";

    return($ADE::ADE_ERR_OK);
}

sub command_handler_cancel_job
{
    my($errstack_ref, $callback_args_ref, $message_hash_ref, $reply_hash_ref) = @_;
    my($dbh, $quit_flag_ref, $schedule_flag_ref, $rc, $sql_statement, $job_state, $pid, @select_results, @sql_value);

    #  Unpack callback argument list.
    ($dbh, $quit_flag_ref, $schedule_flag_ref) = @{$callback_args_ref};

    #  Guts
    #  Is job running?
    ADE::ade_err_debug($errstack_ref, 10, "command_handler_cancel_job: checking status of job ${$message_hash_ref}{'job-id'} ...");
    $sql_statement = "SELECT job_state, pid\n" .
                     "FROM jobs\n" .
                     "WHERE job_id == ?;\n";
    @sql_value = ( ${$message_hash_ref}{'job-id'} );
    if (($rc=sql_select_qm($errstack_ref, $dbh, \@select_results, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
        return($rc);
    }
    if ($#select_results+1 == 0) {
        ADE::ade_err_error(${$reply_hash_ref}{'errstack-ref'}, 'js_err_misc', "${$message_hash_ref}{'job-id'}: no such job");
        ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_FAIL;
        return($ADE::ADE_ERR_OK);
    }
    ( $job_state, $pid ) = @{$select_results[0]};
    ADE::ade_err_debug($errstack_ref, 10, "command_handler_cancel_job: job_state=$job_state, pid=$pid");

    if ($job_state eq 'running') {
        if (kill 'TERM', $pid) {
            ${$reply_hash_ref}{'info'} = "${$message_hash_ref}{'job-id'}: job killed";
            ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_OK;
        } else {
            ADE::ade_err_error(${$reply_hash_ref}{'errstack-ref'}, 'js_err_misc', "${$message_hash_ref}{'job-id'}: failed to kill job (hint: permissions issue?)");
            ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_FAIL;
        }
    } elsif ($job_state eq 'completed') {
        ADE::ade_err_error(${$reply_hash_ref}{'errstack-ref'}, 'js_err_misc', "${$message_hash_ref}{'job-id'}: cannot be cancelled because already completed");
        ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_FAIL;

    } elsif ($job_state eq 'pending') {
        #  THIS SHOULD BE DONE IN A TRANSACTION!
        $sql_statement = "DELETE FROM pending_job_resources\n" .
                         "WHERE job_id = ?;\n";
        @sql_value     = ( ${$message_hash_ref}{'job-id'} );
        if (($rc=sql_execute_qm($errstack_ref, $dbh, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
            return($rc);
        }
        $sql_statement = "UPDATE jobs\n" .
                         "SET job_state = ?\n" .
                         "WHERE job_id == ?;\n";
        @sql_value     = ( 'cancelled', ${$message_hash_ref}{'job-id'} );
        if (($rc=sql_execute_qm($errstack_ref, $dbh, $sql_statement, \@sql_value)) != $ADE::ADE_ERR_OK) {
            return($rc);
        }
        #  THAT SHOULD HAVE BEEN DONE IN A TRANSACTION!
        ${$reply_hash_ref}{'info'} = "${$message_hash_ref}{'job-id'}: cancelled";
        ${$reply_hash_ref}{'rc'} = $ADE::ADE_ERR_OK;
    }

    return($ADE::ADE_ERR_OK);
}

sub sql_execute_qm
{
    my($errstack_ref, $dbh, $sql_statement, $sql_value_ref) = @_;

    if (not defined $dbh->do($sql_statement, undef, @{$sql_value_ref})) {
        #  This was $DBI::errstr, but perlcritic complained.
        ADE::ade_err_error($errstack_ref, 'jsd_err_misc', $dbh->errstr);
        return($ADE::ADE_ERR_FAIL);
    }

    return($ADE::ADE_ERR_OK);
}

sub sql_select_qm
{
    my($errstack_ref, $dbh, $select_results_arrayref, $sql_statement, $sql_value_ref) = @_;
    my($sth, @row);

    @{$select_results_arrayref} = ();
    ADE::ade_err_debug($errstack_ref, 10, "sql_select_qm: running \"$sql_statement\" with value @{$sql_value_ref} ...");
    $sth = $dbh->prepare($sql_statement);
    $sth->execute(@{$sql_value_ref});
    while (@row = $sth->fetchrow_array) {
         #  We can't push a reference to @row onto the caller's array, because
         #  @row will shortly be reset and that means the data in that is lost.
         #  (Oddly this doesn't result in multiple entries of the same anonymous
         #  array, but rather in empty arrays for all but the last element.)
         push @{$select_results_arrayref}, [@row];
    }

    return($ADE::ADE_ERR_OK);
}

ADE::ade_gep_main(\&jsd);
