#!/bin/bash
#  $HeadURL$ $LastChangedRevision$

#  Properties
MY_ID=system_hardware_vm_virsh_storage_drbd
PARENT_ID=system_hardware_vm_virsh_storage

#  Defaults for the questions we ask (should only used here)

#  Hooks

#  Support functions




#  Globals
KNOWN_STORAGESERVER_HOSTNAMES="$KNOWN_VMSERVER_HOSTNAMES"
STORAGESERVER_VOLUME_GROUP=vg0
DRBD_NIC=br2

loading main
old_main()
{
    local TARGET_HOSTNAME TARGET_ROOT_DISK_SIZE VMSERVER_HOSTNAME IGNORE_STATE_FILE_FLAG LIST_FILTER LIST_FORMAT MAILTO
    #local

    #  Defaults for options
    VERBOSELEVEL=2
    IGNORE_STATE_FILE_FLAG=false
    LIST_FORMAT=human
    LIST_FILTER=inprogress

    #  Process options
    while [ "X$1" != X ]; do
        case $1 in 
            #  Application specific options
            --vmserver-hostname=*)        VMSERVER_HOSTNAME=${1#*=} ;;
            --target-hostname=*)          TARGET_HOSTNAME=${1#*=} ;;
            --target-root-disk-size=*)    TARGET_ROOT_DISK_SIZE=${1#*=} ;;
            --mailto=*)                   MAILTO=${1#*=} ;;
            --ignore-state-file-flag=*)   IGNORE_STATE_FILE_FLAG=${1#*=} ;;
            --list-format=*)              LIST_FORMAT=${1#*=} ;;
            --list-filter=*)              LIST_FILTER=${1#*=} ;;
            #  General options
            --help|-h)                    usage 0 ;;
            --verbose|-v)                 VERBOSELEVEL=3 ;;
            -d)                           VERBOSELEVEL=$2; shift ;;
            --debug=*)                    VERBOSELEVEL=${1#*=} ;;
            --)                           shift; break ;;
            -*)                           usage ;;
            *)                            break ;; 
        esac
        shift
    done

    #  Process arguments
    [ $# -ge 1 ] || usage
    MODE=$1; shift
    debug 10 "main: MODE=$MODE"

    #  Sanity checks and derivations
    [ $MODE = get-rootfs-volname ] || validate_vars MODE || return $?

    #  Guts
    case $MODE in
        create) create --vmserver-hostname=$VMSERVER_HOSTNAME --target-hostname=$TARGET_HOSTNAME --target-root-disk-size=$TARGET_ROOT_DISK_SIZE --mailto=$MAILTO "$@" || return $? ;;
        delete) delete --target-hostname=$TARGET_HOSTNAME --ignore-state-file-flag=$IGNORE_STATE_FILE_FLAG "$@" || return $? ;;
        list)   list --list-format=$LIST_FORMAT --list-filter=$LIST_FILTER "$@" ;;
        list-finished)   list_finished "$@" || return $? ;;
        get-rootfs-volname) get_rootfs_volname --vmserver-hostname=$VMSERVER_HOSTNAME --target-hostname=$TARGET_HOSTNAME --target-root-disk-size=$TARGET_ROOT_DISK_SIZE "$@" || return $? ;;
        purge)  purge "$@" || return $? ;;
        *)      usage ;;
    esac

    return 0
}

loading create
old_create()
{
    local TARGET_HOSTNAME TARGET_ROOT_DISK_SIZE VMSERVER_HOSTNAME MAILTO

    #  Process options
    while [ "X$1" != X ]; do
        case $1 in
            #  Application specific options
            --vmserver-hostname=*)        VMSERVER_HOSTNAME=${1#*=} ;;
            --target-hostname=*)          TARGET_HOSTNAME=${1#*=} ;;
            --target-root-disk-size=*)    TARGET_ROOT_DISK_SIZE=${1#*=} ;;
            --mailto=*)                   MAILTO=${1#*=} ;;
            #  General options
            --)                  shift; break ;;
            -*)                  internal "create: $1: invalid option" ;;
            *)                   break ;; 
        esac
        shift
    done

    #  Process arguments
    [ $# = 0 ] || internal "create: $#: wrong arg count"
    
    #  Sanity checks and derivations
    validate_vars TARGET_HOSTNAME VMSERVER_HOSTNAME TARGET_ROOT_DISK_SIZE MAILTO || return $?
    debug 10 "create: TARGET_HOSTNAME=$TARGET_HOSTNAME, VMSERVER_HOSTNAME=$VMSERVER_HOSTNAME, TARGET_ROOT_DISK_SIZE=$TARGET_ROOT_DISK_SIZE, MAILTO=$MAILTO"
    for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do
        check_ssh_access $STORAGESERVER_HOSTNAME || return $?
    done
    #  Work out next available minor number (which doubles as port number offset)
    USED_MINOR_NUMBERS=$(for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do ssh -n $STORAGESERVER_HOSTNAME "cat /etc/drbd.d/*.res 2>/dev/null || true"; done | sed -n 's/.* minor \(.*\);/\1/p' | sort -un | paste -s -d' ')
    debug 10 "create: USED_MINOR_NUMBERS=\"$USED_MINOR_NUMBERS\""
    for ((I=0; I<100; I++)); do
        debug 10 "create: checking if minor number $I available ..."
        MATCHED=false
        for USED_MINOR_NUMBER in $USED_MINOR_NUMBERS; do
            [ $I != $USED_MINOR_NUMBER ] || { MATCHED=true; break; }
        done
        $MATCHED || { NEXT_MINOR_NUMBER=$I; break; }
    done
    debug 10 "create: selected minor number $NEXT_MINOR_NUMBER"

    #  Guts
    #  Manage state file
    mkdir -p $VAR_DIR/$PROGNAME
    debug 10 "create: state file $VAR_DIR/$PROGNAME/$TARGET_HOSTNAME [$(ls -ld $VAR_DIR/$PROGNAME/$TARGET_HOSTNAME 2>&1)]"
    [ ! -f $VAR_DIR/$PROGNAME/$TARGET_HOSTNAME ] || { error "$TARGET_HOSTNAME: installation in progress"; return 1; }
    touch $VAR_DIR/$PROGNAME/$TARGET_HOSTNAME

    #  Create LV
    for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do
        debug 5 "create: creating LV on $STORAGESERVER_HOSTNAME ..."
        CMDLINE="ssh -n $STORAGESERVER_HOSTNAME \"lvcreate --size ${TARGET_ROOT_DISK_SIZE}G --name $TARGET_HOSTNAME $STORAGESERVER_VOLUME_GROUP\""
        debug 10 "create: calling [$CMDLINE] ..."
        LVCREATE_OUTPUT=$(eval "$CMDLINE" 2>&1) || { RC=1; error "lvcreate: failed (output was: $LVCREATE_OUTPUT)"; rm -f $VAR_DIR/$PROGNAME/$TARGET_HOSTNAME; return $RC; }
    done

    #  Do common DRBD setup
    for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do
        debug 5 "create: writing config file $STORAGESERVER_HOSTNAME:/etc/drbd.d/$TARGET_HOSTNAME.res ..."
        {
            echo "resource $TARGET_HOSTNAME {"
            #  xen live migration requires two primaries (see 
            #  http://www.drbd.org/users-guide-emb/s-xen-create-resource.html
            #  but that means protocol C is required; attempting to use protocol
            #  A produced the message '0: Failure: (139) Protocol C required'
            #  when I tried to up the resource. So it's a toss-up: protocol c or
            #  live migration. I want performance so I go for protocol C and no
            #  live migration. Nope, it turns out that virsh also wants the 
            #  non-active node to have write access to the device. So it *must*
            #  be dual primary and that means it must be protocl C.
            #
            #  The saga continues: I keep getting split brain. So now I make one
            #  primary max. So I lose live migrations ... okay, so be it. Let's see
            #  if it works better now.
            echo "  #net {"
            echo "  #  allow-two-primaries;"
            echo "  #}"
            echo "  protocol  C;"
            echo "  device    /dev/drbd_$TARGET_HOSTNAME minor $NEXT_MINOR_NUMBER;"
            echo "  meta-disk internal;"
            for STORAGESERVER_HOSTNAME2 in $KNOWN_STORAGESERVER_HOSTNAMES; do
                echo "  on $STORAGESERVER_HOSTNAME2 {"
                hostnic2ipaddr $STORAGESERVER_HOSTNAME2 $DRBD_NIC IPADDR || return $?
                echo "    address   $IPADDR:$((7789+$NEXT_MINOR_NUMBER));"
                echo "    disk      /dev/$STORAGESERVER_VOLUME_GROUP/$TARGET_HOSTNAME;"
                echo "  }"
            done
            echo "}"
        } | ssh $STORAGESERVER_HOSTNAME "cat > /etc/drbd.d/$TARGET_HOSTNAME.res"
        debug 5 "create: initialising metadata on $STORAGESERVER_HOSTNAME ..."
        #  'drbdadm create-md' can prompt for 'yes' twice.
        echo -e "yes\nyes" | ssh $STORAGESERVER_HOSTNAME "drbdadm create-md $TARGET_HOSTNAME"
        debug 5 "create: upping resource on $STORAGESERVER_HOSTNAME ..."
        ssh -n $STORAGESERVER_HOSTNAME "modprobe drbd; drbdadm up $TARGET_HOSTNAME"
    done

    #  Do one-sided DRBD setup
    for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do
        [ $STORAGESERVER_HOSTNAME = $VMSERVER_HOSTNAME ] || continue
        debug 5 "create: making the resource primary on $STORAGESERVER_HOSTNAME ..."
        ssh -n $STORAGESERVER_HOSTNAME "drbdadm -- --overwrite-data-of-peer primary $TARGET_HOSTNAME"
    done
    for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do
        [ $STORAGESERVER_HOSTNAME != $VMSERVER_HOSTNAME ] || continue
        debug 5 "create: making the resource secondary on $STORAGESERVER_HOSTNAME ..."
        ssh -n $STORAGESERVER_HOSTNAME "drbdadm secondary $TARGET_HOSTNAME"
        ssh -n $STORAGESERVER_HOSTNAME "drbdadm down $TARGET_HOSTNAME"
        send_email <<EOF
To: $MAILTO
Subject: $TARGET_HOSTNAME: last steps to configure this VM

This system's root disk is a new DRBD device. To reduce network traffic
during the installation, the DRBD device has been stopped on the standby
node. To restart it you need to run:

    ssh root@$STORAGESERVER_HOSTNAME
    drbdadm up $TARGET_HOSTNAME
EOF
    done

    #  Tell libvirt
    #
    #  Debian suffers from the bug documented in Ubuntu bug #697046 (creating
    #  volumes inside a disk-based pool fails because libvirt doesn't have
    #  path to parted binary defined). Various attempts to work around this
    #  issue by using libvirt-based interfaces (e.g. virsh) all failed. The
    #  cleanest non-kosher workaround I worked out was to use the 'default'
    #  directory-based pool and create symlinks in it to the pre-allocated
    #  devices.
    #
    #  Oh, and in case I ever do this the proper way again: when I was using
    #  a block device as the pool, it still needed a partition table on it
    #  in order to be able to allocate actual volumes out of it. I did this
    #  with:
    #
    #ssh -n $VMSERVER_HOSTNAME "parted /dev/drbd_$TARGET_HOSTNAME mklabel msdos"

    #  Make the pools on all machines ... 
    for VMSERVER_HOSTNAME2 in $KNOWN_VMSERVER_HOSTNAMES; do
        ssh -n $VMSERVER_HOSTNAME2 "mkdir -p /var/local/lib/pools/$TARGET_HOSTNAME"
        ssh -n $VMSERVER_HOSTNAME2 "ln -s /dev/drbd_$TARGET_HOSTNAME /var/local/lib/pools/$TARGET_HOSTNAME/$TARGET_HOSTNAME"
        ssh -n $VMSERVER_HOSTNAME2 "virsh pool-define-as --name=$TARGET_HOSTNAME --type=dir --target=/var/local/lib/pools/$TARGET_HOSTNAME"
        #  ... but only activate on the VM server where the VM will run.
        [ $VMSERVER_HOSTNAME2 != $VMSERVER_HOSTNAME ] || ssh -n $VMSERVER_HOSTNAME2 "virsh pool-start $TARGET_HOSTNAME"
    done

    return 0
}

loading delete
old_delete()
{
    local TARGET_HOSTNAME IGNORE_STATE_FILE_FLAG
    local LVCREATE_OUTPUT VIRSH_OUTPUT IETADM_OUTPUT NEXT_FREE_TID IETADM_CMDLINE TID IQN LV_NAME LV_PATH IETCONF_RECORD TARGET_ROOT_DISK_SIZE VMSERVER_HOSTNAME

    #  Defaults for options
    IGNORE_STATE_FILE_FLAG=false

    #  Process options
    while [ "X$1" != X ]; do
        case $1 in
            #  Application specific options
            --target-hostname=*)        TARGET_HOSTNAME=${1#*=} ;;
            --ignore-state-file-flag=*) IGNORE_STATE_FILE_FLAG=${1#*=} ;;
            #  General options
            --)                         shift; break ;;
            -*)                         internal "delete: $1: invalid option" ;;
            *)                          break ;;
        esac
        shift
    done

    #  Process arguments
    [ $# = 0 ] || internal "delete: $#: wrong arg count"
   
    #  Sanity checks and derivations
    validate_vars TARGET_HOSTNAME IGNORE_STATE_FILE_FLAG || return $?
    debug 10 "delete: TARGET_HOSTNAME=$TARGET_HOSTNAME, IGNORE_STATE_FILE_FLAG=$IGNORE_STATE_FILE_FLAG"
    
    #  Guts
    #  Manage state file
    mkdir -p $VAR_DIR/$PROGNAME
    $IGNORE_STATE_FILE_FLAG || [ -f $VAR_DIR/$PROGNAME/$TARGET_HOSTNAME ] || { warning "$TARGET_HOSTNAME: installation not in progress using this plugin"; return 0; }

    #  Only delete the storage if not running from the releaser
    if [ "X$RELEASER_FLAG" != Xtrue ]; then
        for VMSERVER_HOSTNAME in $KNOWN_VMSERVER_HOSTNAMES; do
            ssh -n $VMSERVER_HOSTNAME "virsh pool-destroy $TARGET_HOSTNAME"
            ssh -n $VMSERVER_HOSTNAME "virsh pool-undefine $TARGET_HOSTNAME"
            ssh -n $VMSERVER_HOSTNAME "rm -fr /var/local/lib/pools/$TARGET_HOSTNAME"
        done
        for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do
            ssh -n $STORAGESERVER_HOSTNAME "drbdadm down $TARGET_HOSTNAME"
            ssh -n $STORAGESERVER_HOSTNAME "rm -f /etc/drbd.d/$TARGET_HOSTNAME.res"
        done
        for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do
            ssh -n $STORAGESERVER_HOSTNAME "lvremove -f /dev/$STORAGESERVER_VOLUME_GROUP/$TARGET_HOSTNAME || true"
        done
    fi

    #  Manage state file
    rm -f $VAR_DIR/$PROGNAME/$TARGET_HOSTNAME

    return 0
}

loading list
old_list()
{
   local TARGET_HOSTNAME LIST_FORMAT LIST_FILTER
   local CMDLINE 

   #  Defaults for options
   LIST_FORMAT=human
   LIST_FILTER=inprogress

   #  Process options
    while [ "X$1" != X ]; do
        case $1 in
            #  Application specific options
            --list-format=*)          LIST_FORMAT=${1#*=} ;;
            --list-filter=*)          LIST_FILTER=${1#*=} ;;
            #  General options
            --)                  shift; break ;;
            -*)                  internal "list: $1: invalid option" ;;
            *)                   break ;;
        esac
        shift
    done

    #  Process arguments
    [ $# = 0 ] || internal "list: $#: wrong arg count"

    #  Sanity checks and derivations
    validate_vars LIST_FORMAT LIST_FILTER || return $?

    #  Guts
    #  Manage state file
    mkdir -p $VAR_DIR/$PROGNAME

    #  Do the listing
    if [ $LIST_FORMAT = terse -a $LIST_FILTER = inprogress ]; then
        list_inprogress | while read TARGET_HOSTNAME; do format_terse $TARGET_HOSTNAME; done
    elif [ $LIST_FORMAT = human -a $LIST_FILTER = inprogress ]; then
        list_inprogress | while read TARGET_HOSTNAME; do format_human $TARGET_HOSTNAME; done
    elif [ $LIST_FORMAT = terse -a $LIST_FILTER = completed ]; then
        list_completed | while read TARGET_HOSTNAME; do format_terse $TARGET_HOSTNAME; done
    elif [ $LIST_FORMAT = human -a $LIST_FILTER = completed ]; then
        list_completed | while read TARGET_HOSTNAME; do format_human $TARGET_HOSTNAME; done
    fi

    return 0
}

old_list_inprogress()
{
    ls -1 $VAR_DIR/$PROGNAME
}

old_list_completed()
{
    local TARGET_HOSTNAME

    {
        #  We assume that the *first* component of the storage (i.e. the LV)
        #  is enough to determine if there are *any* components for a VM. (We
        #  might come back and add more stanzas like this one later.)
        for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do 
            ssh -n $STORAGESERVER_HOSTNAME "lvs $STORAGESERVER_VOLUME_GROUP" | sed -e '1d' | awk '{ print $1 }'
        done 
        for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do 
            ssh -n $STORAGESERVER_HOSTNAME "drbdadm status" | sed -n 's/.* name=\"\([^\"]*\)\".*/\1/p'
        done
        for VMSERVER_HOSTNAME in $KNOWN_VMSERVER_HOSTNAMES; do
            ssh -n $VMSERVER_HOSTNAME "virsh pool-list --all" | sed -e '1,2d' -e '$d' -e 's/  *.*//'
        done
    } | sort -u | while read TARGET_HOSTNAME; do 
        #  Only display it if there is *not* an in-progress file for it
        [ -f $VAR_DIR/$PROGNAME/$TARGET_HOSTNAME ] || {
            debug 10 "list_completed: TARGET_HOSTNAME=$TARGET_HOSTNAME"
            echo $TARGET_HOSTNAME
        }
    done
}

old_format_terse()
{
    local TARGET_HOSTNAME

    TARGET_HOSTNAME=$1

    debug 10 "format_terse: TARGET_HOSTNAME=$TARGET_HOSTNAME"
    echo $TARGET_HOSTNAME
}

old_format_human()
{
    local TARGET_HOSTNAME

    TARGET_HOSTNAME=$1

    debug 10 "format_human: TARGET_HOSTNAME=$TARGET_HOSTNAME"
    for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do 
        ssh -n $STORAGESERVER_HOSTNAME "lvs $STORAGESERVER_VOLUME_GROUP" | sed -e '1d' -e 's/$/ (LV)/'
    done | grep $TARGET_HOSTNAME
    for STORAGESERVER_HOSTNAME in $KNOWN_STORAGESERVER_HOSTNAMES; do 
        ssh -n $STORAGESERVER_HOSTNAME "drbdadm status" | sed -n 's/\(.* name=.*\)/\1 (DRBD)/p'
    done | grep $TARGET_HOSTNAME
    for VMSERVER_HOSTNAME in $KNOWN_VMSERVER_HOSTNAMES; do
        ssh -n $VMSERVER_HOSTNAME "virsh pool-list --all" | sed -e '1,2d' -e '$d' -e 's/$/ (pool)/'
    done | grep $TARGET_HOSTNAME
}

loading get_rootfs_volname
old_get_rootfs_volname()
{
    local TARGET_HOSTNAME TARGET_ROOT_DISK_SIZE VMSERVER_HOSTNAME

    #  Process options
    while [ "X$1" != X ]; do
        case $1 in
            #  Application specific options
            --vmserver-hostname=*)        VMSERVER_HOSTNAME=${1#*=} ;;
            --target-hostname=*)          TARGET_HOSTNAME=${1#*=} ;;
            --target-root-disk-size=*)    TARGET_ROOT_DISK_SIZE=${1#*=} ;;
            #  General options
            --)                           shift; break ;;
            -*)                           internal "get_rootfs_volname: $1: invalid option" ;;
            *)                            break ;;
        esac
        shift
    done

    #  Process arguments
    [ $# = 0 ] || internal "get_rootfs_volname: $#: wrong arg count"
    
    #  Sanity checks and derivations
    validate_vars TARGET_HOSTNAME VMSERVER_HOSTNAME TARGET_ROOT_DISK_SIZE || return $?
    debug 10 "get_rootfs_volname: TARGET_HOSTNAME=$TARGET_HOSTNAME, VMSERVER_HOSTNAME=$VMSERVER_HOSTNAME, TARGET_ROOT_DISK_SIZE=$TARGET_ROOT_DISK_SIZE"

    #  Guts
    echo "vol=$TARGET_HOSTNAME/$TARGET_HOSTNAME"
    return 0
}

loading purge
old_purge()
{
    rm -f $VAR_DIR/$PROGNAME/*
}

loading hostnic2ipaddr
old_hostnic2ipaddr()
{
    local HOSTNAME NIC

    HOSTNAME=$1
    NIC=$2
    IPADDR_REF=$3

    check_ssh_access $HOSTNAME || return $?
    IPADDR_LOCAL=$(ssh -n $HOSTNAME ip -4 -o addr show dev $NIC | sed 's/.*inet \(.*\)\/.*/\1/') || return $?

    eval "$IPADDR_REF=\"$IPADDR_LOCAL\""
    return 0
}

loading send_email
old_send_email()
{
    /usr/lib/sendmail -t
}

loading usage
old_usage()
{
    local RC

    RC=${1:-1}

    {
        echo "Usage: $PROGNAME [ <options> ] { create | delete | list }"
    } | if [ $RC = 0 ]; then
        cat 
    else
        cat >&2
    fi

    exit $RC
}