#!/bin/bash # $HeadURL: https://svn.pasta.freemyip.com/main/miniade/trunk/bin/nop-sh $ $LastChangedRevision: 10133 $ # Modules . $(miniade) || { echo "${0##*/}: ERROR: miniade failed (hint: run 'miniade' to see error)" >&2; exit 1; } # Configurable stuff # Other globals main() { local MY_ARGS local MODE RECIPE_TITLE OUTPUT_FILE # Defaults for options WORK_DIR= DOWNLOAD_CACHE_DIR= WGET_OPTS= EXIT_AFTER_DOWNLOAD_FLAG=false LOCK_DIR= TIMESTAMPS_FLAG=false MAX_NODES_PER_TILE=default # Process options special_opts_handler() { case $1 in --work-dir=*) WORK_DIR=${1#*=} ;; --lock-dir=*) LOCK_DIR=${1#*=} ;; --out-dir=*) OUT_DIR=${1#*=} ;; --download-cache-dir=*) DOWNLOAD_CACHE_DIR=${1#*=} ;; --family-id=*) FAMILY_ID=${1#*=} ;; --country-name=*) COUNTRY_NAME=${1#*=} ;; --country-abbr=*) COUNTRY_ABBR=${1#*=} ;; --overview-name=*) OVERVIEW_NAME=${1#*=} ;; --exit-after-download) EXIT_AFTER_DOWNLOAD_FLAG=true ;; --osm-input-url=*) OSM_INPUT_URL=${1#*=} ;; --cores=*) CORES=${1#*=} ;; --mbs=*) MBS=${1#*=} ;; --splitter-url=*) SPLITTER_URL=${1#*=} ;; --mkgmap-url=*) MKGMAP_URL=${1#*=} ;; --sea-url=*) SEA_URL=${1#*=} ;; --bounds-url=*) BOUNDS_URL=${1#*=} ;; --timestamps) TIMESTAMPS_FLAG=true ;; --max-nodes-per-tile=*) MAX_NODES_PER_TILE=${1#*=} ;; --wget-opts=*) WGET_OPTS=${1#*=} ;; *) return 1 ;; esac } miniade_process_options --help-handler=help --special-opts-handler=special_opts_handler MY_ARGS "$@" && set -- "${MY_ARGS[@]}" # Process arguments [ $# = 0 ] || miniade_bad_usage # Sanity checks and derivations # loading modules in the job-submitter's enviroment does not # propogate them into the job-runner's enviroment. So we need # to do this inside the job script. Other people should comment # these two lines out. module purge module load jdk for VAR in WORK_DIR OUT_DIR LOCK_DIR FAMILY_ID COUNTRY_NAME COUNTRY_ABBR OVERVIEW_NAME OSM_INPUT_URL CORES MBS SPLITTER_URL MKGMAP_URL SEA_URL BOUNDS_URL MAX_NODES_PER_TILE; do eval "[ \"X\$$VAR\" != X ]" || miniade_bad_usage done ! [[ $OVERVIEW_NAME =~ ' ' ]] || miniade_error "$OVERVIEW_NAME: overview names are not allowed to contain spaces" [[ $MAX_NODES_PER_TILE =~ ^(default|[0-9]+)$ ]] || miniade_error "$MAX_NODES_PER_TILE: max nodes per tile can be numeric or 'default'" #MAPNAME=$(while :; do X=$RANDOM; [ $X -ge 10000 ] || break; done; printf "%08d\n" $((X*10000+1))) MAPNAME="${FAMILY_ID}0001" INSTALL_DIR=Garmin/OSM-$COUNTRY_NAME-$(date +%Y-%m) FAMILY_NAME=OSM_$COUNTRY_ABBR DESCRIPTION=OSM_$COUNTRY_ABBR SERIES_NAME="OSM $COUNTRY_NAME ($(date '+%Y %m'))" REG_SERIES_NAME=$(echo "$SERIES_NAME" | sed -e 's/[^A-Za-z0-9_ ]//g') WINDOWS_INSTALL_DRIVE="C:" FAMILY_ID_LSB=$(printf "%02x\n" $(($FAMILY_ID % 256))) FAMILY_ID_MSB=$(printf "%02x\n" $(($FAMILY_ID / 256))) OSM_INPUT_FILE=$WORK_DIR/${OSM_INPUT_URL##*/} # Lock out other accidentally launched jobs attempting to process the same data. miniade_lock /tmp/$PROGAME.$COUNTRY_ABBR.lock || return 1 # Ensure re-entrant miniade_debug 10 "main: ensuring re-entrant ..." rm -fr $WORK_DIR mkdir -p $WORK_DIR $OUT_DIR [ "X$DOWNLOAD_CACHE_DIR" = X ] || mkdir -p $DOWNLOAD_CACHE_DIR cd $WORK_DIR # Download splitter and mkgmap miniade_debug 10 "main: downloading and unpacking splitter and gmap maker ..." wget_wrapper $WGET_OPTS $SPLITTER_URL || return $? wget_wrapper $WGET_OPTS $MKGMAP_URL || return $? # Download map data miniade_debug 10 "main: downloading map data ..." wget_wrapper $WGET_OPTS $OSM_INPUT_URL || return $? wget_wrapper $WGET_OPTS $BOUNDS_URL || return $? wget_wrapper $WGET_OPTS $SEA_URL || return $? # All downloads must be done above! We assume no internet connection below. (This is due to COBRA design.) ! $EXIT_AFTER_DOWNLOAD_FLAG || return 0 # Unpack splitter and mkgmap mkdir -p $WORK_DIR/splitter/distrib ( cd $WORK_DIR/splitter/distrib unzip -q $WORK_DIR/${SPLITTER_URL##*/} ) mkdir -p $WORK_DIR/mkgmap/distrib ( cd $WORK_DIR/mkgmap/distrib unzip -q $WORK_DIR/${MKGMAP_URL##*/} ) # Can't assign this earlier as contains *, which we expand here 'cos we can't predict it. STYLE_DIR=$(echo $WORK_DIR/mkgmap/distrib/mkgmap-*/examples/styles/default) # Modify mkgmap style to suit personal taste miniade_debug 10 "main: modifing mkgmap style to suit personal taste ..." # Comment out power and boundary specifications perl -pi -e 's/^(power|boundary)/#$1/' $STYLE_DIR/lines SPLITTER_WRAPPER_CMDLINE="splitter_wrapper --mapid=$MAPNAME --search-limit=1000000 $([ $MAX_NODES_PER_TILE = default ] || echo "--max-nodes=$MAX_NODES_PER_TILE") $OSM_INPUT_FILE" miniade_debug 2 "main: calling [$SPLITTER_WRAPPER_CMDLINE] ..." { ! type -p hpcmd >/dev/null; } || hpcmd -m "splitter starts" eval "$SPLITTER_WRAPPER_CMDLINE" > splitter_wrapper.$$.log 2>&1 # Convert OSM to Garmin format. Changes from last time: # - make sea blue # - drop political boundaries (they look like roads on small screens) # - drop power lines (they look like roads ...) # - let mkgmap decide for itself how many cores to use MKGMAP_WRAPPER_CMDLINE="mkgmap_wrapper --code-page=1252 --latin1 \ --route --remove-short-arcs --add-pois-to-areas \ --index --location-autofill=is_in,nearest --tdbfile \ --mapname=$MAPNAME \ --family-id=$FAMILY_ID --family-name=\"$FAMILY_NAME\" \ --overview-mapname=$OVERVIEW_NAME \ --series-name=\"$SERIES_NAME\" --description=\"$DESCRIPTION\" \ --bounds=$WORK_DIR/${BOUNDS_URL##*/} \ --precomp-sea=$WORK_DIR/${SEA_URL##*/} \ --style-file=$STYLE_DIR \ -c template.args" miniade_debug 2 "main: calling [$MKGMAP_WRAPPER_CMDLINE] ..." { ! type -p hpcmd >/dev/null; } || hpcmd -m "mkgmap starts" eval "$MKGMAP_WRAPPER_CMDLINE" > mkgmap_wrapper.$$.log 2>&1 # Create staging area miniade_debug 10 "main: creating staging area ..." BASE_DIR="$(basename "$INSTALL_DIR")" mkdir -p "$BASE_DIR" # Populate staging area miniade_debug 10 "main: populating staging area ($BASE_DIR/$OVERVIEW_NAME.reg) ..." cat > "$BASE_DIR"/$OVERVIEW_NAME.reg < "$BASE_DIR"/INSTALL.txt </dev/null; } || hpcmd -m "zip starts" zip -r9 "$OUT_DIR/$BASE_DIR.zip" "$BASE_DIR" # Clean up miniade_debug 10 "main: cleaning up ..." rm -fr $WORK_DIR } help() { local PROGNAME miniade_get_progname PROGNAME echo "Usage: $PROGNAME [ ] [ --exit-after-download ] --lock-dir= --work-dir= --out-dir= [ --download-cache-dir= ] --family-id= --country-name= --country-abbr= --overview-name= --url= --cores= --mbs= --splitter-version= --mkgmap-version= --sea-version= --bounds-version=" exit 0 } splitter_wrapper() { miniade_debug 10 "splitter_wrapper: calling [$*] ..." # I think Java is narrowly exceeding the memory threshold that it has been # given, or perhaps some other part of a job is. So we actually allow only # 75% of MBS. # 3/4 works # 9/10 works # 19/20 miniade_debug 10 "splitter_wrapper: calling [java -Xmx$((MBS * 19 / 20))M -jar $WORK_DIR/splitter/distrib/splitter-*/splitter.jar $*] ..." java -Xmx$((MBS * 19 / 20))M -jar $WORK_DIR/splitter/distrib/splitter-*/splitter.jar "$@" } mkgmap_wrapper() { miniade_debug 10 "mkgmap_wrapper: calling [$*] ..." java -Xmx$((MBS * 19 / 20))M -jar $WORK_DIR/mkgmap/distrib/mkgmap-*/mkgmap.jar "$@" } wget_wrapper() { local CACHED_FLAG FILE PROGNAME FILE=$(echo "$@" | sed 's/.*\///') # There's a real danger of simultaneously wgetting the same file multiple times # (e.g. run this script in parallel for different maps and they'll all download # the splitter.jar first). So we need some wait-for-lock-style locking. miniade_get_progname PROGNAME lock_with_timeout /tmp/$PROGNAME.${FILE//\//_}.lock 180 || return 1 miniade_debug 10 "wget_wrapper: $FILE: determining if in cache ..." if [ "X$DOWNLOAD_CACHE_DIR" = X ]; then CACHED_FLAG=false elif [ -f $DOWNLOAD_CACHE_DIR/$FILE ]; then CACHED_FLAG=true else CACHED_FLAG=false fi miniade_debug 10 "wget_wrapper: CACHED_FLAG=$CACHED_FLAG" if $CACHED_FLAG; then miniade_debug 10 "wget_wrapper: $FILE: hard-linking out of cache ..." ln $DOWNLOAD_CACHE_DIR/$FILE . || return $? else miniade_debug 10 "wget_wrapper: $FILE: not in cache or cache dir not specified; downloading (wget $*) ..." wget "$@" || return $? miniade_debug 10 "wget_wrapper: $FILE: done" if [ "X$DOWNLOAD_CACHE_DIR" != X ]; then miniade_debug 10 "wget_wrapper: $FILE: download complete; caching [pwd=$(pwd); cmd='cp $FILE $DOWNLOAD_CACHE_DIR/' ..." cp $FILE $DOWNLOAD_CACHE_DIR/ fi fi miniade_unlock /tmp/$PROGNAME.${FILE//\//_}.lock } lock_with_timeout() { local LOCKWHAT TIMEOUT GIVE_UP_TIMESTAMP [ $# = 2 ] || miniade_internal "lock_with_timeout: $#: bad argument count" LOCKWHAT=$1 TIMEOUT=$2 GIVE_UP_TIMESTAMP=$((TIMEOUT+$(date +%s))) while :; do miniade_debug 10 "lock_with_timeout: trying to lock '$LOCKWHAT' ..." if miniade_lock $LOCKWHAT; then miniade_debug 10 "lock_with_timeout: lock succeeded; returning ..." return 0 elif [ $(date +%s) -ge $GIVE_UP_TIMESTAMP ]; then miniade_debug 10 "lock_with_timeout: lock failed and timeout has expired; returning ..." return 1 else miniade_debug 10 "lock_with_timeout: lock failed but timeout has not expired yet; sleeping before trying again ..." sleep $((RANDOM % 10)) fi done } main "$@"