tools/geo-nearest

   1 #!/bin/bash
   2 #
   3 #
   4 #       DO NOT EDIT! This file is generated from geo-nearest.sh
   5 #
   6
   7 #
   8 #       geo-nearest: Fetch list of nearest geocaches.
   9 #
  10 #       Requires: curl; gpsbabel; bash or ksh;
  11 #                 mysql (if using the gpsdrive.sql output option)
  12 #
  13 #       Donated to the public domain by Rick Richardson <rickrich@gmail.com>
  14 #
  15 #       Use at your own risk.  Not suitable for any purpose.  Not legal tender.
  16 #
  17 #       $Id: geo-nearest.sh,v 1.49 2013/02/18 21:41:06 rick Exp $
  18 #
  19
  20 PROGNAME="$0"
  21
  22 usage() {
  23         cat <<EOF
  24 NAME
  25     `basename $PROGNAME` - Fetch a list of nearest geocaches
  26
  27 SYNOPSIS
  28     `basename $PROGNAME` [options]
  29
  30     `basename $PROGNAME` [options] latitude longitude
  31
  32     `basename $PROGNAME` [options] latitude longitude cache-type
  33
  34     `basename $PROGNAME` [options] zipcode
  35
  36     `basename $PROGNAME` [options] u=<username>
  37
  38     `basename $PROGNAME` [options] ul=<username>
  39
  40     `basename $PROGNAME` [options] pq=<pocket-query>
  41
  42     `basename $PROGNAME` [options] tx=<bookmark-id>
  43
  44     `basename $PROGNAME` [options] -b bookmark
  45     `basename $PROGNAME` [options] guid=<bookmark-id>
  46
  47 DESCRIPTION
  48     Fetch a list of nearest geocaches.
  49
  50     Requires:
  51         A premium member (\$30/yr) OR a basic member (free) login at:
  52              http://www.geocaching.com
  53         Visit a cache page and click the "Download to EasyGPS" link at least
  54         once so you can read and agree to the license terms.  Otherwise, you
  55         will not get any waypoint data.
  56
  57         curl            http://curl.haxx.se/
  58         gpsbabel        http://gpsbabel.sourceforge.net/
  59
  60 EOF
  61         gc_usage
  62         cat << EOF
  63
  64 NOTE
  65     A basic member will get caches very slow (20 cache pages per minute)
  66     because we have to get the actual cache pages.  They will be stored in:
  67         ~/.geo/caches/GCXXXX.html.
  68     Of course, after running this command, geo-html2gpx could be run.
  69
  70 EXAMPLES
  71     Nearest 20 caches, display shortnames:
  72
  73         geo-nearest -s
  74
  75     Search nearest 500 caches for virtual caches not yet found:
  76
  77         geo-nearest -n500 -Ivirtual -Xifound
  78
  79     Add nearest 50 caches to a GpsDrive SQL database
  80
  81         geo-nearest -n50 -f -s -S
  82
  83     Purge the existing SQL database of all geocaches, and fetch
  84     200 fresh ones...
  85
  86         geo-nearest -S -P -s -n200
  87
  88     640x480 map of nearest caches using map source 2:
  89
  90         geo-nearest -omap,"-a2 -W640 -H480"
  91
  92     Copy two cachers:
  93
  94         geo-nearest -n200 -Xifound -udyl1231 -pPW | awk '{print \$1}' >1.foo
  95         geo-nearest -n200 -Xifound -urickrich -pPW |awk '{print \$1}' >2.foo
  96         geo-gid -otabsep \$(comm -12 1.foo 2.foo) >both
  97
  98     Fetch by owner placed:
  99
 100         geo-nearest u=team-deadhead
 101
 102     Fetch by owner found:
 103
 104         geo-nearest ul="AAA+of+MichigAn&sortdir=asc&sort=placed"
 105
 106     Fetch by tx method:
 107
 108         # nearby caches of this (puzzle) type, that I haven't found
 109         geo-nearest -n500 -f -otabsep tx=40861821-1835-4e11-b666-8d41064d03fe |
 110             geo-mystery >> Caches/rick.ts
 111
 112         Also, tx=webcam, tx=earth, tx=multi, tx=event, tx=virtual, tx=letter,
 113         tx=unknown, tx=trad (tx=reg is an alias).
 114
 115     Fetch by cache-type method:
 116
 117         # nearby caches of this (puzzle) type, that I haven't found
 118         geo-nearest -n500 -f -otabsep unknown |
 119             geo-mystery >> Caches/rick.ts
 120
 121         Also, cache-type is webcam, earth, multi, event, virtual, letter,
 122         unknown, trad (reg is an alias).
 123
 124     Fetch a bookmark list:
 125
 126         geo-nearest -b acro
 127         or
 128         geo-nearest guid=baae5bf9-4315-4874-b7fb-ac84c9585641
 129
 130     Fetch a PQ query:
 131
 132         geo-nearest -q "Needs Maintenance"
 133         or
 134         geo-nearest pq=08be103b-ffd1-4e27-992f-616e144e24df
 135
 136 FILES
 137     ~/.georc
 138     ~/.geo/caches/
 139
 140 SEE ALSO
 141     geo-newest, geo-found, geo-placed, geo-keyword, geo-code, geo-map,
 142     geo-waypoint,
 143     $WEBHOME
 144 EOF
 145
 146         exit 1
 147 }
 148
 149 ##############################################################################
 150 # begin #include "geo-common"
 151 ##############################################################################
 152
 153 # I doubt this stuff will work in other than english
 154 LANG=en_US
 155
 156 #
 157 #       Common global constants
 158 #
 159 UA="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)"
 160 WEBHOME="http://geo.rkkda.com/"
 161
 162 #
 163 #       Common global variables
 164 #
 165 DEBUG=0
 166 CRUFT=
 167 CURL_OPTS=
 168
 169 #
 170 #       Report an error and exit
 171 #
 172 error() {
 173         echo "`basename $PROGNAME`: $1" >&2
 174         exit 1
 175 }
 176
 177 debug() {
 178         if [ $DEBUG -ge $1 ]; then
 179             echo "`basename $PROGNAME`: $2" >&2
 180         fi
 181 }
 182
 183 verbose() {
 184         if [ $VERBOSE -ge $1 ]; then
 185             echo "$2" >&2
 186         fi
 187 }
 188
 189 dbgcmd() {
 190         if [ $DEBUG -ge $DBGCMD_LVL ]; then
 191             echo "$@" >&2
 192         fi
 193         "$@"
 194 }
 195 DBGCMD_LVL=2
 196
 197 #
 198 #       procedure to remove cruft files
 199 #
 200 remove_cruft() {
 201         if [ $DEBUG = 0 -a "$CRUFT" != "" ]; then
 202             for i in $CRUFT
 203             do
 204                     [ -f $i ] && rm -f $i
 205             done
 206         fi
 207 }
 208
 209 trap remove_cruft EXIT
 210
 211 #
 212 # Convert DegDec, MinDec, or DMS lat/lon to DegDec
 213 #
 214 latlon() {
 215     # Handle NSEW prefixes
 216     arg1=`echo "$1" | sed -e 's/^[nNeE]//' -e 's/^[wW]/-/' -e 's/^[sS]/-/'`
 217     # If negative, print the sign then take the absolute value
 218     case "$arg1" in
 219     -*) echo -n "-"; arg1=`echo "$arg1" | sed 's/^-//'`;;
 220     esac
 221     # Now handle the 3 different formats
 222     case $# in
 223     1)
 224         case "$arg1" in
 225         *.*.*)  echo "$arg1" \
 226                 | sed -e 's/,//' -e 's#\([^.]*\)\.#\1 #' -e 's#$# 6k 60/+p#' \
 227                 | dc
 228                 ;;
 229         lat=*)
 230                 echo "$arg1" | sed 's/^lat=//'
 231                 ;;
 232         lon=*)
 233                 echo "$arg1" | sed 's/^lon=//'
 234                 ;;
 235         *)      echo $arg1
 236                 ;;
 237         esac
 238         ;;
 239     2)  echo "6k $arg1 $2 60/+p" | dc;;
 240     3)  echo "6k $arg1 $2 60/ $3 3600/++p" | dc;;
 241     esac
 242 }
 243
 244 #
 245 #       Convert DegDec to MinDec with optional NS/EW
 246 #
 247 degdec2mindec() {
 248     awk -v v=$1 -v sym=$2 \
 249     '
 250     function abs(x)     { return (x>=0) ? x : -x }
 251     BEGIN{
 252         i=int(v)
 253         if (sym == "")
 254             printf "%d.%06.3f\n", i, abs(v-i) * 60
 255         else
 256             printf "%s%d.%06.3f\n", \
 257                 i >= 0.0 ? substr(sym, 1, 1) : substr(sym, 2, 1), \
 258                 abs(i), abs(v-i) * 60
 259     }'
 260 }
 261
 262 #
 263 #       Read RC file, if there is one
 264 #
 265 read_rc_file() {
 266     if [ -f $HOME/.georc ]; then
 267         . $HOME/.georc
 268         # Allow LAT/LON in rc file to be in any of the formats that we grok
 269         if [ "" != "$LAT" ]; then
 270             if ! is_latlon $LAT 0; then
 271                 error "Latitude '$LAT' is not parsed in .georc!"
 272             fi
 273             LAT=`latlon $LAT`
 274         fi
 275         if [ "" != "$LON" ]; then
 276             if ! is_latlon 0 $LON; then
 277                 error "Longitude '$LON' is not parsed in .georc!"
 278             fi
 279             LON=`latlon $LON`
 280         fi
 281     else
 282         cat <<-EOF > $HOME/.georc
 283                 #
 284                 # These are the default values for the geo-* series of programs
 285                 # Please edit this file as needed.  Setting values for
 286                 # USERNAME, PASSWORD, LAT/LON, and STATE are required.
 287                 #
 288
 289                 #################################
 290                 # Login and paid membership status for www.geocaching.com...
 291                 #USERNAME=name
 292                 #PASSWORD=pasword
 293                 #SOC=0
 294
 295                 #################################
 296                 # Your HOME lat/lon and state...
 297                 #LAT=N44.55.666
 298                 #LON=W93.11.222
 299                 #STATE=MN
 300
 301                 #################################
 302                 # Default map scale, font, and source...
 303                 #MAPSCALE=10K
 304                 #MAPFONT=helvetica
 305                 #MAPSRC=2
 306
 307                 #################################
 308                 # Login for terraserver.com...
 309                 #TSCOM_EMAIL=xxx@yyy.com
 310                 #TSCOM_PW=password
 311
 312                 #################################
 313                 # Miscellaneous...
 314                 #OUTFMT=gpsdrive
 315                 #GPSDRIVE_VER=2.09
 316                 #GEOMYSTERY=$HOME/.geo-mystery
 317                 #OCMYSTERY=$HOME/.oc-mystery
 318                 #DATEFMT=0
 319                 #CURL_OPTS="-4"
 320                 #CURL_OPTS="--sslv3"
 321         EOF
 322         error "First time user: please review and edit $HOME/.georc"
 323     fi
 324 }
 325
 326 if [ `uname` = 'Darwin' ]; then
 327     sed=gsed
 328     date=gdate
 329     touch=gtouch
 330     PATH=$PATH:/usr/local/bin:/opt/local/bin
 331     export PATH
 332 else
 333     sed=sed
 334     date=date
 335     touch=touch
 336 fi
 337
 338 #
 339 #       Get the value from a name= value= pair in a file
 340 #
 341 get_value() {
 342     # <input type="hidden" name="__EVENTTARGET" value=""
 343     what=$1
 344     where=$2
 345     eval $what=`$sed -n "s/^.*\"$what\" *value=\"\([^\"]*\)\".*/\1/p" < $where`
 346 }
 347
 348 #
 349 #       urlencode
 350 #
 351 #       incomplete, just does what we need it to do
 352 #
 353 urlencode() {
 354     echo "$1" | sed -e 's/\+/%2B/g' -e 's/\&/%26/g' #-e 's/\//%2F/'
 355 }
 356
 357 #
 358 #       urlencode2
 359 #
 360 urlencode2() {
 361     echo "$1" |
 362     awk '
 363         BEGIN {
 364             split("1 2 3 4 5 6 7 8 9 A B C D E F", hextab, " ")
 365             hextab[0] = 0
 366             for (i = 1; i <= 255; ++i) ord[ sprintf ("%c", i) "" ] = i + 0
 367         }
 368         {
 369             encoded = ""
 370             for (i = 1; i <= length($0); ++i ) {
 371                 c = substr ($0, i, 1)
 372                 val = ord[c]
 373                 if (val >= 97 && val <= 122)            #0x61-0x7A
 374                     encoded = encoded c
 375                 else if (val >= 65 && val <= 90)        #0x41-0x5A
 376                     encoded = encoded c
 377                 else if (val >= 48 && val <= 57)        #0x30-0x39
 378                     encoded = encoded c
 379                 else if (val >= 45 && val <= 46)        #0x2D-0x2E
 380                     encoded = encoded c
 381                 else if (c == " ")
 382                     encoded = encoded "+"
 383                 else if (val < 128) {
 384                     lo = val % 16
 385                     hi = int(val / 16);
 386                     encoded = encoded "%" hextab[hi] hextab[lo]
 387                 }
 388                 else {
 389                     byte = 192 + val/64
 390                     lo = byte % 16
 391                     hi = int(byte / 16);
 392                     encoded = encoded "%" hextab[hi] hextab[lo]
 393                     byte = 128 + val%64
 394                     lo = byte % 16
 395                     hi = int(byte / 16);
 396                     encoded = encoded "%" hextab[hi] hextab[lo]
 397                 }
 398             }
 399             print encoded
 400         }
 401     '
 402 }
 403
 404 #
 405 #       return true if current arguments appear to be a lat/lon
 406 #
 407 is_latlon() {
 408     if [ "$#" -lt 2 ]; then
 409         return 1
 410     fi
 411     case "$1" in
 412     lat=*)              ;;              # cut/paste from GPX file
 413     [NS])               return 0;;      # cut/paste from gc.com
 414     [NSns][0-9]*)       ;;
 415     [-][0-9]*)          ;;
 416     [0-9]*)             ;;
 417     *)                  return 1;;
 418     esac
 419     case "$2" in
 420     lon=*)              return 0;;
 421     [EWew][0-9]*)       return 0;;
 422     [-][0-9]*)          return 0;;
 423     [0-9]*)             return 0;;
 424     *)                  return 1;;
 425     esac
 426 }
 427
 428 #
 429 #       split lines between two strings
 430 #
 431 #       $1 - string 1
 432 #       $2 - string 2
 433 #       $3 - null or 'g'
 434 #
 435 split_lines_between() {
 436     sed "s@$1$2@$1\\
 437 $2@$3"
 438 }
 439
 440 ##############################################################################
 441 # end #include "geo-common"
 442 ##############################################################################
 443 ##############################################################################
 444 # begin #include "geo-common-gc"
 445 #
 446 # $Id: geo-common-gc,v 1.313 2013/02/01 14:29:33 rick Exp $
 447 ##############################################################################
 448
 449 #
 450 #       Common global constants
 451 #
 452 UA="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)"
 453 UA="Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/XX (KHTML, like Gecko) Version/ZZ Safari/YY"
 454 GEO="http://www.geocaching.com"
 455 GEOS="https://www.geocaching.com"
 456
 457 #
 458 #       Global variables that can be overridden on command line or rc file
 459 #
 460 PASSWORD=dummy
 461 USERNAME=dummy
 462 LAT=44.9472
 463 LON=-93.4914
 464 SOC=${SOC:0}
 465 GEOMYSTERY=/dev/null
 466 GEODIR=$HOME/.geo
 467 case `uname` in
 468 CYGWIN*)        CURL_OPTS=-k;;
 469 *)              CURL_OPTS=
 470 esac
 471
 472 #
 473 #       Global variables
 474 #
 475 COOKIE_FILE=$HOME/.geocookies
 476 NOCOOKIES=0
 477 FOUND=1
 478 USERFOUND=1
 479 BABELFLAGS=
 480 RADIUS=
 481 RADIUS_NUM=
 482 RADIUS_MILES=
 483 OUTFILE=
 484 OUTFMT=gpsdrive
 485 NUM=20
 486 INCLUDE=*
 487 EXCLUDE='-unavail'
 488 VARTIME=found
 489 GEOSLEEP=${GEOSLEEP:-8}
 490
 491 #
 492 #       Common options handling
 493 #
 494 gc_usage() {
 495         cat <<EOF
 496 OPTIONS
 497         -b bookmark     Use list "bookmark" [none]
 498         -q query        geo-nearest: Use PQ list "query" [none]
 499         -q search       geo-newest: Use search "search" [none]
 500         -c              Remove cookie file when done
 501         -f              Do not report any found or unavailable caches
 502         -m              Do not report any members-only caches
 503         -F              Report caches found by the login 'username' as unfound
 504         -n num          Return "num" caches [$NUM]
 505         -s              Output short names for the caches (gpsbabel option)
 506         -I term         Include only caches with 'term' [$INCLUDE]
 507         -X term         Exclude caches with 'term' [$EXCLUDE]
 508                         terms: ~ (exclude none), unfound, ifound, soc, unavail,
 509                         regular, multi, virtual, webcam, event, hybrid, cito
 510         -r radius       Display only caches with radius (e.g. -r 25M)
 511         -M mystery      Use file 'mystery' for unknown/mystery/puzzle
 512                         caches [$GEOMYSTERY]. Awk Format:
 513
 514                         gcid lat lon comment
 515
 516                         i.e.: GC2CBVB n44.45.123 w93.00.321       Final
 517
 518         -u username     Username for http://www.geocaching.com
 519         -p password     Password for http://www.geocaching.com
 520         -o format       Output format, -o? for possibilities [$OUTFMT]
 521                         plus "gpsdrive.sql" for direct insertion into MySQL DB
 522                         plus "map[,geo-map-opts]" to display a geo-map.
 523         -O filename     Output file, if not stdout
 524         -S              Alias for -o gpsdrive.sql
 525         -d              For -S, just delete selected records
 526         -P              For -S, purge all records of type -t $SQLTAG*
 527         -t type         For -ogpsdrive.sql, the waypoint type [$SQLTAG]
 528         -H htmldir      Also fetch the printable HTML pages (slowly)
 529         -L logdir       Also fetch the plain text log entries (slowly)
 530                         For -H or -L, the limit is 1500 updated caches/day.
 531         -! "lpr -Plp"   Print HTML pages
 532         -E var=val      Set environment "var" to "val"
 533                         i.e. DATEFMT=0|1
 534         -D lvl          Debug level [$DEBUG]
 535         -U              Retrieve latest version of this script
 536
 537 DEFAULTS
 538         Defaults can also be set with variables in file \$HOME/.georc:
 539
 540             PASSWORD=password;  USERNAME=username;   SOC=0|1;
 541             LAT=latitude;       LON=logitude;        GEOMYSTERY=/dev/null;
 542             NUM=num;            OUTFMT=format;       BABELFLAGS=-s;
 543             SQLUSER=gast;       SQLPASS=gast;        SQLDB=geoinfo;
 544             DATEFMT=[0|1];
 545 DATE FORMATS
 546         Geocaching.com date formats that are compatible:
 547
 548             GC Format   Example     Compatible
 549             YYYY-MM-DD  2011-07-13  yes
 550             YYYY/MM/DD  2011/07/13  yes
 551             MM/DD/YYYY  07/13/2011  yes
 552             DD/MM/YYYY  13/07/2011  yes if DATEFMT=1 in \$HOME/.georc
 553             DD/Mmm/YYYY 13/Jul/2001 no
 554             Mmm/DD/YYYY Jul/13/2011 no
 555             DD Mmm YY   13 Jul 11   yes (english only)
 556
 557         Change them here:
 558
 559             http://www.geocaching.com/account/ManagePreferences.aspx
 560 EOF
 561 }
 562
 563 gc_getopts() {
 564     #
 565     # Defaults for options that cannot be overriden in the RC file
 566     #
 567     PURGE=0
 568     DELETE=0
 569     SQL=0
 570     MAP=0
 571     HTMLDIR=
 572     LOGDIR=
 573     CMDPIPE=
 574     BOOKMARK=
 575     POCKETQUERY=
 576
 577     while getopts "!:E:H:L:I:X:b:q:cdfFmM:n:o:O:p:Pr:sSt:u:D:Uh?-" opt
 578     do
 579         case $opt in
 580         "!")    CMDPIPE="$OPTARG";;
 581         b)      BOOKMARK="$OPTARG";;
 582         q)      POCKETQUERY="$OPTARG";;
 583         c)      NOCOOKIES=1;;
 584         d)      DELETE=1;;
 585         E)      eval "$OPTARG";;
 586         f)      FOUND=0; EXCLUDE="$EXCLUDE|-ifound";;
 587         m)      SOC=0;;
 588         M)      GEOMYSTERY="$OPTARG";;
 589         I)
 590                 if [ "$INCLUDE" = "*" ]; then
 591                     INCLUDE=
 592                 else
 593                     INCLUDE="$INCLUDE|"
 594                 fi
 595                 INCLUDE="$INCLUDE-$OPTARG"
 596                 ;;
 597         X)
 598                 if [ "" = "$OPTARG" -o "~"  = "$OPTARG" ]; then
 599                     EXCLUDE=-ExClUdEnOtHiNg
 600                 else
 601                     EXCLUDE="$EXCLUDE|-$OPTARG"
 602                 fi
 603                 ;;
 604         F)      USERFOUND=0;;
 605         P)      PURGE=1;;
 606         n)      NUM="$OPTARG"
 607                 case "$NUM" in
 608                 [0-9]*) ;;
 609                 *)      error "Not a number: '$NUM'";;
 610                 esac
 611                 if [ "$NUM" -lt 1 -o "$NUM" -gt 999999 ]; then
 612                     error "NUM is not between 1 and 999999 ($NUM)"
 613                 fi
 614                 ;;
 615         r)      RADIUS="$OPTARG"
 616                 RADIUS_NUM=`awk -v "N=$RADIUS" 'BEGIN{printf "%d\n", N}'`
 617                 case "$RADIUS" in
 618                 *km*|*KM*)
 619                     RADIUS_UNITS=km
 620                     RADIUS_MILES=`dc -e "6k $RADIUS_NUM 1.609344/p" `
 621                     ;;
 622                 *)
 623                     RADIUS_MILES=$RADIUS_NUM
 624                     RADIUS_UNITS=mi
 625                     ;;
 626                 esac
 627                 ;;
 628         s)      BABELFLAGS="$BABELFLAGS -s"; GEONUKE=",nuke_placer";;
 629         S)      OUTFMT="gpsdrive.sql";;
 630         t)      SQLTAG="$OPTARG";;
 631         u)      USERNAME="$OPTARG";;
 632         p)      PASSWORD="$OPTARG";;
 633         o)      OUTFMT="$OPTARG";;
 634         O)      OUTFILE="$OPTARG";;
 635         H)      HTMLDIR="$OPTARG";;
 636         L)      LOGDIR="$OPTARG";;
 637         D)      DEBUG="$OPTARG";;
 638         U)      echo "Getting latest version of this script..."
 639                 curl $CURL_OPTS -o$UPDATE_FILE "$UPDATE_URL"
 640                 chmod +x "$UPDATE_FILE"
 641                 echo "Latest version is in $UPDATE_FILE"
 642                 exit
 643                 ;;
 644         h|\?|-) usage;;
 645         esac
 646     done
 647
 648     shiftamt=`expr $OPTIND - 1`
 649     shift $shiftamt
 650
 651     case "$SOC" in
 652     0)  EXCLUDE="$EXCLUDE|-soc";;
 653     esac
 654
 655     case "$OUTFMT" in
 656     map)
 657         OUTFMT=tiger,newmarker=grnpin,iconismarker
 658         MAP=1
 659         ;;
 660     map,*)
 661         MAPOPTS=`echo "$OUTFMT" | $sed -n 's/map,\(.*\)$/\1/p'`
 662         OUTFMT=tiger,newmarker=grnpin,iconismarker
 663         MAP=1
 664         ;;
 665
 666     gpsdrive.sql)
 667         OUTFMT=gpsdrive
 668         SQL=1
 669         # DEBUG=1
 670         ;;
 671     \?)
 672         gpsbabel_formats
 673         exit
 674         ;;
 675     esac
 676
 677     LOGUSERNAME="$USERNAME"
 678     return $shiftamt
 679 }
 680
 681 #
 682 #       Get viewstate
 683 #
 684 gc_getviewstate() {
 685     file=$1
 686     get_value __EVENTTARGET $file
 687     get_value __EVENTARGUMENT $file
 688     get_value __VIEWSTATEFIELDCOUNT $file
 689     get_value __VIEWSTATE $file; __VIEWSTATE=`urlencode "$__VIEWSTATE"`
 690     get_value __EVENTVALIDATION $file;
 691         __EVENTVALIDATION=`urlencode "$__EVENTVALIDATION"`
 692     viewstate=
 693     viewstate="$viewstate -d __VIEWSTATE=$__VIEWSTATE"
 694     if [ "$__VIEWSTATEFIELDCOUNT" != "" ]; then
 695         viewstate="$viewstate -d __VIEWSTATEFIELDCOUNT=$__VIEWSTATEFIELDCOUNT"
 696         fc=1
 697         while test $fc -lt $__VIEWSTATEFIELDCOUNT; do
 698             get_value __VIEWSTATE$fc $file
 699             eval xxx=\${__VIEWSTATE$fc}
 700             eval __VIEWSTATE$fc=`urlencode "$xxx"`
 701             eval xxx=\${__VIEWSTATE$fc}
 702             viewstate="$viewstate -d __VIEWSTATE$fc=$xxx"
 703             fc=`expr $fc + 1`
 704         done
 705     fi
 706 }
 707
 708 #
 709 #       login to geocaching.com
 710 #
 711 #       Outputs: $__VIEWSTATE
 712 #
 713 gc_login() {
 714     _username=`urlencode "$1"`
 715     _password=`urlencode "$2"`
 716
 717     [ "$_username" != dummy ] || error "You need a www.geocaching.com username"
 718     [ "$_password" != dummy ] || error "You need a www.geocaching.com password"
 719
 720     #
 721     #   Get the login page so we can get a valid viewstate
 722     #
 723     LOGINPAGE=${TMP}-login.html
 724     CRUFT="$CRUFT $LOGINPAGE"
 725
 726     URL="$GEOS/login/default.aspx"
 727     debug 1 "curl $CURL_OPTS $URL"
 728     curl $CURL_OPTS -s -A "$UA" -c$COOKIE_FILE \
 729         -e "$URL;auto" \
 730         -o $LOGINPAGE "$URL"
 731     [ -s $LOGINPAGE ] \
 732         || error "Unable to connect to gc.com; Try: CURL_OPTS=\"--sslv3\" in ~/.georc"
 733     get_value __VIEWSTATE $LOGINPAGE
 734     __VIEWSTATE=`urlencode "$__VIEWSTATE"`
 735     get_value __EVENTVALIDATION $LOGINPAGE
 736     __EVENTVALIDATION=`urlencode "$__EVENTVALIDATION"`
 737
 738     sleep 2
 739
 740     #
 741     #   Now post the login
 742     #
 743     URL="$GEOS/login/default.aspx"
 744     #URL="$URL?RESET=Y"
 745     #
 746     # RER 05/05/12: bug in recent curl/openssl use "--sslv3" for a workaround
 747     debug 1 "curl $URL"
 748     dbgcmd curl $CURL_OPTS --sslv3 -s -A "$UA" -b$COOKIE_FILE -c$COOKIE_FILE \
 749         -d __VIEWSTATE="$__VIEWSTATE" \
 750         -d __EVENTVALIDATION="$__EVENTVALIDATION" \
 751         -e "$URL;auto" \
 752         -dctl00%24ContentBody%24tbUsername="$_username" \
 753         -dctl00%24ContentBody%24tbPassword="$_password" \
 754         -dctl00%24ContentBody%24btnSignIn=Login \
 755         -L "$URL" > $LOGINPAGE
 756     if [ ! -s $LOGINPAGE ]; then
 757         error "curl: returned /dev/null for '$LOGINPAGE'"
 758     fi
 759     if grep -q "FormErrorWidget" $LOGINPAGE; then
 760         error "Login username/password does not match."
 761     fi
 762     if grep -q "If you are human" $LOGINPAGE; then
 763         error "Have to login this time with a web browser because of Recaptcha"
 764     fi
 765     if grep -q "temporarily down for maintenance." $LOGINPAGE; then
 766         error "Geocaching.com is temporarily down for maintenance."
 767     fi
 768     # Make case insignificant
 769     if ! grep -y -q "ctl00_ContentBody_LoggedInPanel" $LOGINPAGE; then
 770         error "BUG: gc.com changed the login page."
 771     fi
 772 }
 773
 774 #
 775 #       procedure to nag about agreeing to EasyGps download license
 776 #
 777 easy_warning() {
 778         cat <<-EOF
 779         You have not agreed to the waypoint download license at $GEO
 780
 781         Click one of the waypoint license agreement links at $GEO,
 782         read and agree to the license terms, then try this program again.
 783         EOF
 784 }
 785
 786 #
 787 #       getcids infile cidfile xtrafile archfile number
 788 #
 789 #       Wade thru the HTML and produce lists of found, notfound and new CIDs
 790 #
 791 getcids() {
 792     awk \
 793         -v "CIDFILE=$2" \
 794         -v "XTRAFILE=$3" \
 795         -v "ARCHFILE=$4" \
 796         -v "NUM=$5" \
 797         -v "USERFOUND=$USERFOUND" \
 798         -v "VARTIME=$VARTIME" \
 799         -v "SOC=$SOC" \
 800         -v "DATE=$date" \
 801         -v "DATEFMT=$DATEFMT" \
 802         -v "DEBUG=$DEBUG" \
 803         < $1 \
 804     '
 805     function debug(lvl, text) {
 806         if (lvl <= DEBUG)
 807             print text > "/dev/stderr"
 808     }
 809     function hex2dec(x,   val) {
 810         for (val = 0; length(x); x = substr(x, 2))
 811             val = 16*val + index("0123456789ABCDEF", substr(x, 1, 1)) - 1
 812         return val
 813     }
 814     # Convert GC0000 to 58913
 815     function wp2id(wp,    val) {
 816         sub("^GC", "", wp)
 817         if (DEBUG > 5)
 818             print "wp2id: " wp " ..." > "/dev/stderr"
 819         if (length(wp) <= 4 && wp < "G000")
 820         {
 821             val = hex2dec(wp)
 822             if (DEBUG > 5)
 823                 print "wp2id hex: " val " ..." > "/dev/stderr"
 824             return val
 825         }
 826         set = "0123456789ABCDEFGHJKMNPQRTVWXYZ"
 827         val = 0
 828         for (pos = 1; pos <= length(wp); ++pos)
 829         {
 830             val *= 31;
 831             val += index(set, substr(wp, pos, 1)) - 1;
 832         }
 833         val = val - 411120
 834         if (DEBUG > 5)
 835             print "wp2id id: " val " ..." > "/dev/stderr"
 836         return val
 837     }
 838     function id2wp(id,    val) {
 839         gid = "";
 840         if (id < 0)
 841             ;
 842         else if (id < 65536)
 843             gid = sprintf("GC%04X", id)
 844         else
 845         {
 846             GcOffset = 16 * 31 * 31 * 31 - 65536
 847             GcSet = "0123456789ABCDEFGHJKMNPQRTVWXYZ"
 848             id += GcOffset;
 849             for (i = 1; i <= 4; ++i)
 850             {
 851                 gid = substr(GcSet, id%31 + 1, 1) gid
 852                 id = int(id / 31)
 853             }
 854             tmp = substr(GcSet, id%31 + 1, 1)
 855             if (tmp != 0)
 856             {
 857                 gid = tmp gid
 858                 id = int(id / 31)
 859             }
 860             if (id)
 861                 gid = ""
 862             else
 863                 gid = "GC" gid
 864         }
 865         if (DEBUG >= 5)
 866             print "id = ", id, "wp = ", gid > "/dev/stderr"
 867         return gid
 868     }
 869
 870     function hash2sdt(text,
 871             day, cmd, i, value, mod, s0, t0, d0) {
 872         if (thiscode == "")
 873         {
 874         code[1] = "PDMTRXBJjZNH9fKW8vetlCgbos24mydknahrcOxYqLwGSE370z65ApVFQ"
 875         code[2] = "6JKt4b70rHoeTLqQB2jANwcYSMW9xPk3aplE5hGvyfsZDgFmndOVRC8zX"
 876         code[3] = "Rl2QaDdTsnK7rOZoywzx0H58NLCAGqgtp43mvYJVkFEejMWB69cXhfSbP"
 877         code[4] = "jShz6gF3OvonKEWkJCbZf4GeDX25VcyPsqwRYpLM8BtATm0x7l9aHQdNr"
 878         code[5] = "aJs2OWpoYfN9zvDHCwZrm83b4SKkFhXcAL0QPgBy7tMdRGlqxjEVn65Te"
 879         code[6] = "g6aXp7rqvkEl5JLnRDhfWGHTeA9OMQYdwS2BcbK0jCyFx4sPo83tmzZVN"
 880         code[7] = "MlocEO5eAgTFfLXzyPDk2qJhjBn98QSdNHR47mGWtY3ZvVxCpr0Ks6abw"
 881         code[8] = "p7SPbvzM4VkQ8f25cJL0gBlWjYmFoKhArXRHZntTGe6CNsyE3a9qOdwDx"
 882         code[9] = "znlBb8GHNdhCKyOpo53ePX9xfkvjD0aTEAVZ72qSQwrmMYct6JgRFWLs4"
 883         code[10] = "KXztRHdrkBwSWEn3AN2Fh8vOL5QMjx6oTgyCD7JYpaGmVcf9Pelb4q0Zs"
 884         code[11] = "M3odkDCgQa7GWK9pYh8tLAl6xFecRwPZNbTz0s42rqfOnSmvEjVyB5JXH"
 885         code[12] = "gweGxNAYCKfLdrb68X93ahjHMDtTWRFJVZvl0po5ySmOQnscPz4Ekq2B7"
 886         code[13] = "p2zelnEox9HqkGF4V5AZ6LbMBsy0dQc8J3fDXvON7jWgrKYCSTaPtRhmw"
 887         code[14] = "cy2gjCAJt3KmMxbr5OzQeYsvHpLF8fRVaoNXdGBW0l7ESZDP9qkw4Th6n"
 888         code[15] = "9KQZlwfBsCqN6oWkYhrSpDAEP48c5m3eJn2jVHLdybMx0gaOGRXzvFTt7"
 889         code[16] = "7j6CJPfqbMygan42HwxFBcrK3RYLstmOXWGeSQZ9kzl0VE8oDTpAhd5Nv"
 890         code[17] = "YScWw7gsx49ebLAqMGotHRX3fnCv6FBOkzVE5jdlmPTarDKQJyp8Z0hN2"
 891         code[18] = "BgwR6oskAcVPqeYdLKr3ONvflXz2nDSbxMQJEhyZC0pWHm759GFa4tT8j"
 892         code[19] = "9QyolrBq37he4nbCTcXEkmGHjYS02RDgMZLfOtavVzKNFxPs58p6JwWAd"
 893         code[20] = "8OwsHykXLJ3tqWZmR2pbVagMcTdnSC9FveN5z0xYjGK6frh7Q4EPDBAlo"
 894         code[21] = "RVBh8qEaN4Aw2Ob5egSytWjQD6oFmKvXfYnCzcx73kJsHrd9TLPZM0Glp"
 895         code[22] = "7MqQGJkKzZvdgmRXBH8CYeAVf3cy29LOW4PNaD0rsnowEpTbF5Sxhtl6j"
 896         code[23] = "8C26QBoWjEPcJ3Sf7TYtexXlMNyqArm0KD9HOLvZakGwnVbFsdp45Rzhg"
 897         code[24] = "6e5fC2S4Bwcm9pVKyqdFE3WlRDroTLJgGtN8ZQsAabMzHYO0Xvj7Pnxhk"
 898         code[25] = "JXHgMGcAVNpDEwWYvyBl025m89dOoTSKPxhstjq6e4QrCk3aZFbzfn7RL"
 899         code[26] = "Yp3ARoesjKXB2cadmkHvFQM7SEJtqf0l9TwzbPLrZDxO8h5C6NnGWyVg4"
 900         code[27] = "9kJaM4HonPWtcQvsYSEK3Vy5pjCRFqNO6lgr0x7mh2b8wdTGDzfLXeABZ"
 901         code[28] = "4n2gSwqzGB7C0MpV3yDY9FTtol8XjkJNcEPr6LxhZQKdRvHOfsA5eWmab"
 902         code[29] = "P6G8jsOxBCqf3k74DnXtAvl2ThabeEHJVgZp5wSzF9NWRcYMyL0dromKQ"
 903         code[30] = "WVwgM93zjmNcLpHvoSK2Q5bCxaATGsryXn6Bhfd0Z8qtERk7PYOJD4Fle"
 904         code[31] = "PZqTVol4HOXz0GWt38Qec59rKMyabfhE2DdsNxpF7RASkBjnYwvCgmJ6L"
 905             cmd = sprintf("%s -u +%%-d", DATE)
 906             cmd | getline day; close(cmd)
 907             thiscode = code[day]
 908         }
 909
 910         value = 0
 911         for (i = 1; i <= length(text); ++i) {
 912             value = 57 * value + index(thiscode, substr(text, i, 1)) - 1
 913         }
 914         #print value
 915
 916         mod = (value - 131586) % 16777216
 917
 918         s0 = int(mod / 131072);
 919         t0 = int((mod % 131072) / 256)
 920         d0 = int((mod % 131072) % 256)
 921         #print s0, t0, d0
 922
 923         # cache size
 924         if (s0 == 0) size = "not chosen"
 925         else if (s0 == 1) size = "Micro"
 926         else if (s0 == 2) size = "Regular"
 927         else if (s0 == 3) size = "Large"
 928         else if (s0 == 4) size = "Virtual"
 929         else if (s0 == 5) size = "Unknown"
 930         else if (s0 == 7) size = "Small"
 931         else size = "BUG"
 932
 933         # terrain rating
 934         if (t0 == 0) terrain = 1.0
 935         else if (t0 == 1) terrain = 1.5
 936         else if (t0 == 2) terrain = 2.0
 937         else if (t0 == 3) terrain = 2.5
 938         else if (t0 == 4) terrain = 3.0
 939         else if (t0 == 5) terrain = 3.5
 940         else if (t0 == 6) terrain = 4.0
 941         else if (t0 == 7) terrain = 4.5
 942         else terrain = 5.0
 943
 944         # difficulty rating
 945         if (d0 == 0) difficulty = 1.0
 946         else if (d0 == 1) difficulty = 1.5
 947         else if (d0 == 2) difficulty = 2.0
 948         else if (d0 == 3) difficulty = 2.5
 949         else if (d0 == 4) difficulty = 3.0
 950         else if (d0 == 5) difficulty = 3.5
 951         else if (d0 == 6) difficulty = 4.0
 952         else if (d0 == 7) difficulty = 4.5
 953         else difficulty = 5.0
 954     }
 955
 956     function begin_new_entry() {
 957         # Beginning of a new entry, reset variables to defaults
 958         inrecord = 1
 959         avail = 1
 960         archived = 0
 961         bugs = 0
 962         ifound = 0
 963         iplaced = 0
 964         soc = 0
 965         unfound = 1
 966         tdcnt = 0
 967         gcid = "GC0000"
 968         foundt = 0
 969         ifoundrec = 0
 970         ifoundt = 0
 971         container = "unknown"
 972         sendgps = 0     # Non-subscription members
 973         placer = ""
 974         # Bug Fix: FIXME
 975         #gctype = "Traditional cache"
 976         #type = "regular"
 977         #difficulty = 0
 978         #terrain = 0
 979         # End Bug Fix: FIXME
 980         if (DEBUG >= 5)
 981             print "Begin " NUM " ..." > "/dev/stderr"
 982     }
 983     function parse_dates(text,    cmd, val) {
 984         sub("[*]$", "", text)
 985         sub("^ *", "", text)    # remove leading blanks
 986         debug(5, "parse_dates: text=" text)
 987         if (text ~ /Today/)
 988         {
 989             cmd = sprintf("%s -d \"%s\" +%%s", DATE, "12am today")
 990             cmd | getline val; close(cmd)
 991         }
 992         else if (text ~ /Yesterday/)
 993         {
 994             cmd = sprintf("%s -d \"%s\" +%%s", DATE, "12am yesterday")
 995             cmd | getline val; close(cmd)
 996         }
 997         else if (text ~ /ago/)
 998         {
 999             cmd = sprintf("%s -d \"12am %s\" +%%s", DATE, text)
1000             cmd | getline val; close(cmd)
1001         }
1002         else if (text ~ /[0-9][^<]*[0-9]/ )
1003         {
1004             # GC Format   Example           Compatible
1005             # YYYY/MM/DD        2011/07/13  yes
1006             # MM/DD/YYYY        07/13/2011  yes
1007             # DD/MM/YYYY        13/07/2011  yes if DATEFMT = 1
1008             # DD/Mmm/YYYY       13/Jul/2001 no
1009             # Mmm/DD/YYYY       Jul/13/2011 no
1010             nf = split(text, fld, "/")
1011             if (nf == 3 && DATEFMT == 1)
1012             {
1013                 # DD before MM
1014                 debug(5, "parse_dates:  fld=" fld[1] "." fld[2] "." fld[3])
1015                 cmd = sprintf("%s -d \"12am %s\" +%%s", DATE,
1016                         fld[2] "/" fld[1] "/" fld[3])
1017                 cmd | getline val; close(cmd)
1018             }
1019             else
1020             {
1021                 # Regular way
1022                 cmd = sprintf("%s -d \"12am %s\" +%%s", DATE, text)
1023                 cmd | getline val; close(cmd)
1024             }
1025         }
1026         else
1027             val = 0
1028         debug(5, "parse_dates: val=" val)
1029         return val
1030     }
1031     BEGIN {
1032         q = sprintf("%c", 39)
1033         ++NUM
1034         gctype = "Traditional cache"
1035         type = "regular"
1036     }
1037     # Test: geo-nearest -s (18)
1038     # Test: geo-newest -s (20)
1039     # Test: geo-nearest -b multi (8)
1040     #
1041     # For geo-nearest AND geo-nearest -b
1042     /class=".* Data BorderTop"/ {
1043         begin_new_entry()
1044         debug(3, "Data BorderTop")
1045         next
1046     }
1047     /<td/ {
1048         ++tdcnt;
1049     }
1050     inrecord && tdcnt >= 1 && tdcnt <= 10 {
1051         if (DEBUG >= 5)
1052             print "tdcnt " tdcnt ", text: " $0 > "/dev/stderr"
1053     }
1054     /<a href="\/seek\/cache_details.aspx?[^>]*>/ {
1055         name = $0
1056
1057         # remove strikeout class
1058         sub("<span class[^>]*>", "", name)
1059         sub("</span>", "", name)
1060         sub(".*<span>", "", name)
1061         sub("</span>.*", "", name)
1062
1063         sub(".*<a href=.../seek/cache_details.aspx?[^>]*>", "", name)
1064         sub("</a>*.", "", name)
1065         sub("</[^>]*>", "", name)
1066         sub("</[^>]*>", "", name)
1067         sub("<[^>]*>", "", name)
1068         sub("<[^>]*>", "", name)
1069         debug(5, "name: " name)
1070     }
1071     # bookmarks by guid=...
1072     /<a href="http:..www.geocaching.com\/seek\/cache_details.aspx.guid=[^>]*>/ {
1073         if ($0 ~ /img src/)
1074         {
1075             name = $0
1076             sub(".*img src[^>]*>", "", name)
1077             sub("</span.*", "", name)
1078             sub(".*Strike.>", "", name)
1079             sub("</a.*", "", name)
1080             debug(5, "bid name: " name)
1081
1082             gctype = $0
1083             sub(".* alt=.", "", gctype)
1084             sub(". />.*", "", gctype)
1085             debug(5, "gctype: " gctype)
1086         }
1087         else
1088         {
1089             begin_new_entry()
1090             gcid = $0
1091             sub(".*<a href=.*/seek/cache_details.aspx?[^>]*>", "", gcid)
1092             sub("<.*", "", gcid)
1093             cid = wp2id(gcid)
1094             debug(5, "gcid: " gcid)
1095         }
1096     }
1097     /"Found It!"/ {
1098         if (USERFOUND)
1099             ifound = 1
1100         next
1101     }
1102     /Premium Member Only Cache/ {
1103         soc = 1
1104     }
1105     tdcnt == (9-0) && /[0-9][0-9]*.[0-9][0-9]*/ {
1106         # RER: May 4, 2011
1107         date = $0
1108         sub(/^ */, "", date)
1109         sub(/.*small.>/, "", date)
1110         sub(/ *<.*/, "", date)
1111         sub(/\015/, "", date)
1112         debug(5, "placedt text: "date)
1113         placedt = parse_dates(date)
1114         debug(5, "placedt: " placedt)
1115     }
1116     # <td>
1117     #   Yesterday<strong>*</strong><br />
1118     #   <span class="Success">
1119     #       3 days ago*</span>
1120     # </td>
1121     #OR
1122     # <td>
1123     #   Yesterday<strong>*</strong><br />
1124     #   <span class="Success">
1125     #   </span>
1126     # </td>
1127     tdcnt == (10-0) && /[a-zA-Z0-9*>]<br .>/ {
1128         # catch dates DD MMM YY, N days ago, Today, Yesterday
1129         unfound = 0
1130         date = $0
1131         sub(".*<td>", "", date)
1132         sub("<.*", "", date)
1133         lastfoundt = parse_dates(date)
1134         foundt = lastfoundt
1135         debug(5, "lastfoundt: " lastfoundt)
1136     }
1137     tdcnt == (10-0) && /[a-zA-Z0-9*>]<.span>/ {
1138         unfound = 0
1139         date = $0
1140         sub(".*Success.>", "", date)
1141         sub("<.*", "", date)
1142         ifoundt = parse_dates(date)
1143         debug(5, "ifoundt: " ifoundt)
1144     }
1145     /<strike><font/ {
1146         # i.e. <font color="red">
1147         avail = 0; archived = 1
1148     }
1149     /<\/strike><\/font/ {
1150         # i.e. <font color="red">
1151         avail = 0; archived = 1
1152     }
1153     /<strike>[^<]/ {
1154         avail = 0;
1155     }
1156     /class=".*Warning Strike.*">/ {
1157         debug(5, "***archived***")
1158         avail = 0; archived = 1
1159     }
1160     /class=".*Strike">/ {
1161         avail = 0
1162     }
1163     tdcnt == 6 && /^ *by / {
1164         placer = $0
1165         sub("^ *by ", "", placer)
1166         sub("\015", "", placer)
1167         debug(5, "placer: <" placer ">")
1168     }
1169     tdcnt == 6 && /^ *GC/ {
1170         gcid = $0
1171         sub("^ *", "", gcid)
1172         sub("\015", "", gcid)
1173         debug(5, "GCID: <" gcid ">")
1174         cid = wp2id(gcid)
1175     }
1176     /alt=.Your Geocache/ {
1177         # altmy cache
1178         # A mistake in the HTML! should be alt="my cache"
1179         if (USERFOUND) {
1180             iplaced = 1
1181             ifound = 1
1182         }
1183     }
1184     /alt="Earthcache"/ {
1185         type = "earth"
1186         gctype = "Earthcache"
1187     }
1188     /alt="Event Cache"/ {
1189         type = "event"
1190         gctype = "Event cache"
1191     }
1192     /alt="Cache In Trash Out Event"/ {
1193         type = "cito"
1194         gctype = "Cache In Trash Out Event"
1195     }
1196     /alt="Traditional Cache"/ {
1197         type = "regular"
1198         gctype = "Traditional cache"
1199     }
1200     /alt="Letterbox Hybrid"/ {
1201         type = "hybrid"
1202         gctype = "Letterbox Hybrid"
1203     }
1204     /alt="Multi-cache"/ {
1205         type = "multi"
1206         gctype = "Multi-Cache"
1207     }
1208     /alt="Unknown Cache"/ {
1209         type = "unknown"
1210         gctype = "Unknown Cache"
1211     }
1212     /alt="Virtual Cache"/ {
1213         type = "virtual"
1214         gctype = "Virtual cache"
1215     }
1216     /alt="Webcam Cache"/ {
1217         type = "webcam"
1218         gctype = "Webcam Cache"
1219     }
1220     /alt="Wherigo Cache"/ {
1221         type = "wherigo"
1222         gctype = "Wherigo Cache"
1223     }
1224     /alt="Mega-Event Cache"/ {
1225         type = "mega"
1226         gctype = "Mega-Event Cache"
1227     }
1228     /name=.CID. / {
1229         i = match($0, "value=.")
1230         cid = substr($0, i+7, 99) + 0
1231     }
1232     /name=.BID. / {
1233         i = match($0, "value=.")
1234         bid = substr($0, i+7, 99) + 0
1235     }
1236     #/ImgGen.seek.CacheInfo.ashx?v=/ {
1237     archived && /[?]v=/ {
1238         text = $0
1239         debug(5, "sdt text: " text)
1240         sub(".*?v=", "", text)
1241         sub(". .*", "", text)
1242         debug(5, "sdt text: " text)
1243         hash2sdt(text)
1244         debug(5, "size: " size " difficulty: " difficulty " terrain: " terrain)
1245         container = size
1246     }
1247     /Unapproved cache/ && inrecord {
1248         avail = 0; archived = 1
1249     }
1250     /class=.Checkbox NoBolding./ {
1251         sendgps = 1     # Subscription members
1252     }
1253     /<\/[tT][rR]>/ && inrecord {
1254         # RER mod 11/22/10: if (0 && ..
1255         if (0 && sendgps == 0)
1256             next
1257         inrecord = 0
1258         debug(5, "End")
1259
1260         strtype = "Geocache"
1261
1262         if (soc) strtype = strtype "-soc"
1263         if (unfound) strtype = strtype "-unfound"
1264         else if (ifound) strtype = strtype "-ifound"
1265         if (!avail) strtype = strtype "-unavail"
1266         if (archived) strtype = strtype "-archived"
1267
1268         strtype = strtype "-" type
1269
1270         # gpsbabel only allows one time in the DB, figure out what
1271         # time to use for this, but always carry all three times
1272         # in the .xtra file
1273         if (iplaced) ifoundt = placedt
1274         vartime=1234
1275         if (VARTIME == "placed") vartime = placedt
1276         else if (VARTIME == "ifound") vartime = ifoundt
1277         #else if (foundt > 0) vartime = foundt
1278         else vartime = placedt
1279
1280         # date -d 1970-01-01 1237093200 sec +%Y%m%d
1281         # 20090315
1282         cmd = sprintf("%s -d \"1970-01-01 %d sec\" +%%Y%%m%%d",
1283             DATE, lastfoundt)
1284         cmd | getline lastfound; close(cmd)
1285
1286         # avail=1 is the choice right now (8/19/05)
1287         # archived=0 is the choice right now (11/14/06)
1288         # sendgps=1 Remove non-soc caches from basic member. (12/18/2012)
1289         debug(5, gcid "\tsendgps: " sendgps " archived: " archived  \
1290                     " SOC: " SOC " soc: " soc " bid: " bid)
1291         if (!archived && (SOC || !soc) && (sendgps == 1 || bid) )
1292         {
1293             if (bid != 0) {
1294                 printf("-dBID=%d\n", bid) > CIDFILE
1295                 gcid = id2wp(bid)
1296             } else if (cid != 0) {
1297                 printf("-dCID=%d\n", cid) > CIDFILE
1298             }
1299             # GCID type vartime ifound soc iplaced tPLACED tFOUND tIFOUND
1300                 #"%.1f\t%.1f\t%s\t"
1301                 #difficulty, terrain, container,
1302             printf "%s\t%s\t%d\t%s\t%s\t%s\t%d\t%d\t%d\t" \
1303                     "%s\t%s\t%s\t%s\n",
1304                     gcid, strtype, vartime,
1305                     ifound, soc, iplaced, placedt, foundt, ifoundt,
1306                     gctype, placer, "hint", lastfound >> XTRAFILE
1307             if (--NUM == 0)
1308                 exit
1309         }
1310         else if (archived)
1311         {
1312             if (DEBUG > 5)
1313                 print "Archived ", gcid, " " name > "/dev/stderr"
1314             if (name == "")
1315                 name = "error-no-name!"
1316             if (placer == "")
1317                 placer = "unknown-placer"
1318             # Use "merge" file output, NOT XTRAFILE!
1319             printf "%s\t%s\t0.0\t0.0\t%s%s%s\t" \
1320                 "%.1f\t%.1f\t%s\t" \
1321                 "%s\t%d\t%d\t%d\t%d\t%d\t%d\t%s\t%s\t%s\t%s\t%s\n",
1322                 gcid, name,
1323                 "http://www.geocaching.com/seek/cache_details.aspx",
1324                 "?log=y&wp=", gcid,
1325                 difficulty, terrain, container,
1326                 strtype, vartime,
1327                 ifound, soc, iplaced, placedt, foundt, ifoundt,
1328                 gctype, placer, "hint", lastfound >> ARCHFILE
1329             if (--NUM == 0)
1330                 exit
1331         }
1332     }
1333     '
1334 }
1335
1336 #
1337 #       A temporary style we can use for merging the loc data with
1338 #       the scraped html data.  This is a dual purpose hack.  We
1339 #       use it as an output format to convert the .loc data to a
1340 #       record-per-line format.  We use it as an input format to
1341 #       read up the merged data.
1342 #
1343 make_scrape_style() {
1344     cat <<-EOF
1345         ENCODING                UTF-8
1346         FIELD_DELIMITER         TAB
1347         RECORD_DELIMITER        NEWLINE
1348         BADCHARS                TAB
1349         IFIELD  SHORTNAME,      "",     "%s"
1350         IFIELD  DESCRIPTION,    "",     "%s"
1351         IFIELD  LAT_DECIMAL,    "",     "%.6f"
1352         IFIELD  LON_DECIMAL,    "",     "%.6f"
1353         IFIELD  URL,    "",     "%s"
1354         IFIELD  GEOCACHE_DIFF,  "",     "%3.1f"         #difficulty
1355         IFIELD  GEOCACHE_TERR,  "",     "%3.1f"         #terrain
1356         IFIELD  GEOCACHE_CONTAINER,"",  "%s"            #container (not set)
1357         IFIELD  ICON_DESCR,     "",     "%s"            #strtype (Geocache-*)
1358         IFIELD  TIMET_TIME,     "",     "%ld"           #variable time
1359         IFIELD  IGNORE,         "",     "%s"            #ifound
1360         IFIELD  IGNORE,         "",     "%s"            #soc
1361         IFIELD  IGNORE,         "",     "%s"            #iplaced
1362         IFIELD  IGNORE,         "",     "%s"            #placed time
1363         IFIELD  IGNORE,         "",     "%s"            #found time
1364         IFIELD  IGNORE,         "",     "%s"            #ifound time
1365         IFIELD  GEOCACHE_TYPE,  "",     "%s"            #gc.com type
1366         IFIELD  GEOCACHE_PLACER, "",    "%s"            #gc.com placer
1367         IFIELD  GEOCACHE_HINT,  "",     "%s"            #gc.com hint
1368         IFIELD  GEOCACHE_LAST_FOUND, "","%s"            #gc.com last found
1369         OFIELD  SHORTNAME,      "",     "%s"
1370         OFIELD  DESCRIPTION,    "",     "%s"
1371         OFIELD  LAT_DECIMAL,    "",     "%.6f"
1372         OFIELD  LON_DECIMAL,    "",     "%.6f"
1373         OFIELD  URL,    "",     "%s"
1374         OFIELD  GEOCACHE_DIFF,  "",     "%3.1f"         #difficulty
1375         OFIELD  GEOCACHE_TERR,  "",     "%3.1f"         #terrain
1376         OFIELD  GEOCACHE_CONTAINER,"",  "%s"            #container (not set)
1377         EOF
1378         # OFIELD        ICON_DESCR,     "",     "%s"
1379 }
1380
1381 #
1382 #       csv2csv geo-mystery-file
1383 #
1384 csv2csv() {
1385     if [ ! -e "$1" ]; then
1386         error "Don't have a -M '$1' file"
1387     fi
1388
1389     awk -v GEOMYSTERY=$1 '
1390     function latlon ( val ) {
1391         if (val ~ ".[.]..*[.].*")
1392         {
1393             if (val ~ "[-wWsS]")
1394             {
1395                 val = substr(val, 2)
1396                 neg = 1
1397             }
1398             else if (val ~ "[nNeE]")
1399             {
1400                 val = substr(val, 2)
1401                 neg = 0
1402             }
1403             else
1404                 neg = 0
1405             dd = int(val)
1406             sub("[^.]*[.]", "", val)
1407             dd += (val+0.0) / 60.0
1408             return neg ? -dd : dd
1409         }
1410         return val
1411     }
1412     BEGIN {
1413         while (getline <GEOMYSTERY > 0)
1414         {
1415             if ($0 ~ "^#")
1416                 continue
1417             if ($0 ~ "^[        ]*$")
1418                 continue
1419             gc=$1
1420             sub(".*/", "", gc)  # remove stuff like http://coord.info/
1421             if ($2 ~ "unk")
1422             {
1423                 lat[gc] = 48
1424                 lon[gc] = -88
1425             }
1426             else
1427             {
1428                 lat[gc] = $2
1429                 lon[gc] = $3
1430             }
1431         }
1432         FS="    "
1433     }
1434     {
1435         if (lat[$1] != 0)
1436         {
1437             deglat = latlon( lat[$1] )
1438             deglon = latlon( lon[$1] )
1439             # GC# Desc lat lon URL diff terr size
1440             printf "%s\t", $1
1441             printf "+%s\t", $2
1442             printf "%.6f\t%.6f\t", deglat, deglon
1443             for (i = 5; i < NF; ++i)
1444                 printf "%s\t", $i
1445             printf "%s\n", $NF
1446         }
1447         else
1448             print
1449     }
1450     '
1451 }
1452
1453 #
1454 #       Fetch the cache patch
1455 #
1456 gc_fetch_cache_page(){
1457     _page=$1    #out
1458     _gcid=$2    #in
1459     _url=$3     #in
1460     _extra=$4   #in
1461
1462     # Fetch the basic cache page
1463     echo "<BASE HREF=http://www.geocaching.com/seek/>" > $_page
1464     debug 1 "${_extra}curl $_url #cache fetch $_gcid.html"
1465     curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" \
1466         $_url \
1467         | tr -d "\001\007\010\013\017\020\031" \
1468         | sed -e '/<\/html>/d' \
1469         >> $_page
1470
1471     size=$(ls -l $_page | awk '{print $5}')
1472     if [ $size -lt 1000 ]; then
1473         debug 0 "Could not retrieve web page for cache $_gcid"
1474         return 1
1475     fi
1476
1477     # JSON for logbooks
1478     JSON_NUM=100
1479     tkn=`grep "userToken = '" $_page | sed -e "s/userToken = '//" -e "s/'.*//" `
1480     jsurl="http://www.geocaching.com/seek/geocache.logbook"
1481     jsurl="$jsurl?tkn=$tkn"
1482     jsurl="$jsurl&idx=1&num=$JSON_NUM&sp=false&sf=false&decrypt=false"
1483     debug 1 "curl $jsurl" >&2
1484     curl $CURL_OPTS -s -A "$UA" -b $COOKIE_FILE \
1485          "$jsurl" >> $_page
1486     echo "" >> $_page
1487     echo "</html>" >> $_page
1488     return 0
1489 }
1490
1491 #
1492 #       Query the gc website
1493 #
1494 gc_query() {
1495     if [ $USERFOUND = 0 ]; then
1496         FOUND=1
1497     fi
1498     if [ $FOUND = 0 ]; then
1499             SEARCH="$SEARCH&f=1"
1500     fi
1501     if [ "$RADIUS_NUM" != "" ]; then
1502             SEARCH="$SEARCH&dist=$RADIUS_MILES"
1503     fi
1504
1505     if [ $DEBUG -gt 0 ]; then
1506         TMP=/tmp/geo
1507     else
1508         TMP=/tmp/geo$$
1509     fi
1510
1511     HTMLPAGE=$TMP.page
1512     CIDFILE=$TMP.cids
1513     LOCFILE=$TMP.loc
1514     LOCTMPFILE=$TMP.tmp
1515     XTRAFILE=$TMP.xtra
1516     CSVFILE=$TMP.csv
1517     CSVFILE2=$TMP.csv2
1518     JOINFILE=$TMP.join
1519     MERGEFILE=$TMP.merge
1520     ARCHFILE=$TMP.arch
1521     OUTWAY=$TMP.way
1522     STYLE=$TMP.newstyle
1523
1524     CRUFT="$CRUFT $HTMLPAGE"
1525     CRUFT="$CRUFT $CIDFILE"
1526     CRUFT="$CRUFT $LOCFILE"
1527     CRUFT="$CRUFT $LOCTMPFILE"
1528     CRUFT="$CRUFT $XTRAFILE"
1529     CRUFT="$CRUFT $CSVFILE"
1530     CRUFT="$CRUFT $CSVFILE2"
1531     CRUFT="$CRUFT $JOINFILE"
1532     CRUFT="$CRUFT $MERGEFILE"
1533     CRUFT="$CRUFT $ARCHFILE"
1534     CRUFT="$CRUFT $OUTWAY"
1535     CRUFT="$CRUFT $STYLE"
1536     if [ $NOCOOKIES = 1 ]; then
1537         CRUFT="$CRUFT $COOKIE_FILE"
1538     fi
1539
1540     #
1541     #   Login to gc.com
1542     #
1543     gc_login "$USERNAME" "$PASSWORD"
1544
1545     #
1546     #   Find the bookmark
1547     #
1548     if [ "$BOOKMARK" != "" ]; then
1549         URL="$GEO/bookmarks"
1550         debug 1 "$start: curl $URL #bookmark"
1551         SEARCH=`
1552             curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" "$URL" \
1553             | grep -y ">$BOOKMARK<" \
1554             | $sed \
1555                 -e 's@. title.*@@' \
1556                 -e 's@^.*href=.http://www.geocaching.com/@@' -e 's/.>.*$//'
1557             `
1558         case "$SEARCH" in
1559         *bookmarks*)    ;;
1560         *)              error "No bookmark with the name '$BOOKMARK'.";;
1561         esac
1562     fi
1563
1564     #
1565     #   Find the pocket query
1566     #
1567     if [ "$POCKETQUERY" != "" ]; then
1568         URL="$GEO/pocket/default.aspx"
1569         debug 1 "$start: curl $URL #pocket-query"
1570         SEARCH=`
1571             curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" "$URL" \
1572             | grep -y "title=\"$POCKETQUERY\"" \
1573             | $sed \
1574                 -e 's@.*guid=@@' -e 's@".*@@'
1575             `
1576         case "$SEARCH" in
1577         *-*-*-*-*)      SEARCH="?pq=$SEARCH";;
1578         *)              error "No PQ with the name '$POCKETQUERY'.";;
1579         esac
1580     fi
1581
1582     #
1583     #   We might combine one or more pages into a single XML, so cobble
1584     #   up a header with the ?xml and loc tags.
1585     #
1586     cat <<-EOF > $LOCFILE
1587         <?xml version="1.0" encoding="UTF-8"?>
1588         <loc version="1.0" src="EasyGPS">
1589         EOF
1590
1591     #
1592     # Loop, getting at least "NUM" locations
1593     #
1594     ((NUM=NUM-1))
1595     if [ $DEBUG -gt 0 ]; then
1596         filter2="tee $TMP.bulk"
1597     else
1598         filter2=cat
1599     fi
1600
1601     subscriber=1
1602     > $XTRAFILE
1603     > $ARCHFILE
1604     ((start=0))
1605     while ((start <= NUM)); do
1606         sleep $GEOSLEEP
1607
1608         #
1609         # Fetch the page of closest caches and scrape the cache ID's
1610         #
1611         case "$SEARCH" in
1612         *bookmark*)
1613             URL="$GEO/$SEARCH"
1614             ;;
1615         *)
1616             URL="$GEO/seek/nearest.aspx"
1617             URL="$URL$SEARCH"
1618             ;;
1619         esac
1620         debug 1 "$start: curl $URL #list"
1621         if ((start > 0)); then
1622             # "postback"... grab the "next" button
1623             case "$SEARCH" in
1624             *bookmark*)
1625                 # __EVENTTARGET="ListInfo\$pgrBMItems\$_ctl08"
1626                 # Get the last TGT (== ctl08)
1627                 TGT=$($sed -n "s/^.*__doPostBack('.*pgrBMItems\$\(.*\)','.*/\1/p" \
1628                     < $HTMLPAGE)
1629                 if [ "$TGT" = "" ]; then
1630                     error "TGT is blank!"
1631                 fi
1632                 __EVENTTARGET="ctl00%24ContentBody%24ListInfo%24pgrBMItems%24$TGT"
1633                 ;;
1634             *)
1635                 TGT=$($sed -n "s/^.*__doPostBack('.*pgrTop\$\(.*\)','.*/\1/p" \
1636                     < $HTMLPAGE)
1637                 if [ "$TGT" = "" ]; then
1638                     error "TGT is blank!"
1639                 fi
1640                 __EVENTTARGET="ctl00%24ContentBody%24pgrTop%24$TGT"
1641                 ;;
1642             esac
1643             curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" \
1644                 -d __EVENTTARGET="$__EVENTTARGET" \
1645                 $viewstate \
1646                 "$URL" \
1647                 | $sed -e "s/&#39;/'/g" -e "s/\r//" > $HTMLPAGE
1648         else
1649             curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" \
1650                 "$URL" \
1651                 | $sed -e "s/&#39;/'/g" -e "s/\r//" > $HTMLPAGE
1652             if [ "$DEBUG" -ge 1 ]; then
1653                 grep "Total Records:.*Top.*" $HTMLPAGE |
1654                     sed -e "s/<.b>.*//" -e "s/^.*span>//" -e "s/<b>//" 1>&2
1655             fi
1656         fi
1657         rc=$?; if [ $rc != 0 ]; then
1658             error "curl: fetch $URL"
1659         fi
1660         if grep -s -q "We encountered an error when requesting that page!" \
1661             $HTMLPAGE; then
1662             error "searching error (1) on $start"
1663         fi
1664         if grep -s -q "has resulted in an error" \
1665             $HTMLPAGE; then
1666             error "searching error (2) on $start"
1667         fi
1668         if grep -s -q "By State" $HTMLPAGE; then
1669             error "searching gave up on $start"
1670         fi
1671         if grep -s -q ">Advanced Search<" $HTMLPAGE; then
1672             error "need a country AND a state!"
1673         fi
1674
1675         #
1676         # Grab a few important values from the page
1677         #
1678         gc_getviewstate $HTMLPAGE
1679         # cp $HTMLPAGE /tmp/view$start.html
1680
1681         #
1682         # Grab the CIDs into two categories: found and notfound
1683         #
1684         > $CIDFILE
1685         debug 3 "getcids $HTMLPAGE $CIDFILE $XTRAFILE $ARCHFILE $((NUM-start))"
1686         getcids $HTMLPAGE $CIDFILE $XTRAFILE $ARCHFILE $((NUM-start))
1687
1688         #
1689         # Fetch the waypoints, rip out the ?xml and loc tags, and
1690         # append to the $LOCFILE file.
1691         #
1692         if [ -s "$CIDFILE" ]; then
1693             sleep $GEOSLEEP
1694             case "$SEARCH" in
1695             *bookmark*)
1696                 URL="$GEO/$SEARCH"
1697                 ;;
1698             *)
1699                 URL="$GEO/seek/nearest.aspx"
1700                 URL="$URL$SEARCH"
1701                 ;;
1702             esac
1703             #
1704             #   Fetch the .loc file
1705             #   1)      -d "Download=Download+Waypoints"...     geo-nearest
1706             #   2)      -d "ListInfo:btnDownload=       ...     old
1707             #   3)      -d 'ctl00$ContentBody           ...     -b acro
1708             #
1709             debug 2 "$start: curl $URL #loc"
1710             curl $CURL_OPTS -s -b $COOKIE_FILE -A "$UA" \
1711                 $viewstate \
1712                 `cat $CIDFILE` \
1713                 -d "Download=Download+Waypoints" \
1714                 -d "ListInfo:btnDownload=Download+to+.Loc" \
1715                 -d 'ctl00$ContentBody$ListInfo$btnDownload=Download+to+.Loc' \
1716                 "$URL" \
1717             | $filter2 \
1718             | $sed -e 's/^<?xml [^>]*>//' \
1719                     -e 's/>[gG]eocache</>Geocache</g' \
1720                     -e 's/<loc [^>]*>//' \
1721                     -e 's#</loc>##' \
1722                     > $LOCTMPFILE
1723             rc=$?; if [ $rc != 0 ]; then
1724                 error "curl: fetch the waypoints"
1725             fi
1726             if grep -s -q "you are not logged in" $LOCTMPFILE; then
1727                 error "you are not logged in on $start"
1728             fi
1729             if grep -s -q "has resulted in an error" $LOCTMPFILE; then
1730                 error "searching error (3) on $start"
1731             fi
1732             if grep -s -q "Geocaching > Search for Geocaches" $LOCTMPFILE; then
1733                 error "searching error (4) on $start"
1734             fi
1735             if grep -s -q "recaptcha_challenge_field" $LOCTMPFILE; then
1736                 #
1737                 # Basic members: do it the SLOW way
1738                 #
1739                 if [ "$GEODIR" != "" -a ! -d "$GEODIR" ]; then
1740                     mkdir "$GEODIR" || error "Couldn't mkdir $GEODIR"
1741                 fi
1742                 if [ "$GEODIR/caches" != "" -a ! -d "$GEODIR/caches" ]; then
1743                     mkdir "$GEODIR/caches" \
1744                         || error "Couldn't mkdir $GEODIR/caches"
1745                 fi
1746
1747                 if [ $subscriber = 1 ]; then
1748                     debug 0 "Basic Member: pay the \$30/yr fee for Premium Member!"
1749                     debug 0 "Basic Member: doing it the REAL SLOW way!!!"
1750                     timestamp="$GEODIR/caches/.timestamp"
1751                     $touch -d "1 day ago" $timestamp
1752                 fi
1753                 subscriber=0
1754
1755                 #
1756                 # Pull 20 GCIDs...
1757                 #
1758                 bstart=$start
1759                 grep "^ *GC.*" $HTMLPAGE \
1760                 | $sed -e "s/ //g" \
1761                 | while read gcid; do
1762                     #
1763                     #   Get the cache page
1764                     #
1765                     cachepage="$GEODIR/caches/$gcid.html"
1766                     if [ ! -s $cachepage -o $timestamp -nt $cachepage ]; then
1767                         # cache page doesn't exist or out of date
1768                         sleep 3 #$GEOSLEEP
1769
1770                         urlpage="http://www.geocaching.com"
1771                         urlpage="$urlpage/seek/cache_details.aspx"
1772                         urlpage="$urlpage?log=y&wp=$gcid"
1773                         gc_fetch_cache_page $cachepage $gcid $urlpage \
1774                             "$bstart: "
1775                         if [ $? != 0 ]; then
1776                             continue
1777                         fi
1778                     fi
1779
1780                     # 01/07/12: remove soc's!
1781                     if grep -q 'alt="Premium Members only"' $cachepage; then
1782                         continue
1783                     fi
1784                     # 12/18/12: remove soc's
1785                     if grep -q 'uxPremiumSubmitBottom' $cachepage; then
1786                         continue
1787                     fi
1788                     # 02/04/12: removed archived!
1789                     if grep -q '>This cache has been archived' $cachepage; then
1790                         continue
1791                     fi
1792
1793                     #
1794                     #   Cobble up the .loc file entry
1795                     #
1796                     name=` grep -A1 "<head><title>" $cachepage \
1797                         | tail -1 | tr -d "\r"  \
1798                         | $sed -e 's/[  ]*GC[^ ]* //' -e 's/ (.*by/ by/' `
1799                     diff=` grep ctl00_ContentBody_uxLegendScale $cachepage \
1800                         | $sed -e 's/.*alt="//' -e 's/ .*//' `
1801                     terr=` grep ctl00_ContentBody_Localize6 $cachepage \
1802                         | $sed -e 's/.*alt="//' -e 's/ .*//' `
1803                     size=` grep "alt=.Size:" $cachepage \
1804                         | $sed -e 's/.*alt="Size: //' -e 's/" .*//' `
1805                     case $size in
1806                     Unknown)    cont=1;;
1807                     Micro)      cont=2;;
1808                     Regular)    cont=3;;
1809                     Large)      cont=4;;
1810                     Virtual)    cont=5;;
1811                     Other)      cont=6;;
1812                     Small)      cont=8;;
1813                     *)          cont=1;;
1814                     esac
1815
1816                     echo "<waypoint>"
1817                     echo "<name id=\"$gcid\"><![CDATA[$name]]></name>"
1818                     grep "lnkConversions" $cachepage \
1819                         | $sed -e 's/.*lat=/<coord lat="/' \
1820                             -e 's/&amp;lon=/" lon="/' -e 's@&.*@"/>@'
1821                     echo "<type>Geocache</type>"
1822                     echo -n "<link text=\"Cache Details\">"
1823                     urlpage="http://www.geocaching.com"
1824                     urlpage="$urlpage/seek/cache_details.aspx"
1825                     urlpage="$urlpage?wp=$gcid"
1826                     echo -n "$urlpage"
1827                     echo "</link>"
1828                     echo "<difficulty>$diff</difficulty>"
1829                     echo "<terrain>$terr</terrain>"
1830                     echo "<container>$cont</container>"
1831                     echo "</waypoint>"
1832
1833                     debug 3 "bstart: $bstart"
1834                     if [ $bstart == $NUM ]; then
1835                         break
1836                     fi
1837                     ((++bstart))
1838                     #break
1839                 done >> $LOCFILE
1840             else
1841                 cat $LOCTMPFILE >> $LOCFILE
1842             fi
1843         fi
1844
1845         #
1846         # Check to see if the user hasn't agreed to license terms
1847         #
1848         if grep -s -q "lblAgreementText" $LOCFILE; then
1849                 easy_warning >&2
1850                 remove_cruft
1851                 exit
1852         fi
1853
1854         ((start=start+20))
1855
1856         # If the Next button is disabled, break this loop
1857         # grep "Records: <b>[1-9].*disabled.><b>Next<" $HTMLPAGE
1858         if grep -s -q "Records: <b>[1-9].*disabled.><b>Next " $HTMLPAGE; then
1859             # echo "$start: dis"
1860             break;
1861         fi
1862     done
1863
1864     #
1865     # Finish off the .loc file
1866     #
1867     echo "</loc>" >> $LOCFILE
1868
1869     #
1870     #   Convert the .loc data to .csv format and join it with
1871     #   the extra data scraped from the HTML page.  Filter out
1872     #   the data according to the -I and -X options.
1873     #
1874     # http://www.geocaching.com/seek/cache_details.aspx?wp=GCG2H4
1875     # http://www.geocaching.com/seek/cache_details.aspx?log=y&wp=GCG2H4
1876     # http://www.geocaching.com/seek/cache_details.aspx?ID=92117&log=y
1877     #
1878     # The joined .csv format looks like this:
1879     #   GCH636  Jidana 3 by rickrich    44.94520        -93.47540 \
1880     #   http://www.geocaching.com/seek/cache_details.aspx?log=y&wp=GCH636 \
1881     #   Geocache-ifound-regular 1070285077      1       0       1 \
1882     #   1067925600      1070285077      0
1883     #
1884     make_scrape_style > $STYLE
1885
1886     dbgcmd gpsbabel -i geo$GEONUKE -f $LOCFILE \
1887         -o xcsv,style=$STYLE -F $CSVFILE2
1888     if [ $? != 0 ]; then
1889         cp $LOCFILE /tmp/geo.err
1890         error "gpsbabel returned error code [1]"
1891     fi
1892
1893     # Convert CSV into CSV with puzzles...
1894     csv2csv $GEOMYSTERY < $CSVFILE2 > $CSVFILE
1895
1896     # 01/07/12: NO --nocheck-order; it is wrong!
1897     join -t '   ' $CSVFILE $XTRAFILE > $JOINFILE
1898     if [ $? != 0 ]; then
1899         cp $CSVFILE /tmp/geo.err.csv
1900         cp $XTRAFILE /tmp/geo.err.xtra
1901         error "joined file error: see join /tmp/geo.err.{csv,xtra}"
1902     fi
1903     egrep -- "$INCLUDE" < $JOINFILE \
1904     | egrep -v -- "$EXCLUDE" \
1905     | $sed -e 's/wp=/log=y\&&/' > $MERGEFILE
1906
1907     if [ $FOUND = 1 ]; then
1908         cat $ARCHFILE \
1909         | egrep -- "$INCLUDE" | egrep -v -- "$EXCLUDE" \
1910         | $sed -e 's/wp=/log=y\&&/' >> $MERGEFILE
1911     fi
1912
1913     if [ $DEBUG -ge 2 ]; then
1914         # First two of these should be the same number of lines!
1915         wc -l $CSVFILE >&2
1916         wc -l $XTRAFILE >&2
1917         wc -l $MERGEFILE >&2
1918     fi
1919
1920     #
1921     # Convert to the desired format
1922     #
1923     BABELFILT=
1924     if [ "$RADIUS" != "" ]; then
1925         BABELFILT="-x radius,distance=$RADIUS,lat=$LAT,lon=$LON"
1926     fi
1927
1928     if [ $SQL = 1 ]; then
1929             #
1930             # add it via mysql
1931             #
1932             if [ "$OUTFILE" != "" ]; then
1933                 >"$OUTFILE"
1934             fi
1935
1936             if [ $PURGE = 1 ]; then
1937                 gpsdrive_purge | gpsdrive_mysql
1938                 PURGE=2
1939             fi
1940
1941             dbgcmd gpsbabel $BABELFLAGS \
1942                 -i xcsv,style=$STYLE -f $MERGEFILE \
1943                 $BABELFILT -o "$OUTFMT" -F $OUTWAY
1944             if [ $? != 0 ]; then
1945                 error "gpsbabel returned error code [2]"
1946             fi
1947             gpsdrive_add <$OUTWAY $SQLTAG | gpsdrive_mysql
1948     elif [ $MAP = 1 ]; then
1949             dbgcmd gpsbabel $BABELFLAGS \
1950                 -i xcsv,style=$STYLE -f $MERGEFILE \
1951                 $BABELFILT -o "$OUTFMT" -F $OUTWAY
1952             if [ $? != 0 ]; then
1953                 error "gpsbabel returned error code [3]"
1954             fi
1955             if [ "$OUTFILE" = "" ]; then
1956                 dbgcmd geo-map -s0 $MAPOPTS -t$OUTWAY
1957             else
1958                 dbgcmd geo-map -s0 $MAPOPTS -t$OUTWAY -o"$OUTFILE"
1959             fi
1960     else
1961             #
1962             # output to stdout or to a file
1963             #
1964             if [ "$OUTFILE" = "" ]; then
1965                 OUTTMP="$TMP.way";  CRUFT="$CRUFT $OUTTMP"
1966                 dbgcmd gpsbabel $BABELFLAGS \
1967                     -i xcsv,style=$STYLE -f $MERGEFILE \
1968                     $BABELFILT -o "$OUTFMT" -F $OUTTMP
1969                 if [ $? != 0 ]; then
1970                     error "gpsbabel returned error code [4]"
1971                 fi
1972                 cat $OUTTMP
1973             else
1974                 dbgcmd gpsbabel $BABELFLAGS \
1975                     -i xcsv,style=$STYLE -f $MERGEFILE \
1976                     $BABELFILT -o "$OUTFMT" -F $OUTFILE
1977                 if [ $? != 0 ]; then
1978                     error "gpsbabel returned error code [5]"
1979                 fi
1980             fi
1981     fi
1982
1983     #
1984     #   Optionally, print the HTML pages
1985     #
1986     if [ "$CMDPIPE" != "" ]; then
1987         OIFS="$IFS"
1988         IFS="   "
1989         while read id desc lat lon url diff terr container strtype \
1990             vartime ifound soc iplaced placedt foundt ifoundt extra; do
1991             url="$url&decrypt=y"
1992             echo "Print: $url"
1993             HTMLPAGE2=$TMP.html
1994             CRUFT="$CRUFT $HTMLPAGE2"
1995             debug 1 "curl $url" >&2
1996             dbgcmd curl $CURL_OPTS -s -A "$UA" -b $COOKIE_FILE "$url" > $HTMLPAGE2
1997             htmldoc --quiet -t ps --nup 2 --fontsize 14 --webpage $HTMLPAGE2 \
1998                 | psselect -q -p1-1 | eval $CMDPIPE
1999             # exit
2000         done < $MERGEFILE
2001         IFS="$OIFS"
2002     fi
2003
2004     #
2005     #   Optionally, fetch printable HTML pages
2006     #
2007     if [ "$HTMLDIR" != ""  -o "$LOGDIR" != "" ]; then
2008         if [ "$HTMLDIR" != "" -a ! -d "$HTMLDIR" ]; then
2009             mkdir "$HTMLDIR" || error "Couldn't mkdir $HTMLDIR"
2010         fi
2011         if [ "$LOGDIR" != "" -a ! -d "$LOGDIR" ]; then
2012             mkdir "$LOGDIR" || error "Couldn't mkdir $LOGDIR"
2013         fi
2014
2015         HTMLPAGE2=$TMP.html
2016         CRUFT="$CRUFT $HTMLPAGE2"
2017         TIMESTAMP=$TMP.time
2018         CRUFT="$CRUFT $TIMESTAMP"
2019
2020         fetchcnt=0
2021         fetchmax=1000
2022         if [ $DEBUG -ge 3 ]; then
2023             fetchmax=3
2024         fi
2025         OIFS="$IFS"
2026         IFS="   "
2027         while read id desc lat lon url diff terr container strtype \
2028             vartime ifound soc iplaced placedt foundt ifoundt extra; do
2029             #
2030             #   Don't fetch page if we already have a current version
2031             #
2032             #debug 1 "placedt=<$placedt> <$foundt>"
2033             if [ $placedt -gt $foundt ]; then
2034                 filetime="$placedt"
2035             else
2036                 filetime="$foundt"
2037             fi
2038             $touch -d "1/1/70 $filetime seconds last second" $TIMESTAMP
2039             if [ "$filetime" -gt 0 -a "$HTMLDIR/$id.html" -nt $TIMESTAMP ]; then
2040                 continue
2041             fi
2042
2043             # Basic member...
2044             if [ "$subscriber" == 0 -a "$HTMLDIR" != "" ]; then
2045                 debug 2 "cp $GEODIR/caches/$id.html $HTMLDIR/$id.html"
2046                 cp "$GEODIR/caches/$id.html" "$HTMLDIR/$id.html"
2047                 # debug 2 "f: $filetime"
2048                 if [ "$filetime" -gt 0 ]; then
2049                     $touch -d "1/1/70 $filetime seconds" "$HTMLDIR/$id.html"
2050                 fi
2051                 continue
2052             fi
2053
2054             # Limit to 1000 caches/day
2055             ((fetchcnt=fetchcnt+1))
2056             if [ $fetchcnt -gt $fetchmax ]; then
2057                 error "Fetch count exceeded $fetchmax.  Try tomorrow!"
2058             fi
2059
2060             # Be kind to the server.  Do not remove this sleep
2061             sleep $GEOSLEEP
2062
2063             gc_fetch_cache_page $HTMLPAGE $id $url ""
2064             if [ $? != 0 ]; then
2065                 continue
2066             fi
2067
2068             if [ "$HTMLDIR" != "" ]; then
2069                 cp $HTMLPAGE "$HTMLDIR/$id.html" ||
2070                     error "Couldn't copy $id cache page"
2071                 if [ "$filetime" -gt 0 ]; then
2072                     $touch -d "1/1/70 $filetime seconds" "$HTMLDIR/$id.html"
2073                 fi
2074             fi
2075
2076             if [ "$LOGDIR" != "" ]; then
2077                 $sed -e '1,/Logged Visits/d' \
2078                     -e 's/<strong><img src=/\
2079 &/g' \
2080                     < $HTMLPAGE |
2081                 egrep ">$LOGUSERNAME<|strong> \($LOGUSERNAME\) \(" |
2082                 $sed -e 's#<a href="log.aspx?LUID.*##' \
2083                     -e 's#<a href=./profile[^ ]*##' \
2084                     > $HTMLPAGE2
2085                 lynx -dump $HTMLPAGE2 > $LOGDIR/$id.log
2086                 if [ "$filetime" -gt 0 ]; then
2087                     $touch -d "1/1/70 $filetime seconds" "$LOGDIR/$id.log"
2088                 fi
2089             fi
2090         done < $MERGEFILE
2091         IFS="$OIFS"
2092
2093         if [ "$LOGDIR" != "" ]; then
2094             # This is a hack, and might be inaccurate
2095             echo -n "Finds:" >&2
2096             MONTHS="  January |  February |  March |  April |  May |  June "
2097             MONTHS="$MONTHS|  July |  August | September |  October "
2098             MONTHS="$MONTHS|  November |  December "
2099             egrep "$MONTHS" $LOGDIR/*.log |
2100             egrep "Found it  |icon_smile|icon_happy|icon_camera" | wc -l >&2
2101         fi
2102
2103     fi
2104 }
2105
2106 ##############################################################################
2107 # end #include "geo-common-gc"
2108 ##############################################################################
2109 ##############################################################################
2110 # begin #include "geo-common-gpsdrive"
2111 ##############################################################################
2112
2113 #
2114 # default MySQL global options...
2115 #
2116 SQLUSER=gast
2117 SQLPASS=gast
2118 SQLDB=geoinfo
2119 SQLITEDB=$HOME/.gpsdrive/waypoints.db
2120 SQLTAG=Geocache
2121
2122 #
2123 # procedures for updating gpsdrive database via MySQL
2124 #
2125 #       Global Vars:    $SQLDB, $SQLTAG $OUTFILE
2126 #
2127 gpsdrive_create_210() {
2128     echo "CREATE TABLE poi ("
2129     echo "poi_id        INTEGER      PRIMARY KEY AUTOINCREMENT,"
2130     echo "name          VARCHAR(80)  NOT NULL default 'not specified',"
2131     echo "poi_type      VARCHAR(160) NOT NULL default 'unknown',"
2132     echo "lat           DOUBLE       NOT NULL default '0',"
2133     echo "lon           DOUBLE       NOT NULL default '0',"
2134     echo "alt           DOUBLE                default '0',"
2135     echo "comment       VARCHAR(255)          default NULL,"
2136     echo "last_modified DATETIME     NOT NULL default '0000-00-00',"
2137     echo "source_id     INTEGER      NOT NULL default '1',"
2138     echo "private       CHAR(1)               default NULL);"
2139     echo "CREATE TABLE poi_extra ("
2140     echo "poi_id         INTEGER       NOT NULL default '0',"
2141     echo "field_name     VARCHAR(160)  NOT NULL default '0',"
2142     echo "entry          VARCHAR(8192) default NULL);"
2143 }
2144
2145 gpsdrive_purge() {
2146     case "$GPSDRIVE_VER" in
2147     ""|"2.09")
2148         delcmd="delete from waypoints"
2149         echo "use $SQLDB;"
2150         echo "$delcmd where type like '$SQLTAG%';"
2151         ;;
2152     "2.10")
2153         if [ ! -s $SQLITEDB ]; then
2154             gpsdrive_create_210
2155         fi
2156         SQLTAG=`echo $SQLTAG | tr A-Z a-z`
2157         delcmd="delete from poi"
2158         echo "$delcmd where poi_type like '$SQLTAG%';"
2159         ;;
2160     esac
2161 }
2162
2163 gpsdrive_add_209() {
2164         delcmd="delete from waypoints"
2165         addcmd="replace into waypoints (name,lat,lon,type)"
2166         addcmd="insert into waypoints (name,lat,lon,type)"
2167         sqltag="$1"
2168         echo "use $SQLDB;"
2169         while read name lat lon type extra
2170         do
2171                 name=`echo "$name" | tr -d "'"`
2172                 # Primary key is autoincrementing id number, so delete
2173                 # the old record (if any) by name and type
2174                 if [ $PURGE = 0 ]; then
2175                     echo "$delcmd where name='$name' and type like '$SQLTAG%';"
2176                 fi
2177
2178                 if [ $DELETE = 0 ]; then
2179                     # Add the new record
2180                     if [ "$sqltag" = "Geocache" ]; then
2181                         tag="$type"
2182                     else
2183                         tag="$sqltag"
2184                     fi
2185                     echo "$addcmd values ('$name','$lat','$lon','$tag');"
2186                 fi
2187         done
2188 }
2189
2190 gpsdrive_add_210(){
2191     if [ ! -s $SQLITEDB ]; then
2192         gpsdrive_create_210
2193     fi
2194     echo "PRAGMA synchronous=OFF;"
2195     delcmd="delete from poi"
2196     addcmd="replace into poi (name,poi_type,lat,lon,alt,comment,last_modified)"
2197     addcmd="insert into poi (name,poi_type,lat,lon,alt,comment,last_modified)"
2198     sqltag="$1"
2199     sqltag=`echo "$sqltag" | tr A-Z a-z`
2200     OIFS="$IFS"
2201     IFS="|"
2202     #while read name lat lon type extra
2203     poi_id=0
2204     egrep -v 'Geocache Found|-ifound' \
2205         | tr -d "|" | tr '      ' '|' | tr -d "'" |
2206     while read index shortname description notes url urltext icon lat lon \
2207         lat32 lon32 latdecdir londecdir latdirdec londirdec latdir londir \
2208         altfeet altmeters excel timet diff terr container type \
2209         pathmiles pathkm placer yyyymmmdd hint lastfound extra
2210     do
2211         if [ "$placer" = "$USERNAME" ]; then
2212             continue
2213         fi
2214         if [ "$placer" = "dyl1231" ]; then
2215             continue
2216         fi
2217
2218         #name=`echo "$description" | tr -d "'"`
2219         #For Performance...
2220         name=${description//\'/}
2221
2222         # Primary key is autoincrementing id number, so delete
2223         # the old record (if any) by name and type
2224         if [ $PURGE = 0 ]; then
2225             echo "$delcmd where name='$name' and poi_type like '%$sqltag%';"
2226         fi
2227
2228         ((poi_id=poi_id+1))
2229         if [ $DELETE = 0 ]; then
2230             # Add the new record
2231             if [ "$sqltag" = "geocache" ]; then
2232                 tag="$type"
2233             else
2234                 tag="$sqltag"
2235             fi
2236             case "$type" in
2237             "Earthcache")       poi_type="geocache.geocache_earth";;
2238             "Event Cache")      poi_type="geocache.geocache_event";;
2239             "Mega-Event Cache") poi_type="geocache.geocache_event";;
2240             "Cache In Trash Out Event") poi_type="geocache.geocache_event";;
2241             "found")            poi_type="geocache.geocache_found";;
2242             "Multi-cache")      poi_type="geocache.geocache_multi";;
2243             "Unknown Cache")    poi_type="geocache.geocache_mystery";;
2244             "night")            poi_type="geocache.geocache_night";;
2245             "Traditional Cache") poi_type="geocache.geocache_traditional";;
2246             "Virtual Cache")    poi_type="geocache.geocache_virtual";;
2247             "Webcam Cache")     poi_type="geocache.geocache_webcam";;
2248             *)                      poi_type="geocache";;
2249             esac
2250             echo -n "$addcmd values ('$name','$poi_type',"
2251             echo -n "'$lat','$lon',"
2252             echo -n "'0.0'," #alt
2253             echo -n "'$type $container $diff/$terr. Last Found: $lastfound.  $hint',"
2254             echo "'0');"
2255         fi
2256     done
2257     IFS="$OIFS"
2258 }
2259
2260 gpsdrive_add() {
2261     case "$GPSDRIVE_VER" in
2262     ""|"2.09")  gpsdrive_add_209 "$1";;
2263     "2.10")     gpsdrive_add_210 "$1";;
2264     esac
2265 }
2266
2267 gpsdrive_mysql() {
2268     if [ "$OUTFILE" != "" ]; then
2269         cat >> $OUTFILE
2270     elif [ $DEBUG -gt 0 ]; then
2271         cat
2272     else
2273         mysql -u$SQLUSER -p$SQLPASS
2274     fi
2275 }
2276
2277 gpsdrive_sqlite3() {
2278     if [ "$OUTFILE" != "" ]; then
2279         cat >> $OUTFILE
2280     elif [ $DEBUG -gt 0 ]; then
2281         cat
2282     else
2283         sqlite3 $HOME/.gpsdrive/waypoints.db
2284     fi
2285 }
2286
2287 #
2288 # Extended list of gpsbabel output formats
2289 #
2290 gpsbabel_formats() {
2291     gpsbabel -? | sed -e '1,/File Types/d' -e '/Supported data filters/,$d'
2292     echo        "       gpsdrive.sql         " \
2293             "GpsDrive direct MySQL database insertion"
2294     echo        "       map[,geo-map-opts]   " \
2295             "Display map of waypoints using geo-map"
2296 }
2297
2298 ##############################################################################
2299 # end #include "geo-common-gpsdrive"
2300 ##############################################################################
2301
2302 #
2303 #       Set default options, can be overriden on command line or in rc file
2304 #
2305 UPDATE_URL=$WEBHOME/geo-nearest
2306 UPDATE_FILE=geo-nearest.new
2307
2308 read_rc_file
2309
2310 #
2311 #       Process the options
2312 #
2313
2314 gc_getopts "$@"
2315 shift $?
2316
2317 #
2318 #       Main Program
2319 #
2320 case "$#" in
2321 6)
2322         # Cut and paste from geocaching.com cache page
2323         # N 44° 58.630 W 093° 09.310
2324         LAT=`echo "$1$2.$3" | tr -d '\260\302' `
2325         LAT=`latlon $LAT`
2326         LON=`echo "$4$5.$6" | tr -d '\260\302' `
2327         LON=`latlon $LON`
2328         SEARCH="?origin_lat=$LAT&origin_long=$LON"
2329         ;;
2330 3)
2331         # lat lon cache-type
2332         LAT=`latlon $1`
2333         LON=`latlon $2`
2334         case "$3" in
2335         webcam)         cFilter="31d2ae3c-c358-4b5f-8dcd-2185bf472d3d";;
2336         earth*)         cFilter="c66f5cf3-9523-4549-b8dd-759cd2f18db8";;
2337         multi*)         cFilter="a5f6d0ad-d2f2-4011-8c14-940a9ebf3c74";;
2338         event)          cFilter="69eb8534-b718-4b35-ae3c-a856a55b0874";;
2339         virtual)        cFilter="294d4360-ac86-4c83-84dd-8113ef678d7e";;
2340         letter*)        cFilter="4bdd8fb2-d7bc-453f-a9c5-968563b15d24";;
2341         unknown)        cFilter="40861821-1835-4e11-b666-8d41064d03fe";;
2342         trad*)          cFilter="32bc9333-5e52-4957-b0f6-5a2c8fc7b257";;
2343         reg*)           cFilter="32bc9333-5e52-4957-b0f6-5a2c8fc7b257";;
2344         *)              error "Unknown cache-type '$3'";;
2345         esac
2346         SEARCH="?lat=$LAT&lng=$LON&cFilter=$cFilter"
2347         ;;
2348 2)
2349         LAT=`latlon $1`
2350         LON=`latlon $2`
2351         SEARCH="?origin_lat=$LAT&origin_long=$LON"
2352         ;;
2353 1)
2354         case "$1" in
2355         iraq|Iraq) SEARCH="?country_id=97" ;;
2356         u=*)
2357             SEARCH="?$1"
2358             ;;
2359         ul=*)
2360             SEARCH="?$1"
2361             ;;
2362         pq=*)
2363             SEARCH="?$1"
2364             ;;
2365         tx=*\&*)
2366             # tx=NNN&ul=USERNAME
2367             SEARCH="?$1"
2368             ;;
2369         tx=*)
2370             LAT=`latlon $LAT`
2371             LON=`latlon $LON`
2372             case "$1" in
2373             tx=webcam)  tx="tx=31d2ae3c-c358-4b5f-8dcd-2185bf472d3d";;
2374             tx=earth*)  tx="tx=c66f5cf3-9523-4549-b8dd-759cd2f18db8";;
2375             tx=multi*)  tx="tx=a5f6d0ad-d2f2-4011-8c14-940a9ebf3c74";;
2376             tx=event)   tx="tx=69eb8534-b718-4b35-ae3c-a856a55b0874";;
2377             tx=virtual) tx="tx=294d4360-ac86-4c83-84dd-8113ef678d7e";;
2378             tx=letter*) tx="tx=4bdd8fb2-d7bc-453f-a9c5-968563b15d24";;
2379             tx=unknown) tx="tx=40861821-1835-4e11-b666-8d41064d03fe";;
2380             tx=trad*)   tx="tx=32bc9333-5e52-4957-b0f6-5a2c8fc7b257";;
2381             tx=reg*)    tx="tx=32bc9333-5e52-4957-b0f6-5a2c8fc7b257";;
2382             *)          tx=$1;;
2383             esac
2384             SEARCH="?$tx&lat=$LAT&lng=$LON"
2385             ;;
2386         guid=*)
2387             SEARCH="bookmarks/view.aspx?$1"
2388             ;;
2389         *[0-9]*)
2390             ZIP=$1
2391             SEARCH="?zip=$ZIP"
2392             ;;
2393         *)
2394             error "'$1' isn't something I understand how to search for"
2395             ;;
2396         esac
2397         ;;
2398 0)
2399         SEARCH="?origin_lat=$LAT&origin_long=$LON"
2400         ;;
2401 *)
2402         usage
2403         ;;
2404 esac
2405
2406 gc_query