4 # DO NOT EDIT! This file is generated from geo-nearest.sh
8 # geo-nearest: Fetch list of nearest geocaches.
10 # Requires: curl; gpsbabel; bash or ksh;
11 # mysql (if using the gpsdrive.sql output option)
13 # Donated to the public domain by Rick Richardson <rickrich@gmail.com>
15 # Use at your own risk. Not suitable for any purpose. Not legal tender.
17 # $Id: geo-nearest.sh,v 1.49 2013/02/18 21:41:06 rick Exp $
25 `basename $PROGNAME` - Fetch a list of nearest geocaches
28 `basename $PROGNAME` [options]
30 `basename $PROGNAME` [options] latitude longitude
32 `basename $PROGNAME` [options] latitude longitude cache-type
34 `basename $PROGNAME` [options] zipcode
36 `basename $PROGNAME` [options] u=<username>
38 `basename $PROGNAME` [options] ul=<username>
40 `basename $PROGNAME` [options] pq=<pocket-query>
42 `basename $PROGNAME` [options] tx=<bookmark-id>
44 `basename $PROGNAME` [options] -b bookmark
45 `basename $PROGNAME` [options] guid=<bookmark-id>
48 Fetch a list of nearest geocaches.
51 A premium member (\$30/yr) OR a basic member (free) login at:
52 http://www.geocaching.com
53 Visit a cache page and click the "Download to EasyGPS" link at least
54 once so you can read and agree to the license terms. Otherwise, you
55 will not get any waypoint data.
57 curl http://curl.haxx.se/
58 gpsbabel http://gpsbabel.sourceforge.net/
65 A basic member will get caches very slow (20 cache pages per minute)
66 because we have to get the actual cache pages. They will be stored in:
67 ~/.geo/caches/GCXXXX.html.
68 Of course, after running this command, geo-html2gpx could be run.
71 Nearest 20 caches, display shortnames:
75 Search nearest 500 caches for virtual caches not yet found:
77 geo-nearest -n500 -Ivirtual -Xifound
79 Add nearest 50 caches to a GpsDrive SQL database
81 geo-nearest -n50 -f -s -S
83 Purge the existing SQL database of all geocaches, and fetch
86 geo-nearest -S -P -s -n200
88 640x480 map of nearest caches using map source 2:
90 geo-nearest -omap,"-a2 -W640 -H480"
94 geo-nearest -n200 -Xifound -udyl1231 -pPW | awk '{print \$1}' >1.foo
95 geo-nearest -n200 -Xifound -urickrich -pPW |awk '{print \$1}' >2.foo
96 geo-gid -otabsep \$(comm -12 1.foo 2.foo) >both
98 Fetch by owner placed:
100 geo-nearest u=team-deadhead
102 Fetch by owner found:
104 geo-nearest ul="AAA+of+MichigAn&sortdir=asc&sort=placed"
108 # nearby caches of this (puzzle) type, that I haven't found
109 geo-nearest -n500 -f -otabsep tx=40861821-1835-4e11-b666-8d41064d03fe |
110 geo-mystery >> Caches/rick.ts
112 Also, tx=webcam, tx=earth, tx=multi, tx=event, tx=virtual, tx=letter,
113 tx=unknown, tx=trad (tx=reg is an alias).
115 Fetch by cache-type method:
117 # nearby caches of this (puzzle) type, that I haven't found
118 geo-nearest -n500 -f -otabsep unknown |
119 geo-mystery >> Caches/rick.ts
121 Also, cache-type is webcam, earth, multi, event, virtual, letter,
122 unknown, trad (reg is an alias).
124 Fetch a bookmark list:
128 geo-nearest guid=baae5bf9-4315-4874-b7fb-ac84c9585641
132 geo-nearest -q "Needs Maintenance"
134 geo-nearest pq=08be103b-ffd1-4e27-992f-616e144e24df
141 geo-newest, geo-found, geo-placed, geo-keyword, geo-code, geo-map,
149 ##############################################################################
150 # begin #include "geo-common"
151 ##############################################################################
153 # I doubt this stuff will work in other than english
157 # Common global constants
159 UA="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)"
160 WEBHOME="http://geo.rkkda.com/"
163 # Common global variables
170 # Report an error and exit
173 echo "`basename $PROGNAME`: $1" >&2
178 if [ $DEBUG -ge $1 ]; then
179 echo "`basename $PROGNAME`: $2" >&2
184 if [ $VERBOSE -ge $1 ]; then
190 if [ $DEBUG -ge $DBGCMD_LVL ]; then
198 # procedure to remove cruft files
201 if [ $DEBUG = 0 -a "$CRUFT" != "" ]; then
204 [ -f $i ] && rm -f $i
209 trap remove_cruft EXIT
212 # Convert DegDec, MinDec, or DMS lat/lon to DegDec
215 # Handle NSEW prefixes
216 arg1=`echo "$1" | sed -e 's/^[nNeE]//' -e 's/^[wW]/-/' -e 's/^[sS]/-/'`
217 # If negative, print the sign then take the absolute value
219 -*) echo -n "-"; arg1=`echo "$arg1" | sed 's/^-//'`;;
221 # Now handle the 3 different formats
225 *.*.*) echo "$arg1" \
226 | sed -e 's/,//' -e 's#\([^.]*\)\.#\1 #' -e 's#$# 6k 60/+p#' \
230 echo "$arg1" | sed 's/^lat=//'
233 echo "$arg1" | sed 's/^lon=//'
239 2) echo "6k $arg1 $2 60/+p" | dc;;
240 3) echo "6k $arg1 $2 60/ $3 3600/++p" | dc;;
245 # Convert DegDec to MinDec with optional NS/EW
248 awk -v v=$1 -v sym=$2 \
250 function abs(x) { return (x>=0) ? x : -x }
254 printf "%d.%06.3f\n", i, abs(v-i) * 60
256 printf "%s%d.%06.3f\n", \
257 i >= 0.0 ? substr(sym, 1, 1) : substr(sym, 2, 1), \
258 abs(i), abs(v-i) * 60
263 # Read RC file, if there is one
266 if [ -f $HOME/.georc ]; then
268 # Allow LAT/LON in rc file to be in any of the formats that we grok
269 if [ "" != "$LAT" ]; then
270 if ! is_latlon $LAT 0; then
271 error "Latitude '$LAT' is not parsed in .georc!"
275 if [ "" != "$LON" ]; then
276 if ! is_latlon 0 $LON; then
277 error "Longitude '$LON' is not parsed in .georc!"
282 cat <<-EOF > $HOME/.georc
284 # These are the default values for the geo-* series of programs
285 # Please edit this file as needed. Setting values for
286 # USERNAME, PASSWORD, LAT/LON, and STATE are required.
289 #################################
290 # Login and paid membership status for www.geocaching.com...
295 #################################
296 # Your HOME lat/lon and state...
301 #################################
302 # Default map scale, font, and source...
307 #################################
308 # Login for terraserver.com...
309 #TSCOM_EMAIL=xxx@yyy.com
312 #################################
316 #GEOMYSTERY=$HOME/.geo-mystery
317 #OCMYSTERY=$HOME/.oc-mystery
322 error "First time user: please review and edit $HOME/.georc"
326 if [ `uname` = 'Darwin' ]; then
330 PATH=$PATH:/usr/local/bin:/opt/local/bin
339 # Get the value from a name= value= pair in a file
342 # <input type="hidden" name="__EVENTTARGET" value=""
345 eval $what=`$sed -n "s/^.*\"$what\" *value=\"\([^\"]*\)\".*/\1/p" < $where`
351 # incomplete, just does what we need it to do
354 echo "$1" | sed -e 's/\+/%2B/g' -e 's/\&/%26/g' #-e 's/\//%2F/'
364 split("1 2 3 4 5 6 7 8 9 A B C D E F", hextab, " ")
366 for (i = 1; i <= 255; ++i) ord[ sprintf ("%c", i) "" ] = i + 0
370 for (i = 1; i <= length($0); ++i ) {
371 c = substr ($0, i, 1)
373 if (val >= 97 && val <= 122) #0x61-0x7A
375 else if (val >= 65 && val <= 90) #0x41-0x5A
377 else if (val >= 48 && val <= 57) #0x30-0x39
379 else if (val >= 45 && val <= 46) #0x2D-0x2E
382 encoded = encoded "+"
383 else if (val < 128) {
386 encoded = encoded "%" hextab[hi] hextab[lo]
392 encoded = encoded "%" hextab[hi] hextab[lo]
396 encoded = encoded "%" hextab[hi] hextab[lo]
405 # return true if current arguments appear to be a lat/lon
408 if [ "$#" -lt 2 ]; then
412 lat=*) ;; # cut/paste from GPX file
413 [NS]) return 0;; # cut/paste from gc.com
421 [EWew][0-9]*) return 0;;
422 [-][0-9]*) return 0;;
429 # split lines between two strings
435 split_lines_between() {
440 ##############################################################################
441 # end #include "geo-common"
442 ##############################################################################
443 ##############################################################################
444 # begin #include "geo-common-gc"
446 # $Id: geo-common-gc,v 1.313 2013/02/01 14:29:33 rick Exp $
447 ##############################################################################
450 # Common global constants
452 UA="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)"
453 UA="Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en) AppleWebKit/XX (KHTML, like Gecko) Version/ZZ Safari/YY"
454 GEO="http://www.geocaching.com"
455 GEOS="https://www.geocaching.com"
458 # Global variables that can be overridden on command line or rc file
468 CYGWIN*) CURL_OPTS=-k;;
475 COOKIE_FILE=$HOME/.geocookies
489 GEOSLEEP=${GEOSLEEP:-8}
492 # Common options handling
497 -b bookmark Use list "bookmark" [none]
498 -q query geo-nearest: Use PQ list "query" [none]
499 -q search geo-newest: Use search "search" [none]
500 -c Remove cookie file when done
501 -f Do not report any found or unavailable caches
502 -m Do not report any members-only caches
503 -F Report caches found by the login 'username' as unfound
504 -n num Return "num" caches [$NUM]
505 -s Output short names for the caches (gpsbabel option)
506 -I term Include only caches with 'term' [$INCLUDE]
507 -X term Exclude caches with 'term' [$EXCLUDE]
508 terms: ~ (exclude none), unfound, ifound, soc, unavail,
509 regular, multi, virtual, webcam, event, hybrid, cito
510 -r radius Display only caches with radius (e.g. -r 25M)
511 -M mystery Use file 'mystery' for unknown/mystery/puzzle
512 caches [$GEOMYSTERY]. Awk Format:
516 i.e.: GC2CBVB n44.45.123 w93.00.321 Final
518 -u username Username for http://www.geocaching.com
519 -p password Password for http://www.geocaching.com
520 -o format Output format, -o? for possibilities [$OUTFMT]
521 plus "gpsdrive.sql" for direct insertion into MySQL DB
522 plus "map[,geo-map-opts]" to display a geo-map.
523 -O filename Output file, if not stdout
524 -S Alias for -o gpsdrive.sql
525 -d For -S, just delete selected records
526 -P For -S, purge all records of type -t $SQLTAG*
527 -t type For -ogpsdrive.sql, the waypoint type [$SQLTAG]
528 -H htmldir Also fetch the printable HTML pages (slowly)
529 -L logdir Also fetch the plain text log entries (slowly)
530 For -H or -L, the limit is 1500 updated caches/day.
531 -! "lpr -Plp" Print HTML pages
532 -E var=val Set environment "var" to "val"
534 -D lvl Debug level [$DEBUG]
535 -U Retrieve latest version of this script
538 Defaults can also be set with variables in file \$HOME/.georc:
540 PASSWORD=password; USERNAME=username; SOC=0|1;
541 LAT=latitude; LON=logitude; GEOMYSTERY=/dev/null;
542 NUM=num; OUTFMT=format; BABELFLAGS=-s;
543 SQLUSER=gast; SQLPASS=gast; SQLDB=geoinfo;
546 Geocaching.com date formats that are compatible:
548 GC Format Example Compatible
549 YYYY-MM-DD 2011-07-13 yes
550 YYYY/MM/DD 2011/07/13 yes
551 MM/DD/YYYY 07/13/2011 yes
552 DD/MM/YYYY 13/07/2011 yes if DATEFMT=1 in \$HOME/.georc
553 DD/Mmm/YYYY 13/Jul/2001 no
554 Mmm/DD/YYYY Jul/13/2011 no
555 DD Mmm YY 13 Jul 11 yes (english only)
559 http://www.geocaching.com/account/ManagePreferences.aspx
565 # Defaults for options that cannot be overriden in the RC file
577 while getopts "!:E:H:L:I:X:b:q:cdfFmM:n:o:O:p:Pr:sSt:u:D:Uh?-" opt
580 "!") CMDPIPE="$OPTARG";;
581 b) BOOKMARK="$OPTARG";;
582 q) POCKETQUERY="$OPTARG";;
586 f) FOUND=0; EXCLUDE="$EXCLUDE|-ifound";;
588 M) GEOMYSTERY="$OPTARG";;
590 if [ "$INCLUDE" = "*" ]; then
595 INCLUDE="$INCLUDE-$OPTARG"
598 if [ "" = "$OPTARG" -o "~" = "$OPTARG" ]; then
599 EXCLUDE=-ExClUdEnOtHiNg
601 EXCLUDE="$EXCLUDE|-$OPTARG"
609 *) error "Not a number: '$NUM'";;
611 if [ "$NUM" -lt 1 -o "$NUM" -gt 999999 ]; then
612 error "NUM is not between 1 and 999999 ($NUM)"
616 RADIUS_NUM=`awk -v "N=$RADIUS" 'BEGIN{printf "%d\n", N}'`
620 RADIUS_MILES=`dc -e "6k $RADIUS_NUM 1.609344/p" `
623 RADIUS_MILES=$RADIUS_NUM
628 s) BABELFLAGS="$BABELFLAGS -s"; GEONUKE=",nuke_placer";;
629 S) OUTFMT="gpsdrive.sql";;
630 t) SQLTAG="$OPTARG";;
631 u) USERNAME="$OPTARG";;
632 p) PASSWORD="$OPTARG";;
633 o) OUTFMT="$OPTARG";;
634 O) OUTFILE="$OPTARG";;
635 H) HTMLDIR="$OPTARG";;
636 L) LOGDIR="$OPTARG";;
638 U) echo "Getting latest version of this script..."
639 curl $CURL_OPTS -o$UPDATE_FILE "$UPDATE_URL"
640 chmod +x "$UPDATE_FILE"
641 echo "Latest version is in $UPDATE_FILE"
648 shiftamt=`expr $OPTIND - 1`
652 0) EXCLUDE="$EXCLUDE|-soc";;
657 OUTFMT=tiger,newmarker=grnpin,iconismarker
661 MAPOPTS=`echo "$OUTFMT" | $sed -n 's/map,\(.*\)$/\1/p'`
662 OUTFMT=tiger,newmarker=grnpin,iconismarker
677 LOGUSERNAME="$USERNAME"
686 get_value __EVENTTARGET $file
687 get_value __EVENTARGUMENT $file
688 get_value __VIEWSTATEFIELDCOUNT $file
689 get_value __VIEWSTATE $file; __VIEWSTATE=`urlencode "$__VIEWSTATE"`
690 get_value __EVENTVALIDATION $file;
691 __EVENTVALIDATION=`urlencode "$__EVENTVALIDATION"`
693 viewstate="$viewstate -d __VIEWSTATE=$__VIEWSTATE"
694 if [ "$__VIEWSTATEFIELDCOUNT" != "" ]; then
695 viewstate="$viewstate -d __VIEWSTATEFIELDCOUNT=$__VIEWSTATEFIELDCOUNT"
697 while test $fc -lt $__VIEWSTATEFIELDCOUNT; do
698 get_value __VIEWSTATE$fc $file
699 eval xxx=\${__VIEWSTATE$fc}
700 eval __VIEWSTATE$fc=`urlencode "$xxx"`
701 eval xxx=\${__VIEWSTATE$fc}
702 viewstate="$viewstate -d __VIEWSTATE$fc=$xxx"
709 # login to geocaching.com
711 # Outputs: $__VIEWSTATE
714 _username=`urlencode "$1"`
715 _password=`urlencode "$2"`
717 [ "$_username" != dummy ] || error "You need a www.geocaching.com username"
718 [ "$_password" != dummy ] || error "You need a www.geocaching.com password"
721 # Get the login page so we can get a valid viewstate
723 LOGINPAGE=${TMP}-login.html
724 CRUFT="$CRUFT $LOGINPAGE"
726 URL="$GEOS/login/default.aspx"
727 debug 1 "curl $CURL_OPTS $URL"
728 curl $CURL_OPTS -s -A "$UA" -c$COOKIE_FILE \
732 || error "Unable to connect to gc.com; Try: CURL_OPTS=\"--sslv3\" in ~/.georc"
733 get_value __VIEWSTATE $LOGINPAGE
734 __VIEWSTATE=`urlencode "$__VIEWSTATE"`
735 get_value __EVENTVALIDATION $LOGINPAGE
736 __EVENTVALIDATION=`urlencode "$__EVENTVALIDATION"`
743 URL="$GEOS/login/default.aspx"
746 # RER 05/05/12: bug in recent curl/openssl use "--sslv3" for a workaround
748 dbgcmd curl $CURL_OPTS --sslv3 -s -A "$UA" -b$COOKIE_FILE -c$COOKIE_FILE \
749 -d __VIEWSTATE="$__VIEWSTATE" \
750 -d __EVENTVALIDATION="$__EVENTVALIDATION" \
752 -dctl00%24ContentBody%24tbUsername="$_username" \
753 -dctl00%24ContentBody%24tbPassword="$_password" \
754 -dctl00%24ContentBody%24btnSignIn=Login \
755 -L "$URL" > $LOGINPAGE
756 if [ ! -s $LOGINPAGE ]; then
757 error "curl: returned /dev/null for '$LOGINPAGE'"
759 if grep -q "FormErrorWidget" $LOGINPAGE; then
760 error "Login username/password does not match."
762 if grep -q "If you are human" $LOGINPAGE; then
763 error "Have to login this time with a web browser because of Recaptcha"
765 if grep -q "temporarily down for maintenance." $LOGINPAGE; then
766 error "Geocaching.com is temporarily down for maintenance."
768 # Make case insignificant
769 if ! grep -y -q "ctl00_ContentBody_LoggedInPanel" $LOGINPAGE; then
770 error "BUG: gc.com changed the login page."
775 # procedure to nag about agreeing to EasyGps download license
779 You have not agreed to the waypoint download license at $GEO
781 Click one of the waypoint license agreement links at $GEO,
782 read and agree to the license terms, then try this program again.
787 # getcids infile cidfile xtrafile archfile number
789 # Wade thru the HTML and produce lists of found, notfound and new CIDs
797 -v "USERFOUND=$USERFOUND" \
798 -v "VARTIME=$VARTIME" \
801 -v "DATEFMT=$DATEFMT" \
805 function debug(lvl, text) {
807 print text > "/dev/stderr"
809 function hex2dec(x, val) {
810 for (val = 0; length(x); x = substr(x, 2))
811 val = 16*val + index("0123456789ABCDEF", substr(x, 1, 1)) - 1
814 # Convert GC0000 to 58913
815 function wp2id(wp, val) {
818 print "wp2id: " wp " ..." > "/dev/stderr"
819 if (length(wp) <= 4 && wp < "G000")
823 print "wp2id hex: " val " ..." > "/dev/stderr"
826 set = "0123456789ABCDEFGHJKMNPQRTVWXYZ"
828 for (pos = 1; pos <= length(wp); ++pos)
831 val += index(set, substr(wp, pos, 1)) - 1;
835 print "wp2id id: " val " ..." > "/dev/stderr"
838 function id2wp(id, val) {
843 gid = sprintf("GC%04X", id)
846 GcOffset = 16 * 31 * 31 * 31 - 65536
847 GcSet = "0123456789ABCDEFGHJKMNPQRTVWXYZ"
849 for (i = 1; i <= 4; ++i)
851 gid = substr(GcSet, id%31 + 1, 1) gid
854 tmp = substr(GcSet, id%31 + 1, 1)
866 print "id = ", id, "wp = ", gid > "/dev/stderr"
870 function hash2sdt(text,
871 day, cmd, i, value, mod, s0, t0, d0) {
874 code[1] = "PDMTRXBJjZNH9fKW8vetlCgbos24mydknahrcOxYqLwGSE370z65ApVFQ"
875 code[2] = "6JKt4b70rHoeTLqQB2jANwcYSMW9xPk3aplE5hGvyfsZDgFmndOVRC8zX"
876 code[3] = "Rl2QaDdTsnK7rOZoywzx0H58NLCAGqgtp43mvYJVkFEejMWB69cXhfSbP"
877 code[4] = "jShz6gF3OvonKEWkJCbZf4GeDX25VcyPsqwRYpLM8BtATm0x7l9aHQdNr"
878 code[5] = "aJs2OWpoYfN9zvDHCwZrm83b4SKkFhXcAL0QPgBy7tMdRGlqxjEVn65Te"
879 code[6] = "g6aXp7rqvkEl5JLnRDhfWGHTeA9OMQYdwS2BcbK0jCyFx4sPo83tmzZVN"
880 code[7] = "MlocEO5eAgTFfLXzyPDk2qJhjBn98QSdNHR47mGWtY3ZvVxCpr0Ks6abw"
881 code[8] = "p7SPbvzM4VkQ8f25cJL0gBlWjYmFoKhArXRHZntTGe6CNsyE3a9qOdwDx"
882 code[9] = "znlBb8GHNdhCKyOpo53ePX9xfkvjD0aTEAVZ72qSQwrmMYct6JgRFWLs4"
883 code[10] = "KXztRHdrkBwSWEn3AN2Fh8vOL5QMjx6oTgyCD7JYpaGmVcf9Pelb4q0Zs"
884 code[11] = "M3odkDCgQa7GWK9pYh8tLAl6xFecRwPZNbTz0s42rqfOnSmvEjVyB5JXH"
885 code[12] = "gweGxNAYCKfLdrb68X93ahjHMDtTWRFJVZvl0po5ySmOQnscPz4Ekq2B7"
886 code[13] = "p2zelnEox9HqkGF4V5AZ6LbMBsy0dQc8J3fDXvON7jWgrKYCSTaPtRhmw"
887 code[14] = "cy2gjCAJt3KmMxbr5OzQeYsvHpLF8fRVaoNXdGBW0l7ESZDP9qkw4Th6n"
888 code[15] = "9KQZlwfBsCqN6oWkYhrSpDAEP48c5m3eJn2jVHLdybMx0gaOGRXzvFTt7"
889 code[16] = "7j6CJPfqbMygan42HwxFBcrK3RYLstmOXWGeSQZ9kzl0VE8oDTpAhd5Nv"
890 code[17] = "YScWw7gsx49ebLAqMGotHRX3fnCv6FBOkzVE5jdlmPTarDKQJyp8Z0hN2"
891 code[18] = "BgwR6oskAcVPqeYdLKr3ONvflXz2nDSbxMQJEhyZC0pWHm759GFa4tT8j"
892 code[19] = "9QyolrBq37he4nbCTcXEkmGHjYS02RDgMZLfOtavVzKNFxPs58p6JwWAd"
893 code[20] = "8OwsHykXLJ3tqWZmR2pbVagMcTdnSC9FveN5z0xYjGK6frh7Q4EPDBAlo"
894 code[21] = "RVBh8qEaN4Aw2Ob5egSytWjQD6oFmKvXfYnCzcx73kJsHrd9TLPZM0Glp"
895 code[22] = "7MqQGJkKzZvdgmRXBH8CYeAVf3cy29LOW4PNaD0rsnowEpTbF5Sxhtl6j"
896 code[23] = "8C26QBoWjEPcJ3Sf7TYtexXlMNyqArm0KD9HOLvZakGwnVbFsdp45Rzhg"
897 code[24] = "6e5fC2S4Bwcm9pVKyqdFE3WlRDroTLJgGtN8ZQsAabMzHYO0Xvj7Pnxhk"
898 code[25] = "JXHgMGcAVNpDEwWYvyBl025m89dOoTSKPxhstjq6e4QrCk3aZFbzfn7RL"
899 code[26] = "Yp3ARoesjKXB2cadmkHvFQM7SEJtqf0l9TwzbPLrZDxO8h5C6NnGWyVg4"
900 code[27] = "9kJaM4HonPWtcQvsYSEK3Vy5pjCRFqNO6lgr0x7mh2b8wdTGDzfLXeABZ"
901 code[28] = "4n2gSwqzGB7C0MpV3yDY9FTtol8XjkJNcEPr6LxhZQKdRvHOfsA5eWmab"
902 code[29] = "P6G8jsOxBCqf3k74DnXtAvl2ThabeEHJVgZp5wSzF9NWRcYMyL0dromKQ"
903 code[30] = "WVwgM93zjmNcLpHvoSK2Q5bCxaATGsryXn6Bhfd0Z8qtERk7PYOJD4Fle"
904 code[31] = "PZqTVol4HOXz0GWt38Qec59rKMyabfhE2DdsNxpF7RASkBjnYwvCgmJ6L"
905 cmd = sprintf("%s -u +%%-d", DATE)
906 cmd | getline day; close(cmd)
911 for (i = 1; i <= length(text); ++i) {
912 value = 57 * value + index(thiscode, substr(text, i, 1)) - 1
916 mod = (value - 131586) % 16777216
918 s0 = int(mod / 131072);
919 t0 = int((mod % 131072) / 256)
920 d0 = int((mod % 131072) % 256)
924 if (s0 == 0) size = "not chosen"
925 else if (s0 == 1) size = "Micro"
926 else if (s0 == 2) size = "Regular"
927 else if (s0 == 3) size = "Large"
928 else if (s0 == 4) size = "Virtual"
929 else if (s0 == 5) size = "Unknown"
930 else if (s0 == 7) size = "Small"
934 if (t0 == 0) terrain = 1.0
935 else if (t0 == 1) terrain = 1.5
936 else if (t0 == 2) terrain = 2.0
937 else if (t0 == 3) terrain = 2.5
938 else if (t0 == 4) terrain = 3.0
939 else if (t0 == 5) terrain = 3.5
940 else if (t0 == 6) terrain = 4.0
941 else if (t0 == 7) terrain = 4.5
945 if (d0 == 0) difficulty = 1.0
946 else if (d0 == 1) difficulty = 1.5
947 else if (d0 == 2) difficulty = 2.0
948 else if (d0 == 3) difficulty = 2.5
949 else if (d0 == 4) difficulty = 3.0
950 else if (d0 == 5) difficulty = 3.5
951 else if (d0 == 6) difficulty = 4.0
952 else if (d0 == 7) difficulty = 4.5
953 else difficulty = 5.0
956 function begin_new_entry() {
957 # Beginning of a new entry, reset variables to defaults
971 container = "unknown"
972 sendgps = 0 # Non-subscription members
975 #gctype = "Traditional cache"
981 print "Begin " NUM " ..." > "/dev/stderr"
983 function parse_dates(text, cmd, val) {
984 sub("[*]$", "", text)
985 sub("^ *", "", text) # remove leading blanks
986 debug(5, "parse_dates: text=" text)
989 cmd = sprintf("%s -d \"%s\" +%%s", DATE, "12am today")
990 cmd | getline val; close(cmd)
992 else if (text ~ /Yesterday/)
994 cmd = sprintf("%s -d \"%s\" +%%s", DATE, "12am yesterday")
995 cmd | getline val; close(cmd)
997 else if (text ~ /ago/)
999 cmd = sprintf("%s -d \"12am %s\" +%%s", DATE, text)
1000 cmd | getline val; close(cmd)
1002 else if (text ~ /[0-9][^<]*[0-9]/ )
1004 # GC Format Example Compatible
1005 # YYYY/MM/DD 2011/07/13 yes
1006 # MM/DD/YYYY 07/13/2011 yes
1007 # DD/MM/YYYY 13/07/2011 yes if DATEFMT = 1
1008 # DD/Mmm/YYYY 13/Jul/2001 no
1009 # Mmm/DD/YYYY Jul/13/2011 no
1010 nf = split(text, fld, "/")
1011 if (nf == 3 && DATEFMT == 1)
1014 debug(5, "parse_dates: fld=" fld[1] "." fld[2] "." fld[3])
1015 cmd = sprintf("%s -d \"12am %s\" +%%s", DATE,
1016 fld[2] "/" fld[1] "/" fld[3])
1017 cmd | getline val; close(cmd)
1022 cmd = sprintf("%s -d \"12am %s\" +%%s", DATE, text)
1023 cmd | getline val; close(cmd)
1028 debug(5, "parse_dates: val=" val)
1032 q = sprintf("%c", 39)
1034 gctype = "Traditional cache"
1037 # Test: geo-nearest -s (18)
1038 # Test: geo-newest -s (20)
1039 # Test: geo-nearest -b multi (8)
1041 # For geo-nearest AND geo-nearest -b
1042 /class=".* Data BorderTop"/ {
1044 debug(3, "Data BorderTop")
1050 inrecord && tdcnt >= 1 && tdcnt <= 10 {
1052 print "tdcnt " tdcnt ", text: " $0 > "/dev/stderr"
1054 /<a href="\/seek\/cache_details.aspx?[^>]*>/ {
1057 # remove strikeout class
1058 sub("<span class[^>]*>", "", name)
1059 sub("</span>", "", name)
1060 sub(".*<span>", "", name)
1061 sub("</span>.*", "", name)
1063 sub(".*<a href=.../seek/cache_details.aspx?[^>]*>", "", name)
1064 sub("</a>*.", "", name)
1065 sub("</[^>]*>", "", name)
1066 sub("</[^>]*>", "", name)
1067 sub("<[^>]*>", "", name)
1068 sub("<[^>]*>", "", name)
1069 debug(5, "name: " name)
1071 # bookmarks by guid=...
1072 /<a href="http:..www.geocaching.com\/seek\/cache_details.aspx.guid=[^>]*>/ {
1076 sub(".*img src[^>]*>", "", name)
1077 sub("</span.*", "", name)
1078 sub(".*Strike.>", "", name)
1079 sub("</a.*", "", name)
1080 debug(5, "bid name: " name)
1083 sub(".* alt=.", "", gctype)
1084 sub(". />.*", "", gctype)
1085 debug(5, "gctype: " gctype)
1091 sub(".*<a href=.*/seek/cache_details.aspx?[^>]*>", "", gcid)
1092 sub("<.*", "", gcid)
1094 debug(5, "gcid: " gcid)
1102 /Premium Member Only Cache/ {
1105 tdcnt == (9-0) && /[0-9][0-9]*.[0-9][0-9]*/ {
1108 sub(/^ */, "", date)
1109 sub(/.*small.>/, "", date)
1110 sub(/ *<.*/, "", date)
1111 sub(/\015/, "", date)
1112 debug(5, "placedt text: "date)
1113 placedt = parse_dates(date)
1114 debug(5, "placedt: " placedt)
1117 # Yesterday<strong>*</strong><br />
1118 # <span class="Success">
1119 # 3 days ago*</span>
1123 # Yesterday<strong>*</strong><br />
1124 # <span class="Success">
1127 tdcnt == (10-0) && /[a-zA-Z0-9*>]<br .>/ {
1128 # catch dates DD MMM YY, N days ago, Today, Yesterday
1131 sub(".*<td>", "", date)
1132 sub("<.*", "", date)
1133 lastfoundt = parse_dates(date)
1135 debug(5, "lastfoundt: " lastfoundt)
1137 tdcnt == (10-0) && /[a-zA-Z0-9*>]<.span>/ {
1140 sub(".*Success.>", "", date)
1141 sub("<.*", "", date)
1142 ifoundt = parse_dates(date)
1143 debug(5, "ifoundt: " ifoundt)
1146 # i.e. <font color="red">
1147 avail = 0; archived = 1
1149 /<\/strike><\/font/ {
1150 # i.e. <font color="red">
1151 avail = 0; archived = 1
1156 /class=".*Warning Strike.*">/ {
1157 debug(5, "***archived***")
1158 avail = 0; archived = 1
1160 /class=".*Strike">/ {
1163 tdcnt == 6 && /^ *by / {
1165 sub("^ *by ", "", placer)
1166 sub("\015", "", placer)
1167 debug(5, "placer: <" placer ">")
1169 tdcnt == 6 && /^ *GC/ {
1171 sub("^ *", "", gcid)
1172 sub("\015", "", gcid)
1173 debug(5, "GCID: <" gcid ">")
1176 /alt=.Your Geocache/ {
1178 # A mistake in the HTML! should be alt="my cache"
1184 /alt="Earthcache"/ {
1186 gctype = "Earthcache"
1188 /alt="Event Cache"/ {
1190 gctype = "Event cache"
1192 /alt="Cache In Trash Out Event"/ {
1194 gctype = "Cache In Trash Out Event"
1196 /alt="Traditional Cache"/ {
1198 gctype = "Traditional cache"
1200 /alt="Letterbox Hybrid"/ {
1202 gctype = "Letterbox Hybrid"
1204 /alt="Multi-cache"/ {
1206 gctype = "Multi-Cache"
1208 /alt="Unknown Cache"/ {
1210 gctype = "Unknown Cache"
1212 /alt="Virtual Cache"/ {
1214 gctype = "Virtual cache"
1216 /alt="Webcam Cache"/ {
1218 gctype = "Webcam Cache"
1220 /alt="Wherigo Cache"/ {
1222 gctype = "Wherigo Cache"
1224 /alt="Mega-Event Cache"/ {
1226 gctype = "Mega-Event Cache"
1229 i = match($0, "value=.")
1230 cid = substr($0, i+7, 99) + 0
1233 i = match($0, "value=.")
1234 bid = substr($0, i+7, 99) + 0
1236 #/ImgGen.seek.CacheInfo.ashx?v=/ {
1237 archived && /[?]v=/ {
1239 debug(5, "sdt text: " text)
1240 sub(".*?v=", "", text)
1241 sub(". .*", "", text)
1242 debug(5, "sdt text: " text)
1244 debug(5, "size: " size " difficulty: " difficulty " terrain: " terrain)
1247 /Unapproved cache/ && inrecord {
1248 avail = 0; archived = 1
1250 /class=.Checkbox NoBolding./ {
1251 sendgps = 1 # Subscription members
1253 /<\/[tT][rR]>/ && inrecord {
1254 # RER mod 11/22/10: if (0 && ..
1255 if (0 && sendgps == 0)
1260 strtype = "Geocache"
1262 if (soc) strtype = strtype "-soc"
1263 if (unfound) strtype = strtype "-unfound"
1264 else if (ifound) strtype = strtype "-ifound"
1265 if (!avail) strtype = strtype "-unavail"
1266 if (archived) strtype = strtype "-archived"
1268 strtype = strtype "-" type
1270 # gpsbabel only allows one time in the DB, figure out what
1271 # time to use for this, but always carry all three times
1273 if (iplaced) ifoundt = placedt
1275 if (VARTIME == "placed") vartime = placedt
1276 else if (VARTIME == "ifound") vartime = ifoundt
1277 #else if (foundt > 0) vartime = foundt
1278 else vartime = placedt
1280 # date -d 1970-01-01 1237093200 sec +%Y%m%d
1282 cmd = sprintf("%s -d \"1970-01-01 %d sec\" +%%Y%%m%%d",
1284 cmd | getline lastfound; close(cmd)
1286 # avail=1 is the choice right now (8/19/05)
1287 # archived=0 is the choice right now (11/14/06)
1288 # sendgps=1 Remove non-soc caches from basic member. (12/18/2012)
1289 debug(5, gcid "\tsendgps: " sendgps " archived: " archived \
1290 " SOC: " SOC " soc: " soc " bid: " bid)
1291 if (!archived && (SOC || !soc) && (sendgps == 1 || bid) )
1294 printf("-dBID=%d\n", bid) > CIDFILE
1296 } else if (cid != 0) {
1297 printf("-dCID=%d\n", cid) > CIDFILE
1299 # GCID type vartime ifound soc iplaced tPLACED tFOUND tIFOUND
1301 #difficulty, terrain, container,
1302 printf "%s\t%s\t%d\t%s\t%s\t%s\t%d\t%d\t%d\t" \
1304 gcid, strtype, vartime,
1305 ifound, soc, iplaced, placedt, foundt, ifoundt,
1306 gctype, placer, "hint", lastfound >> XTRAFILE
1313 print "Archived ", gcid, " " name > "/dev/stderr"
1315 name = "error-no-name!"
1317 placer = "unknown-placer"
1318 # Use "merge" file output, NOT XTRAFILE!
1319 printf "%s\t%s\t0.0\t0.0\t%s%s%s\t" \
1320 "%.1f\t%.1f\t%s\t" \
1321 "%s\t%d\t%d\t%d\t%d\t%d\t%d\t%s\t%s\t%s\t%s\t%s\n",
1323 "http://www.geocaching.com/seek/cache_details.aspx",
1325 difficulty, terrain, container,
1327 ifound, soc, iplaced, placedt, foundt, ifoundt,
1328 gctype, placer, "hint", lastfound >> ARCHFILE
1337 # A temporary style we can use for merging the loc data with
1338 # the scraped html data. This is a dual purpose hack. We
1339 # use it as an output format to convert the .loc data to a
1340 # record-per-line format. We use it as an input format to
1341 # read up the merged data.
1343 make_scrape_style() {
1347 RECORD_DELIMITER NEWLINE
1349 IFIELD SHORTNAME, "", "%s"
1350 IFIELD DESCRIPTION, "", "%s"
1351 IFIELD LAT_DECIMAL, "", "%.6f"
1352 IFIELD LON_DECIMAL, "", "%.6f"
1353 IFIELD URL, "", "%s"
1354 IFIELD GEOCACHE_DIFF, "", "%3.1f" #difficulty
1355 IFIELD GEOCACHE_TERR, "", "%3.1f" #terrain
1356 IFIELD GEOCACHE_CONTAINER,"", "%s" #container (not set)
1357 IFIELD ICON_DESCR, "", "%s" #strtype (Geocache-*)
1358 IFIELD TIMET_TIME, "", "%ld" #variable time
1359 IFIELD IGNORE, "", "%s" #ifound
1360 IFIELD IGNORE, "", "%s" #soc
1361 IFIELD IGNORE, "", "%s" #iplaced
1362 IFIELD IGNORE, "", "%s" #placed time
1363 IFIELD IGNORE, "", "%s" #found time
1364 IFIELD IGNORE, "", "%s" #ifound time
1365 IFIELD GEOCACHE_TYPE, "", "%s" #gc.com type
1366 IFIELD GEOCACHE_PLACER, "", "%s" #gc.com placer
1367 IFIELD GEOCACHE_HINT, "", "%s" #gc.com hint
1368 IFIELD GEOCACHE_LAST_FOUND, "","%s" #gc.com last found
1369 OFIELD SHORTNAME, "", "%s"
1370 OFIELD DESCRIPTION, "", "%s"
1371 OFIELD LAT_DECIMAL, "", "%.6f"
1372 OFIELD LON_DECIMAL, "", "%.6f"
1373 OFIELD URL, "", "%s"
1374 OFIELD GEOCACHE_DIFF, "", "%3.1f" #difficulty
1375 OFIELD GEOCACHE_TERR, "", "%3.1f" #terrain
1376 OFIELD GEOCACHE_CONTAINER,"", "%s" #container (not set)
1378 # OFIELD ICON_DESCR, "", "%s"
1382 # csv2csv geo-mystery-file
1385 if [ ! -e "$1" ]; then
1386 error "Don't have a -M '$1' file"
1389 awk -v GEOMYSTERY=$1 '
1390 function latlon ( val ) {
1391 if (val ~ ".[.]..*[.].*")
1393 if (val ~ "[-wWsS]")
1395 val = substr(val, 2)
1398 else if (val ~ "[nNeE]")
1400 val = substr(val, 2)
1406 sub("[^.]*[.]", "", val)
1407 dd += (val+0.0) / 60.0
1408 return neg ? -dd : dd
1413 while (getline <GEOMYSTERY > 0)
1420 sub(".*/", "", gc) # remove stuff like http://coord.info/
1437 deglat = latlon( lat[$1] )
1438 deglon = latlon( lon[$1] )
1439 # GC# Desc lat lon URL diff terr size
1442 printf "%.6f\t%.6f\t", deglat, deglon
1443 for (i = 5; i < NF; ++i)
1454 # Fetch the cache patch
1456 gc_fetch_cache_page(){
1462 # Fetch the basic cache page
1463 echo "<BASE HREF=http://www.geocaching.com/seek/>" > $_page
1464 debug 1 "${_extra}curl $_url #cache fetch $_gcid.html"
1465 curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" \
1467 | tr -d "\001\007\010\013\017\020\031" \
1468 | sed -e '/<\/html>/d' \
1471 size=$(ls -l $_page | awk '{print $5}')
1472 if [ $size -lt 1000 ]; then
1473 debug 0 "Could not retrieve web page for cache $_gcid"
1479 tkn=`grep "userToken = '" $_page | sed -e "s/userToken = '//" -e "s/'.*//" `
1480 jsurl="http://www.geocaching.com/seek/geocache.logbook"
1481 jsurl="$jsurl?tkn=$tkn"
1482 jsurl="$jsurl&idx=1&num=$JSON_NUM&sp=false&sf=false&decrypt=false"
1483 debug 1 "curl $jsurl" >&2
1484 curl $CURL_OPTS -s -A "$UA" -b $COOKIE_FILE \
1487 echo "</html>" >> $_page
1492 # Query the gc website
1495 if [ $USERFOUND = 0 ]; then
1498 if [ $FOUND = 0 ]; then
1499 SEARCH="$SEARCH&f=1"
1501 if [ "$RADIUS_NUM" != "" ]; then
1502 SEARCH="$SEARCH&dist=$RADIUS_MILES"
1505 if [ $DEBUG -gt 0 ]; then
1519 MERGEFILE=$TMP.merge
1524 CRUFT="$CRUFT $HTMLPAGE"
1525 CRUFT="$CRUFT $CIDFILE"
1526 CRUFT="$CRUFT $LOCFILE"
1527 CRUFT="$CRUFT $LOCTMPFILE"
1528 CRUFT="$CRUFT $XTRAFILE"
1529 CRUFT="$CRUFT $CSVFILE"
1530 CRUFT="$CRUFT $CSVFILE2"
1531 CRUFT="$CRUFT $JOINFILE"
1532 CRUFT="$CRUFT $MERGEFILE"
1533 CRUFT="$CRUFT $ARCHFILE"
1534 CRUFT="$CRUFT $OUTWAY"
1535 CRUFT="$CRUFT $STYLE"
1536 if [ $NOCOOKIES = 1 ]; then
1537 CRUFT="$CRUFT $COOKIE_FILE"
1543 gc_login "$USERNAME" "$PASSWORD"
1548 if [ "$BOOKMARK" != "" ]; then
1549 URL="$GEO/bookmarks"
1550 debug 1 "$start: curl $URL #bookmark"
1552 curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" "$URL" \
1553 | grep -y ">$BOOKMARK<" \
1555 -e 's@. title.*@@' \
1556 -e 's@^.*href=.http://www.geocaching.com/@@' -e 's/.>.*$//'
1560 *) error "No bookmark with the name '$BOOKMARK'.";;
1565 # Find the pocket query
1567 if [ "$POCKETQUERY" != "" ]; then
1568 URL="$GEO/pocket/default.aspx"
1569 debug 1 "$start: curl $URL #pocket-query"
1571 curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" "$URL" \
1572 | grep -y "title=\"$POCKETQUERY\"" \
1574 -e 's@.*guid=@@' -e 's@".*@@'
1577 *-*-*-*-*) SEARCH="?pq=$SEARCH";;
1578 *) error "No PQ with the name '$POCKETQUERY'.";;
1583 # We might combine one or more pages into a single XML, so cobble
1584 # up a header with the ?xml and loc tags.
1586 cat <<-EOF > $LOCFILE
1587 <?xml version="1.0" encoding="UTF-8"?>
1588 <loc version="1.0" src="EasyGPS">
1592 # Loop, getting at least "NUM" locations
1595 if [ $DEBUG -gt 0 ]; then
1596 filter2="tee $TMP.bulk"
1605 while ((start <= NUM)); do
1609 # Fetch the page of closest caches and scrape the cache ID's
1616 URL="$GEO/seek/nearest.aspx"
1620 debug 1 "$start: curl $URL #list"
1621 if ((start > 0)); then
1622 # "postback"... grab the "next" button
1625 # __EVENTTARGET="ListInfo\$pgrBMItems\$_ctl08"
1626 # Get the last TGT (== ctl08)
1627 TGT=$($sed -n "s/^.*__doPostBack('.*pgrBMItems\$\(.*\)','.*/\1/p" \
1629 if [ "$TGT" = "" ]; then
1630 error "TGT is blank!"
1632 __EVENTTARGET="ctl00%24ContentBody%24ListInfo%24pgrBMItems%24$TGT"
1635 TGT=$($sed -n "s/^.*__doPostBack('.*pgrTop\$\(.*\)','.*/\1/p" \
1637 if [ "$TGT" = "" ]; then
1638 error "TGT is blank!"
1640 __EVENTTARGET="ctl00%24ContentBody%24pgrTop%24$TGT"
1643 curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" \
1644 -d __EVENTTARGET="$__EVENTTARGET" \
1647 | $sed -e "s/'/'/g" -e "s/\r//" > $HTMLPAGE
1649 curl $CURL_OPTS -L -s -b $COOKIE_FILE -A "$UA" \
1651 | $sed -e "s/'/'/g" -e "s/\r//" > $HTMLPAGE
1652 if [ "$DEBUG" -ge 1 ]; then
1653 grep "Total Records:.*Top.*" $HTMLPAGE |
1654 sed -e "s/<.b>.*//" -e "s/^.*span>//" -e "s/<b>//" 1>&2
1657 rc=$?; if [ $rc != 0 ]; then
1658 error "curl: fetch $URL"
1660 if grep -s -q "We encountered an error when requesting that page!" \
1662 error "searching error (1) on $start"
1664 if grep -s -q "has resulted in an error" \
1666 error "searching error (2) on $start"
1668 if grep -s -q "By State" $HTMLPAGE; then
1669 error "searching gave up on $start"
1671 if grep -s -q ">Advanced Search<" $HTMLPAGE; then
1672 error "need a country AND a state!"
1676 # Grab a few important values from the page
1678 gc_getviewstate $HTMLPAGE
1679 # cp $HTMLPAGE /tmp/view$start.html
1682 # Grab the CIDs into two categories: found and notfound
1685 debug 3 "getcids $HTMLPAGE $CIDFILE $XTRAFILE $ARCHFILE $((NUM-start))"
1686 getcids $HTMLPAGE $CIDFILE $XTRAFILE $ARCHFILE $((NUM-start))
1689 # Fetch the waypoints, rip out the ?xml and loc tags, and
1690 # append to the $LOCFILE file.
1692 if [ -s "$CIDFILE" ]; then
1699 URL="$GEO/seek/nearest.aspx"
1704 # Fetch the .loc file
1705 # 1) -d "Download=Download+Waypoints"... geo-nearest
1706 # 2) -d "ListInfo:btnDownload= ... old
1707 # 3) -d 'ctl00$ContentBody ... -b acro
1709 debug 2 "$start: curl $URL #loc"
1710 curl $CURL_OPTS -s -b $COOKIE_FILE -A "$UA" \
1713 -d "Download=Download+Waypoints" \
1714 -d "ListInfo:btnDownload=Download+to+.Loc" \
1715 -d 'ctl00$ContentBody$ListInfo$btnDownload=Download+to+.Loc' \
1718 | $sed -e 's/^<?xml [^>]*>//' \
1719 -e 's/>[gG]eocache</>Geocache</g' \
1720 -e 's/<loc [^>]*>//' \
1723 rc=$?; if [ $rc != 0 ]; then
1724 error "curl: fetch the waypoints"
1726 if grep -s -q "you are not logged in" $LOCTMPFILE; then
1727 error "you are not logged in on $start"
1729 if grep -s -q "has resulted in an error" $LOCTMPFILE; then
1730 error "searching error (3) on $start"
1732 if grep -s -q "Geocaching > Search for Geocaches" $LOCTMPFILE; then
1733 error "searching error (4) on $start"
1735 if grep -s -q "recaptcha_challenge_field" $LOCTMPFILE; then
1737 # Basic members: do it the SLOW way
1739 if [ "$GEODIR" != "" -a ! -d "$GEODIR" ]; then
1740 mkdir "$GEODIR" || error "Couldn't mkdir $GEODIR"
1742 if [ "$GEODIR/caches" != "" -a ! -d "$GEODIR/caches" ]; then
1743 mkdir "$GEODIR/caches" \
1744 || error "Couldn't mkdir $GEODIR/caches"
1747 if [ $subscriber = 1 ]; then
1748 debug 0 "Basic Member: pay the \$30/yr fee for Premium Member!"
1749 debug 0 "Basic Member: doing it the REAL SLOW way!!!"
1750 timestamp="$GEODIR/caches/.timestamp"
1751 $touch -d "1 day ago" $timestamp
1759 grep "^ *GC.*" $HTMLPAGE \
1760 | $sed -e "s/ //g" \
1761 | while read gcid; do
1763 # Get the cache page
1765 cachepage="$GEODIR/caches/$gcid.html"
1766 if [ ! -s $cachepage -o $timestamp -nt $cachepage ]; then
1767 # cache page doesn't exist or out of date
1770 urlpage="http://www.geocaching.com"
1771 urlpage="$urlpage/seek/cache_details.aspx"
1772 urlpage="$urlpage?log=y&wp=$gcid"
1773 gc_fetch_cache_page $cachepage $gcid $urlpage \
1775 if [ $? != 0 ]; then
1780 # 01/07/12: remove soc's!
1781 if grep -q 'alt="Premium Members only"' $cachepage; then
1784 # 12/18/12: remove soc's
1785 if grep -q 'uxPremiumSubmitBottom' $cachepage; then
1788 # 02/04/12: removed archived!
1789 if grep -q '>This cache has been archived' $cachepage; then
1794 # Cobble up the .loc file entry
1796 name=` grep -A1 "<head><title>" $cachepage \
1797 | tail -1 | tr -d "\r" \
1798 | $sed -e 's/[ ]*GC[^ ]* //' -e 's/ (.*by/ by/' `
1799 diff=` grep ctl00_ContentBody_uxLegendScale $cachepage \
1800 | $sed -e 's/.*alt="//' -e 's/ .*//' `
1801 terr=` grep ctl00_ContentBody_Localize6 $cachepage \
1802 | $sed -e 's/.*alt="//' -e 's/ .*//' `
1803 size=` grep "alt=.Size:" $cachepage \
1804 | $sed -e 's/.*alt="Size: //' -e 's/" .*//' `
1817 echo "<name id=\"$gcid\"><![CDATA[$name]]></name>"
1818 grep "lnkConversions" $cachepage \
1819 | $sed -e 's/.*lat=/<coord lat="/' \
1820 -e 's/&lon=/" lon="/' -e 's@&.*@"/>@'
1821 echo "<type>Geocache</type>"
1822 echo -n "<link text=\"Cache Details\">"
1823 urlpage="http://www.geocaching.com"
1824 urlpage="$urlpage/seek/cache_details.aspx"
1825 urlpage="$urlpage?wp=$gcid"
1828 echo "<difficulty>$diff</difficulty>"
1829 echo "<terrain>$terr</terrain>"
1830 echo "<container>$cont</container>"
1833 debug 3 "bstart: $bstart"
1834 if [ $bstart == $NUM ]; then
1841 cat $LOCTMPFILE >> $LOCFILE
1846 # Check to see if the user hasn't agreed to license terms
1848 if grep -s -q "lblAgreementText" $LOCFILE; then
1856 # If the Next button is disabled, break this loop
1857 # grep "Records: <b>[1-9].*disabled.><b>Next<" $HTMLPAGE
1858 if grep -s -q "Records: <b>[1-9].*disabled.><b>Next " $HTMLPAGE; then
1859 # echo "$start: dis"
1865 # Finish off the .loc file
1867 echo "</loc>" >> $LOCFILE
1870 # Convert the .loc data to .csv format and join it with
1871 # the extra data scraped from the HTML page. Filter out
1872 # the data according to the -I and -X options.
1874 # http://www.geocaching.com/seek/cache_details.aspx?wp=GCG2H4
1875 # http://www.geocaching.com/seek/cache_details.aspx?log=y&wp=GCG2H4
1876 # http://www.geocaching.com/seek/cache_details.aspx?ID=92117&log=y
1878 # The joined .csv format looks like this:
1879 # GCH636 Jidana 3 by rickrich 44.94520 -93.47540 \
1880 # http://www.geocaching.com/seek/cache_details.aspx?log=y&wp=GCH636 \
1881 # Geocache-ifound-regular 1070285077 1 0 1 \
1882 # 1067925600 1070285077 0
1884 make_scrape_style > $STYLE
1886 dbgcmd gpsbabel -i geo$GEONUKE -f $LOCFILE \
1887 -o xcsv,style=$STYLE -F $CSVFILE2
1888 if [ $? != 0 ]; then
1889 cp $LOCFILE /tmp/geo.err
1890 error "gpsbabel returned error code [1]"
1893 # Convert CSV into CSV with puzzles...
1894 csv2csv $GEOMYSTERY < $CSVFILE2 > $CSVFILE
1896 # 01/07/12: NO --nocheck-order; it is wrong!
1897 join -t ' ' $CSVFILE $XTRAFILE > $JOINFILE
1898 if [ $? != 0 ]; then
1899 cp $CSVFILE /tmp/geo.err.csv
1900 cp $XTRAFILE /tmp/geo.err.xtra
1901 error "joined file error: see join /tmp/geo.err.{csv,xtra}"
1903 egrep -- "$INCLUDE" < $JOINFILE \
1904 | egrep -v -- "$EXCLUDE" \
1905 | $sed -e 's/wp=/log=y\&&/' > $MERGEFILE
1907 if [ $FOUND = 1 ]; then
1909 | egrep -- "$INCLUDE" | egrep -v -- "$EXCLUDE" \
1910 | $sed -e 's/wp=/log=y\&&/' >> $MERGEFILE
1913 if [ $DEBUG -ge 2 ]; then
1914 # First two of these should be the same number of lines!
1917 wc -l $MERGEFILE >&2
1921 # Convert to the desired format
1924 if [ "$RADIUS" != "" ]; then
1925 BABELFILT="-x radius,distance=$RADIUS,lat=$LAT,lon=$LON"
1928 if [ $SQL = 1 ]; then
1932 if [ "$OUTFILE" != "" ]; then
1936 if [ $PURGE = 1 ]; then
1937 gpsdrive_purge | gpsdrive_mysql
1941 dbgcmd gpsbabel $BABELFLAGS \
1942 -i xcsv,style=$STYLE -f $MERGEFILE \
1943 $BABELFILT -o "$OUTFMT" -F $OUTWAY
1944 if [ $? != 0 ]; then
1945 error "gpsbabel returned error code [2]"
1947 gpsdrive_add <$OUTWAY $SQLTAG | gpsdrive_mysql
1948 elif [ $MAP = 1 ]; then
1949 dbgcmd gpsbabel $BABELFLAGS \
1950 -i xcsv,style=$STYLE -f $MERGEFILE \
1951 $BABELFILT -o "$OUTFMT" -F $OUTWAY
1952 if [ $? != 0 ]; then
1953 error "gpsbabel returned error code [3]"
1955 if [ "$OUTFILE" = "" ]; then
1956 dbgcmd geo-map -s0 $MAPOPTS -t$OUTWAY
1958 dbgcmd geo-map -s0 $MAPOPTS -t$OUTWAY -o"$OUTFILE"
1962 # output to stdout or to a file
1964 if [ "$OUTFILE" = "" ]; then
1965 OUTTMP="$TMP.way"; CRUFT="$CRUFT $OUTTMP"
1966 dbgcmd gpsbabel $BABELFLAGS \
1967 -i xcsv,style=$STYLE -f $MERGEFILE \
1968 $BABELFILT -o "$OUTFMT" -F $OUTTMP
1969 if [ $? != 0 ]; then
1970 error "gpsbabel returned error code [4]"
1974 dbgcmd gpsbabel $BABELFLAGS \
1975 -i xcsv,style=$STYLE -f $MERGEFILE \
1976 $BABELFILT -o "$OUTFMT" -F $OUTFILE
1977 if [ $? != 0 ]; then
1978 error "gpsbabel returned error code [5]"
1984 # Optionally, print the HTML pages
1986 if [ "$CMDPIPE" != "" ]; then
1989 while read id desc lat lon url diff terr container strtype \
1990 vartime ifound soc iplaced placedt foundt ifoundt extra; do
1991 url="$url&decrypt=y"
1994 CRUFT="$CRUFT $HTMLPAGE2"
1995 debug 1 "curl $url" >&2
1996 dbgcmd curl $CURL_OPTS -s -A "$UA" -b $COOKIE_FILE "$url" > $HTMLPAGE2
1997 htmldoc --quiet -t ps --nup 2 --fontsize 14 --webpage $HTMLPAGE2 \
1998 | psselect -q -p1-1 | eval $CMDPIPE
2005 # Optionally, fetch printable HTML pages
2007 if [ "$HTMLDIR" != "" -o "$LOGDIR" != "" ]; then
2008 if [ "$HTMLDIR" != "" -a ! -d "$HTMLDIR" ]; then
2009 mkdir "$HTMLDIR" || error "Couldn't mkdir $HTMLDIR"
2011 if [ "$LOGDIR" != "" -a ! -d "$LOGDIR" ]; then
2012 mkdir "$LOGDIR" || error "Couldn't mkdir $LOGDIR"
2016 CRUFT="$CRUFT $HTMLPAGE2"
2018 CRUFT="$CRUFT $TIMESTAMP"
2022 if [ $DEBUG -ge 3 ]; then
2027 while read id desc lat lon url diff terr container strtype \
2028 vartime ifound soc iplaced placedt foundt ifoundt extra; do
2030 # Don't fetch page if we already have a current version
2032 #debug 1 "placedt=<$placedt> <$foundt>"
2033 if [ $placedt -gt $foundt ]; then
2038 $touch -d "1/1/70 $filetime seconds last second" $TIMESTAMP
2039 if [ "$filetime" -gt 0 -a "$HTMLDIR/$id.html" -nt $TIMESTAMP ]; then
2044 if [ "$subscriber" == 0 -a "$HTMLDIR" != "" ]; then
2045 debug 2 "cp $GEODIR/caches/$id.html $HTMLDIR/$id.html"
2046 cp "$GEODIR/caches/$id.html" "$HTMLDIR/$id.html"
2047 # debug 2 "f: $filetime"
2048 if [ "$filetime" -gt 0 ]; then
2049 $touch -d "1/1/70 $filetime seconds" "$HTMLDIR/$id.html"
2054 # Limit to 1000 caches/day
2055 ((fetchcnt=fetchcnt+1))
2056 if [ $fetchcnt -gt $fetchmax ]; then
2057 error "Fetch count exceeded $fetchmax. Try tomorrow!"
2060 # Be kind to the server. Do not remove this sleep
2063 gc_fetch_cache_page $HTMLPAGE $id $url ""
2064 if [ $? != 0 ]; then
2068 if [ "$HTMLDIR" != "" ]; then
2069 cp $HTMLPAGE "$HTMLDIR/$id.html" ||
2070 error "Couldn't copy $id cache page"
2071 if [ "$filetime" -gt 0 ]; then
2072 $touch -d "1/1/70 $filetime seconds" "$HTMLDIR/$id.html"
2076 if [ "$LOGDIR" != "" ]; then
2077 $sed -e '1,/Logged Visits/d' \
2078 -e 's/<strong><img src=/\
2081 egrep ">$LOGUSERNAME<|strong> \($LOGUSERNAME\) \(" |
2082 $sed -e 's#<a href="log.aspx?LUID.*##' \
2083 -e 's#<a href=./profile[^ ]*##' \
2085 lynx -dump $HTMLPAGE2 > $LOGDIR/$id.log
2086 if [ "$filetime" -gt 0 ]; then
2087 $touch -d "1/1/70 $filetime seconds" "$LOGDIR/$id.log"
2093 if [ "$LOGDIR" != "" ]; then
2094 # This is a hack, and might be inaccurate
2095 echo -n "Finds:" >&2
2096 MONTHS=" January | February | March | April | May | June "
2097 MONTHS="$MONTHS| July | August | September | October "
2098 MONTHS="$MONTHS| November | December "
2099 egrep "$MONTHS" $LOGDIR/*.log |
2100 egrep "Found it |icon_smile|icon_happy|icon_camera" | wc -l >&2
2106 ##############################################################################
2107 # end #include "geo-common-gc"
2108 ##############################################################################
2109 ##############################################################################
2110 # begin #include "geo-common-gpsdrive"
2111 ##############################################################################
2114 # default MySQL global options...
2119 SQLITEDB=$HOME/.gpsdrive/waypoints.db
2123 # procedures for updating gpsdrive database via MySQL
2125 # Global Vars: $SQLDB, $SQLTAG $OUTFILE
2127 gpsdrive_create_210() {
2128 echo "CREATE TABLE poi ("
2129 echo "poi_id INTEGER PRIMARY KEY AUTOINCREMENT,"
2130 echo "name VARCHAR(80) NOT NULL default 'not specified',"
2131 echo "poi_type VARCHAR(160) NOT NULL default 'unknown',"
2132 echo "lat DOUBLE NOT NULL default '0',"
2133 echo "lon DOUBLE NOT NULL default '0',"
2134 echo "alt DOUBLE default '0',"
2135 echo "comment VARCHAR(255) default NULL,"
2136 echo "last_modified DATETIME NOT NULL default '0000-00-00',"
2137 echo "source_id INTEGER NOT NULL default '1',"
2138 echo "private CHAR(1) default NULL);"
2139 echo "CREATE TABLE poi_extra ("
2140 echo "poi_id INTEGER NOT NULL default '0',"
2141 echo "field_name VARCHAR(160) NOT NULL default '0',"
2142 echo "entry VARCHAR(8192) default NULL);"
2146 case "$GPSDRIVE_VER" in
2148 delcmd="delete from waypoints"
2150 echo "$delcmd where type like '$SQLTAG%';"
2153 if [ ! -s $SQLITEDB ]; then
2156 SQLTAG=`echo $SQLTAG | tr A-Z a-z`
2157 delcmd="delete from poi"
2158 echo "$delcmd where poi_type like '$SQLTAG%';"
2163 gpsdrive_add_209() {
2164 delcmd="delete from waypoints"
2165 addcmd="replace into waypoints (name,lat,lon,type)"
2166 addcmd="insert into waypoints (name,lat,lon,type)"
2169 while read name lat lon type extra
2171 name=`echo "$name" | tr -d "'"`
2172 # Primary key is autoincrementing id number, so delete
2173 # the old record (if any) by name and type
2174 if [ $PURGE = 0 ]; then
2175 echo "$delcmd where name='$name' and type like '$SQLTAG%';"
2178 if [ $DELETE = 0 ]; then
2179 # Add the new record
2180 if [ "$sqltag" = "Geocache" ]; then
2185 echo "$addcmd values ('$name','$lat','$lon','$tag');"
2191 if [ ! -s $SQLITEDB ]; then
2194 echo "PRAGMA synchronous=OFF;"
2195 delcmd="delete from poi"
2196 addcmd="replace into poi (name,poi_type,lat,lon,alt,comment,last_modified)"
2197 addcmd="insert into poi (name,poi_type,lat,lon,alt,comment,last_modified)"
2199 sqltag=`echo "$sqltag" | tr A-Z a-z`
2202 #while read name lat lon type extra
2204 egrep -v 'Geocache Found|-ifound' \
2205 | tr -d "|" | tr ' ' '|' | tr -d "'" |
2206 while read index shortname description notes url urltext icon lat lon \
2207 lat32 lon32 latdecdir londecdir latdirdec londirdec latdir londir \
2208 altfeet altmeters excel timet diff terr container type \
2209 pathmiles pathkm placer yyyymmmdd hint lastfound extra
2211 if [ "$placer" = "$USERNAME" ]; then
2214 if [ "$placer" = "dyl1231" ]; then
2218 #name=`echo "$description" | tr -d "'"`
2220 name=${description//\'/}
2222 # Primary key is autoincrementing id number, so delete
2223 # the old record (if any) by name and type
2224 if [ $PURGE = 0 ]; then
2225 echo "$delcmd where name='$name' and poi_type like '%$sqltag%';"
2229 if [ $DELETE = 0 ]; then
2230 # Add the new record
2231 if [ "$sqltag" = "geocache" ]; then
2237 "Earthcache") poi_type="geocache.geocache_earth";;
2238 "Event Cache") poi_type="geocache.geocache_event";;
2239 "Mega-Event Cache") poi_type="geocache.geocache_event";;
2240 "Cache In Trash Out Event") poi_type="geocache.geocache_event";;
2241 "found") poi_type="geocache.geocache_found";;
2242 "Multi-cache") poi_type="geocache.geocache_multi";;
2243 "Unknown Cache") poi_type="geocache.geocache_mystery";;
2244 "night") poi_type="geocache.geocache_night";;
2245 "Traditional Cache") poi_type="geocache.geocache_traditional";;
2246 "Virtual Cache") poi_type="geocache.geocache_virtual";;
2247 "Webcam Cache") poi_type="geocache.geocache_webcam";;
2248 *) poi_type="geocache";;
2250 echo -n "$addcmd values ('$name','$poi_type',"
2251 echo -n "'$lat','$lon',"
2252 echo -n "'0.0'," #alt
2253 echo -n "'$type $container $diff/$terr. Last Found: $lastfound. $hint',"
2261 case "$GPSDRIVE_VER" in
2262 ""|"2.09") gpsdrive_add_209 "$1";;
2263 "2.10") gpsdrive_add_210 "$1";;
2268 if [ "$OUTFILE" != "" ]; then
2270 elif [ $DEBUG -gt 0 ]; then
2273 mysql -u$SQLUSER -p$SQLPASS
2277 gpsdrive_sqlite3() {
2278 if [ "$OUTFILE" != "" ]; then
2280 elif [ $DEBUG -gt 0 ]; then
2283 sqlite3 $HOME/.gpsdrive/waypoints.db
2288 # Extended list of gpsbabel output formats
2290 gpsbabel_formats() {
2291 gpsbabel -? | sed -e '1,/File Types/d' -e '/Supported data filters/,$d'
2292 echo " gpsdrive.sql " \
2293 "GpsDrive direct MySQL database insertion"
2294 echo " map[,geo-map-opts] " \
2295 "Display map of waypoints using geo-map"
2298 ##############################################################################
2299 # end #include "geo-common-gpsdrive"
2300 ##############################################################################
2303 # Set default options, can be overriden on command line or in rc file
2305 UPDATE_URL=$WEBHOME/geo-nearest
2306 UPDATE_FILE=geo-nearest.new
2311 # Process the options
2322 # Cut and paste from geocaching.com cache page
2323 # N 44° 58.630 W 093° 09.310
2324 LAT=`echo "$1$2.$3" | tr -d '\260\302' `
2326 LON=`echo "$4$5.$6" | tr -d '\260\302' `
2328 SEARCH="?origin_lat=$LAT&origin_long=$LON"
2331 # lat lon cache-type
2335 webcam) cFilter="31d2ae3c-c358-4b5f-8dcd-2185bf472d3d";;
2336 earth*) cFilter="c66f5cf3-9523-4549-b8dd-759cd2f18db8";;
2337 multi*) cFilter="a5f6d0ad-d2f2-4011-8c14-940a9ebf3c74";;
2338 event) cFilter="69eb8534-b718-4b35-ae3c-a856a55b0874";;
2339 virtual) cFilter="294d4360-ac86-4c83-84dd-8113ef678d7e";;
2340 letter*) cFilter="4bdd8fb2-d7bc-453f-a9c5-968563b15d24";;
2341 unknown) cFilter="40861821-1835-4e11-b666-8d41064d03fe";;
2342 trad*) cFilter="32bc9333-5e52-4957-b0f6-5a2c8fc7b257";;
2343 reg*) cFilter="32bc9333-5e52-4957-b0f6-5a2c8fc7b257";;
2344 *) error "Unknown cache-type '$3'";;
2346 SEARCH="?lat=$LAT&lng=$LON&cFilter=$cFilter"
2351 SEARCH="?origin_lat=$LAT&origin_long=$LON"
2355 iraq|Iraq) SEARCH="?country_id=97" ;;
2366 # tx=NNN&ul=USERNAME
2373 tx=webcam) tx="tx=31d2ae3c-c358-4b5f-8dcd-2185bf472d3d";;
2374 tx=earth*) tx="tx=c66f5cf3-9523-4549-b8dd-759cd2f18db8";;
2375 tx=multi*) tx="tx=a5f6d0ad-d2f2-4011-8c14-940a9ebf3c74";;
2376 tx=event) tx="tx=69eb8534-b718-4b35-ae3c-a856a55b0874";;
2377 tx=virtual) tx="tx=294d4360-ac86-4c83-84dd-8113ef678d7e";;
2378 tx=letter*) tx="tx=4bdd8fb2-d7bc-453f-a9c5-968563b15d24";;
2379 tx=unknown) tx="tx=40861821-1835-4e11-b666-8d41064d03fe";;
2380 tx=trad*) tx="tx=32bc9333-5e52-4957-b0f6-5a2c8fc7b257";;
2381 tx=reg*) tx="tx=32bc9333-5e52-4957-b0f6-5a2c8fc7b257";;
2384 SEARCH="?$tx&lat=$LAT&lng=$LON"
2387 SEARCH="bookmarks/view.aspx?$1"
2394 error "'$1' isn't something I understand how to search for"
2399 SEARCH="?origin_lat=$LAT&origin_long=$LON"